From d7cf898e9007ba11ecdd9691d8547a752b2fe9e0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 9 Sep 2026 10:37:13 +0000 Subject: [PATCH 001/196] feat(operator): add LPX integration Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 6 +- .../components/operator/files/role.yaml | 22 + .../operator/templates/_validation.tpl | 10 + .../operator/templates/deployment.yaml | 7 + .../operator/templates/operator-config.yaml | 10 + .../platform/components/operator/values.yaml | 13 + .../platform/templates/validate-values.yaml | 20 + .../platform/tests/lpx_config_test.yaml | 56 + .../tests/validate_removed_values_test.yaml | 16 + deploy/helm/charts/platform/values.yaml | 14 + deploy/operator/Makefile | 13 +- deploy/operator/api/config/v1alpha1/lpx.go | 18 + deploy/operator/api/config/v1alpha1/types.go | 15 + .../config/v1alpha1/zz_generated.deepcopy.go | 32 + .../api/config/validation/validation.go | 7 + .../api/config/validation/validation_test.go | 35 + .../operator/api/conversion_webhook_test.go | 30 +- deploy/operator/api/roundtrip_fuzz_test.go | 11 +- deploy/operator/api/scripts/lpx-crd-schema.yq | 31 + deploy/operator/api/v1alpha1/common.go | 6 +- .../conversion_field_coverage_test.go | 23 +- ...namocomponentdeployment_conversion_test.go | 1 + .../dynamocomponentdeployment_types.go | 52 +- .../dynamographdeployment_conversion.go | 26 +- .../dynamographdeployment_conversion_test.go | 163 +- .../v1alpha1/dynamographdeployment_types.go | 45 +- .../dynamographdeployment_types_test.go | 48 + .../api/v1alpha1/groupversion_info.go | 5 + .../api/v1alpha1/lpx_conversion_test.go | 232 + deploy/operator/api/v1alpha1/lpx_types.go | 11 + .../api/v1alpha1/lpxgraphdeployment_types.go | 70 + .../api/v1alpha1/shared_spec_conversion.go | 50 +- .../shared_spec_conversion_bugs_test.go | 93 + .../api/v1alpha1/zz_generated.deepcopy.go | 154 +- deploy/operator/api/v1beta1/common.go | 30 +- .../dynamocomponentdeployment_types.go | 43 +- .../v1beta1/dynamographdeployment_types.go | 13 +- deploy/operator/api/v1beta1/lpx_types.go | 98 + .../api/v1beta1/zz_generated.deepcopy.go | 162 +- deploy/operator/cmd/crd-apply/main.go | 4 +- deploy/operator/cmd/crd-apply/main_test.go | 137 + deploy/operator/cmd/crd-apply/schema_test.go | 168 + deploy/operator/cmd/main.go | 6 + ...nvidia.com_dynamocomponentdeployments.yaml | 67 +- .../nvidia.com_dynamographdeployments.yaml | 12793 +--------------- .../bases/nvidia.com_lpxgraphdeployments.yaml | 341 + deploy/operator/docs/crd-ref-docs-config.yaml | 2 + deploy/operator/docs/fix-api-anchors.py | 5 +- deploy/operator/go.mod | 10 +- deploy/operator/go.sum | 22 +- deploy/operator/internal/consts/lpx.go | 15 + .../internal/controller/dgd_grove_program.go | 21 +- .../controller/dgd_grove_watch_setup.go | 71 +- .../controller/dgd_grove_watch_setup_test.go | 112 + .../controller/dgd_grove_workload_renderer.go | 97 +- .../dgd_grove_workloads_reconciler.go | 25 +- .../dgd_grove_workloads_reconciler_test.go | 14 +- .../controller/dgd_lpx_envtest_test.go | 110 + .../internal/controller/dgd_lpx_handoff.go | 234 + .../dgd_lpx_handoff_envtest_test.go | 263 + .../controller/dgd_lpx_handoff_test.go | 689 + .../dgd_scaling_adapters_reconciler.go | 30 +- .../controller/dgd_ssh_keys_reconciler.go | 2 +- .../dgd_worker_rollout_reconciler_test.go | 12 + .../controller/dgd_workload_program.go | 7 +- .../dynamocomponentdeployment_controller.go | 7 +- ...namocomponentdeployment_controller_test.go | 9 +- .../dynamographdeployment_controller.go | 16 +- .../dynamographdeployment_controller_test.go | 303 +- ...dynamographdeploymentrequest_controller.go | 8 +- .../controller/lpx/attempt_deadline.go | 460 + .../controller/lpx/attempt_deadline_test.go | 303 + .../controller/lpx/component_roles_test.go | 109 + .../internal/controller/lpx/controller.go | 424 + .../controller/lpx/controller_test.go | 632 + .../internal/controller/lpx/dependencies.go | 154 + .../internal/controller/lpx/eviction.go | 264 + .../internal/controller/lpx/eviction_test.go | 368 + .../operator/internal/controller/lpx/grove.go | 105 + .../internal/controller/lpx/lifecycle.go | 1598 ++ .../internal/controller/lpx/lifecycle_test.go | 2459 +++ .../internal/controller/lpx/modeldownload.go | 170 + .../controller/lpx/modeldownload_test.go | 355 + .../internal/controller/lpx/preflight.go | 141 + .../internal/controller/lpx/render.go | 117 + .../internal/controller/lpx/render_test.go | 1595 ++ .../operator/internal/controller/lpx/watch.go | 292 + .../internal/controller/lpx/watch_test.go | 406 + .../controller/primary_resource_predicate.go | 18 +- .../primary_resource_predicate_test.go | 36 +- deploy/operator/internal/controller/setup.go | 4 + .../internal/controller/upgrade_test.go | 5 + .../internal/controller_common/resource.go | 64 +- .../controller_common/resource_test.go | 34 +- .../controller_common/workload_predicates.go | 89 + deploy/operator/internal/dynamo/graph.go | 361 +- deploy/operator/internal/dynamo/graph_lpx.go | 507 + .../internal/dynamo/graph_lpx_test.go | 375 + deploy/operator/internal/dynamo/graph_test.go | 180 +- deploy/operator/internal/dynamo/grove.go | 212 +- deploy/operator/internal/dynamo/grove_lpx.go | 233 + .../internal/dynamo/grove_lpx_test.go | 96 + deploy/operator/internal/dynamo/grove_test.go | 591 +- deploy/operator/internal/dynamo/lpx/build.go | 142 + .../internal/dynamo/lpx/build_snapshot.go | 239 + .../dynamo/lpx/build_snapshot_gcs_test.go | 119 + .../dynamo/lpx/build_snapshot_test.go | 138 + .../internal/dynamo/lpx/build_test.go | 51 + .../lpx/commands/lpu_partition_agent.sh | 165 + .../dynamo/lpx/commands/lpu_partition_env.sh | 6 + .../lpx/commands/lpu_partition_metadata.sh | 71 + .../dynamo/lpx/commands/lpu_system_init.sh | 49 + .../dynamo/lpx/commands/lpu_worker.sh | 42 + .../internal/dynamo/lpx/commands/ssh_setup.sh | 15 + .../dynamo/lpx/compiler_fixture_test.go | 286 + .../internal/dynamo/lpx/conductor_runtime.go | 140 + .../internal/dynamo/lpx/cyborg_render.go | 74 + .../internal/dynamo/lpx/cyborg_render_test.go | 131 + .../internal/dynamo/lpx/cyborg_resources.go | 70 + .../dynamo/lpx/cyborg_resources_test.go | 65 + .../internal/dynamo/lpx/cyborg_runtime.go | 155 + .../dynamo/lpx/cyborg_runtime_test.go | 97 + deploy/operator/internal/dynamo/lpx/doc.go | 8 + .../dynamo/lpx/hybrid_cyborg_bindings.go | 176 + .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 83 + .../internal/dynamo/lpx/lpu_commands.go | 145 + .../internal/dynamo/lpx/lpu_commands_test.go | 124 + .../internal/dynamo/lpx/lpu_configmap.go | 427 + .../internal/dynamo/lpx/lpu_configmap_test.go | 206 + .../operator/internal/dynamo/lpx/lpu_pod.go | 86 + .../internal/dynamo/lpx/materialization.go | 228 + .../dynamo/lpx/materialization_test.go | 66 + .../internal/dynamo/lpx/model_fixture_test.go | 203 + .../dynamo/lpx/model_manifest_common.go | 288 + .../dynamo/lpx/model_manifest_common_test.go | 113 + .../internal/dynamo/lpx/model_manifest_v2.go | 493 + .../dynamo/lpx/model_manifest_v2_test.go | 374 + .../internal/dynamo/lpx/model_projection.go | 150 + .../internal/dynamo/lpx/model_registry.go | 366 + .../dynamo/lpx/model_registry_test.go | 360 + .../internal/dynamo/lpx/modelexpress.go | 61 + .../internal/dynamo/lpx/modelexpress_test.go | 62 + .../internal/dynamo/lpx/pod_mutation.go | 55 + .../operator/internal/dynamo/lpx/pod_roles.go | 86 + .../internal/dynamo/lpx/pod_roles_test.go | 465 + deploy/operator/internal/dynamo/lpx/render.go | 322 + .../internal/dynamo/lpx/render_test.go | 364 + .../dynamo/lpx/resolve_selected_workload.go | 190 + .../lpx/resolve_selected_workload_test.go | 320 + .../internal/dynamo/lpx/runtime_agent.go | 446 + .../internal/dynamo/lpx/runtime_settings.go | 201 + .../dynamo/lpx/runtime_settings_test.go | 368 + .../internal/dynamo/lpx/runtime_shape.go | 15 + .../internal/dynamo/lpx/runtime_ssh.go | 76 + .../internal/dynamo/lpx/selected_prop_sync.go | 134 + .../internal/dynamo/lpx/selected_workload.go | 52 + .../operator/internal/dynamo/lpx/selection.go | 89 + .../internal/dynamo/lpx/selection_test.go | 117 + .../dynamo/lpx/selection_validation.go | 241 + .../operator/internal/dynamo/lpx/topology.go | 94 + .../internal/dynamo/lpx/topology_test.go | 179 + .../dynamo/lpx/workload_projection.go | 219 + .../dynamo/lpx/workload_projection_test.go | 120 + .../dynamo/lpx/workload_projection_v2.go | 238 + .../dynamo/lpx/workload_projection_v2_test.go | 358 + .../dynamo/lpx/workload_projection_v3.go | 166 + .../dynamo/lpx/workload_projection_v3_test.go | 327 + deploy/operator/internal/dynamo/patch.go | 129 + deploy/operator/internal/dynamo/patch_test.go | 294 + .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 230 + .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 975 ++ .../node-local-v2-hybrid.input.yaml | 59 + .../from_dgd_yaml/node-local-v2-hybrid.yaml | 436 + .../node-local-v2-lpu-only.input.yaml | 37 + .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 432 + .../node-local-v2-specdecode.input.yaml | 56 + .../node-local-v2-specdecode.yaml | 1061 ++ .../node-local-v3-hx-hybrid.input.yaml | 53 + .../node-local-v3-hx-hybrid.yaml | 603 + .../node-local-v3-hx-lpu-only.input.yaml | 36 + .../node-local-v3-hx-lpu-only.yaml | 551 + .../node-local-v3-hx-specdecode.input.yaml | 62 + .../node-local-v3-hx-specdecode.yaml | 1083 ++ .../from_dgd_yaml/single_v2.input.yaml | 47 + .../testdata/from_dgd_yaml/single_v2.yaml | 604 + .../single_v2_manifest_defaults.input.yaml | 28 + deploy/operator/internal/features/gates.go | 9 + .../operator/internal/features/gates_test.go | 1 + deploy/operator/internal/features/lpx.go | 39 + .../observability/resource_counter.go | 8 +- .../observability/resource_counter_test.go | 46 + .../operator/internal/podcache/transform.go | 18 +- .../internal/podcache/transform_test.go | 12 +- .../internal/thirdparty/capnp/LICENSE | 201 + .../v2/gbuild_manifest.capnp.go | 2656 ++++ .../internal/thirdparty/lpxscheduler/LICENSE | 202 + .../lpxscheduler/v1alpha1/controller_gen.go | 6 + .../thirdparty/lpxscheduler/v1alpha1/types.go | 1627 ++ .../dynamographdeployment_handler.go | 1 - ...onentdeployment_validation_envtest_test.go | 29 +- .../validation/dynamographdeployment.go | 80 +- .../dynamographdeployment_handler_test.go | 10 +- .../dynamographdeployment_helpers.go | 24 - ...hdeployment_lpx_validation_envtest_test.go | 473 + ...graphdeployment_validation_envtest_test.go | 73 +- ...ploymentrequest_validation_envtest_test.go | 1 - .../webhook/validation/shared_v1alpha1.go | 4 +- .../webhook/validation/shared_v1beta1.go | 87 +- .../webhook/validation/suite_envtest_test.go | 60 +- .../model-deployment/deploy-with-dgd.md | 29 + .../additional-resources/api-reference-k8s.md | 276 +- .../dynamo-component-deployment.mdx | 32 +- .../dynamo-graph-deployment.mdx | 26 +- .../kubernetes-api/full-api-reference.mdx | 372 +- docs/fern/scripts/kubernetes_api_rendering.py | 10 +- .../scripts/tests/test_gen_kubernetes_api.py | 18 +- 216 files changed, 40659 insertions(+), 13911 deletions(-) create mode 100644 deploy/helm/charts/platform/tests/lpx_config_test.yaml create mode 100644 deploy/operator/api/config/v1alpha1/lpx.go create mode 100644 deploy/operator/api/scripts/lpx-crd-schema.yq create mode 100644 deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go create mode 100644 deploy/operator/api/v1alpha1/lpx_conversion_test.go create mode 100644 deploy/operator/api/v1alpha1/lpx_types.go create mode 100644 deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go create mode 100644 deploy/operator/api/v1beta1/lpx_types.go create mode 100644 deploy/operator/cmd/crd-apply/main_test.go create mode 100644 deploy/operator/cmd/crd-apply/schema_test.go create mode 100644 deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml create mode 100644 deploy/operator/internal/consts/lpx.go create mode 100644 deploy/operator/internal/controller/dgd_grove_watch_setup_test.go create mode 100644 deploy/operator/internal/controller/dgd_lpx_envtest_test.go create mode 100644 deploy/operator/internal/controller/dgd_lpx_handoff.go create mode 100644 deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go create mode 100644 deploy/operator/internal/controller/dgd_lpx_handoff_test.go create mode 100644 deploy/operator/internal/controller/lpx/attempt_deadline.go create mode 100644 deploy/operator/internal/controller/lpx/attempt_deadline_test.go create mode 100644 deploy/operator/internal/controller/lpx/component_roles_test.go create mode 100644 deploy/operator/internal/controller/lpx/controller.go create mode 100644 deploy/operator/internal/controller/lpx/controller_test.go create mode 100644 deploy/operator/internal/controller/lpx/dependencies.go create mode 100644 deploy/operator/internal/controller/lpx/eviction.go create mode 100644 deploy/operator/internal/controller/lpx/eviction_test.go create mode 100644 deploy/operator/internal/controller/lpx/grove.go create mode 100644 deploy/operator/internal/controller/lpx/lifecycle.go create mode 100644 deploy/operator/internal/controller/lpx/lifecycle_test.go create mode 100644 deploy/operator/internal/controller/lpx/modeldownload.go create mode 100644 deploy/operator/internal/controller/lpx/modeldownload_test.go create mode 100644 deploy/operator/internal/controller/lpx/preflight.go create mode 100644 deploy/operator/internal/controller/lpx/render.go create mode 100644 deploy/operator/internal/controller/lpx/render_test.go create mode 100644 deploy/operator/internal/controller/lpx/watch.go create mode 100644 deploy/operator/internal/controller/lpx/watch_test.go create mode 100644 deploy/operator/internal/controller_common/workload_predicates.go create mode 100644 deploy/operator/internal/dynamo/graph_lpx.go create mode 100644 deploy/operator/internal/dynamo/graph_lpx_test.go create mode 100644 deploy/operator/internal/dynamo/grove_lpx.go create mode 100644 deploy/operator/internal/dynamo/grove_lpx_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/build.go create mode 100644 deploy/operator/internal/dynamo/lpx/build_snapshot.go create mode 100644 deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/build_snapshot_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/build_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh create mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh create mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh create mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh create mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh create mode 100644 deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh create mode 100644 deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/conductor_runtime.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_render.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_render_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_resources.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_resources_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_runtime.go create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/doc.go create mode 100644 deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go create mode 100644 deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_commands.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_commands_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_configmap.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_pod.go create mode 100644 deploy/operator/internal/dynamo/lpx/materialization.go create mode 100644 deploy/operator/internal/dynamo/lpx/materialization_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_fixture_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_manifest_common.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_manifest_common_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_manifest_v2.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_projection.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_registry.go create mode 100644 deploy/operator/internal/dynamo/lpx/model_registry_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/modelexpress.go create mode 100644 deploy/operator/internal/dynamo/lpx/modelexpress_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/pod_mutation.go create mode 100644 deploy/operator/internal/dynamo/lpx/pod_roles.go create mode 100644 deploy/operator/internal/dynamo/lpx/pod_roles_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/render.go create mode 100644 deploy/operator/internal/dynamo/lpx/render_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go create mode 100644 deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_agent.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_settings.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_settings_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_shape.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_ssh.go create mode 100644 deploy/operator/internal/dynamo/lpx/selected_prop_sync.go create mode 100644 deploy/operator/internal/dynamo/lpx/selected_workload.go create mode 100644 deploy/operator/internal/dynamo/lpx/selection.go create mode 100644 deploy/operator/internal/dynamo/lpx/selection_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/selection_validation.go create mode 100644 deploy/operator/internal/dynamo/lpx/topology.go create mode 100644 deploy/operator/internal/dynamo/lpx/topology_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection_v2.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection_v3.go create mode 100644 deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go create mode 100644 deploy/operator/internal/dynamo/patch.go create mode 100644 deploy/operator/internal/dynamo/patch_test.go create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml create mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml create mode 100644 deploy/operator/internal/features/lpx.go create mode 100644 deploy/operator/internal/thirdparty/capnp/LICENSE create mode 100644 deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go create mode 100644 deploy/operator/internal/thirdparty/lpxscheduler/LICENSE create mode 100644 deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go create mode 100644 deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index fd7b93a021cd..b07982c39e81 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -162,14 +162,16 @@ Kubernetes: `>=1.30.0-0` | global.kai-scheduler.install | bool | `false` | Whether this chart should install the bundled kai-scheduler subchart. When true, deploys kai-scheduler and its CRDs. Integration is automatically enabled. NOTE: For production environments, it is recommended to install kai-scheduler separately. | | global.kai-scheduler.enabled | bool | `false` | Whether to enable Kai Scheduler integration (queue creation, schedulerName injection). Set to true when kai-scheduler is available in the cluster (installed externally). Automatically enabled when install=true. The operator uses this to decide whether to inject schedulerName and queue labels into pod templates. | | global.volcano-scheduler.enabled | bool | `false` | EXPERIMENTAL: Whether to enable Volcano scheduler integration for Grove PodCliqueSets. Set to true when Volcano is available in the cluster and Grove is configured with Volcano scheduler support. The operator uses this to inject schedulerName and map nvidia.com/volcano-queue to Grove's Volcano queue annotation. | -| global.grove.install | bool | `false` | Whether this chart should install the bundled Grove subchart. When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. NOTE: For production environments, it is recommended to install Grove separately. | -| global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | +| global.grove.install | bool | `false` | Whether this chart should install the bundled Grove subchart. When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. The bundled version cannot be used with LPX; LPX requires an externally installed scheduler-scoped Grove build that supports disjoint schedulers in one PodCliqueSet. NOTE: For production environments, it is recommended to install Grove separately. | +| global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). LPX users must provide the scheduler-scoped Grove build and enable its lpx-scheduler profile. Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | | global.snapshot.install | bool | `false` | Whether this chart should install the bundled Snapshot subchart (github.com/ai-dynamo/snapshot): its operator, node agent, and CRDs. NOTE: For production environments, it is recommended to install Snapshot separately. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | +| dynamo-operator.lpx.enabled | bool | `false` | Enable LPX integration. Startup verifies that the externally installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails otherwise. | +| dynamo-operator.lpx.modelRegistryURL | string | `""` | Configure the LPU model registry used by the LPX integration. | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. | | dynamo-operator.namespaceRestriction.targetNamespace | string | `nil` | DEVELOPMENT AND TESTING ONLY: Target namespace. Defaults to the Helm release namespace. | diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 086eaf818fa2..9c2eab7c0113 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -254,6 +254,7 @@ rules: - dynamographdeployments - dynamographdeploymentscalingadapters - dynamomodels + - lpxgraphdeployments verbs: - create - delete @@ -269,6 +270,7 @@ rules: - dynamographdeploymentrequests/finalizers - dynamographdeployments/finalizers - dynamomodels/finalizers + - lpxgraphdeployments/finalizers verbs: - update - apiGroups: @@ -279,6 +281,7 @@ rules: - dynamographdeployments/status - dynamographdeploymentscalingadapters/status - dynamomodels/status + - lpxgraphdeployments/status verbs: - get - patch @@ -354,6 +357,25 @@ rules: - patch - update - watch +- apiGroups: + - scheduler.grove.io + resources: + - podgangs + verbs: + - get + - list + - watch +- apiGroups: + - scheduling.lpu.nvidia.com + resources: + - lpupipelinerequests + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - scheduling.run.ai resources: diff --git a/deploy/helm/charts/platform/components/operator/templates/_validation.tpl b/deploy/helm/charts/platform/components/operator/templates/_validation.tpl index 829733970a5e..a76020aa934f 100644 --- a/deploy/helm/charts/platform/components/operator/templates/_validation.tpl +++ b/deploy/helm/charts/platform/components/operator/templates/_validation.tpl @@ -117,3 +117,13 @@ Validation for discoveryBackend configuration {{- fail (printf "VALIDATION ERROR: discoveryBackend must be 'kubernetes' (default) or 'etcd'. Got: '%s'" $discoveryBackend) -}} {{- end -}} {{- end -}} + +{{/* +Validation for defaultExtraPodSpecMergeStrategy configuration +*/}} +{{- define "dynamo-operator.validateExtraPodSpecMergeStrategy" -}} +{{- $strategy := default "override" .Values.controllerManager.manager.defaultExtraPodSpecMergeStrategy -}} +{{- if and (ne $strategy "override") (ne $strategy "strategic") -}} + {{- fail (printf "VALIDATION ERROR: controllerManager.manager.defaultExtraPodSpecMergeStrategy must be 'override' or 'strategic'. Got: '%s'" $strategy) -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index fc5e976a60ca..79d53e438f73 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -17,6 +17,7 @@ {{- include "dynamo-operator.validateClusterWideInstallation" . -}} {{- include "dynamo-operator.validateConfiguration" . -}} {{- include "dynamo-operator.validateDiscoveryBackend" . -}} +{{- include "dynamo-operator.validateExtraPodSpecMergeStrategy" . -}} --- apiVersion: apps/v1 @@ -136,6 +137,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + {{- with .Values.controllerManager.manager.volumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} securityContext: runAsNonRoot: true serviceAccountName: {{ include "dynamo-operator.fullname" . }}-controller-manager @@ -151,3 +155,6 @@ spec: # Allow the pod to start before the secret exists so the # in-operator cert-controller can bootstrap it at runtime. optional: true + {{- with .Values.controllerManager.manager.volumes }} + {{- toYaml . | nindent 6 }} + {{- end }} diff --git a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml index a03246322916..1139fa4069d4 100644 --- a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml @@ -205,6 +205,11 @@ data: {{- end }} {{- end }} {{- end }} + {{- $defaultExtraPodSpecMergeStrategy := default "override" .Values.controllerManager.manager.defaultExtraPodSpecMergeStrategy }} + {{- if ne ($defaultExtraPodSpecMergeStrategy | toString) "override" }} + podGeneration: + defaultExtraPodSpecMergeStrategy: {{ $defaultExtraPodSpecMergeStrategy | quote }} + {{- end }} {{- if and .Values.discoveryBackend (ne (.Values.discoveryBackend | toString) "kubernetes") }} discovery: backend: {{ .Values.discoveryBackend | quote }} @@ -213,3 +218,8 @@ data: gpu: discoveryEnabled: {{ .Values.gpuDiscovery.enabled }} {{- end }} + lpx: + enabled: {{ .Values.lpx.enabled }} + {{- with .Values.lpx.modelRegistryURL }} + modelRegistryURL: {{ . | quote }} + {{- end }} diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 0d0adfdd0d23..7fb30cf2ef58 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -22,6 +22,16 @@ # the cluster-wide operator. upgradeCRD: true +# LPX is an externally installed scheduler integration. Enabling it only +# configures the Dynamo operator; this chart never installs LPX resources. +lpx: + # -- Enable node-local LpuPipelineRequest production. Startup verifies that the + # externally installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 + # LpuPipelineRequest resources and fails otherwise. + enabled: false + # -- Configure the LPU model registry used by the LPX integration. + modelRegistryURL: "" + # Environment variables to pass to operator Deployment. env: [] @@ -70,6 +80,7 @@ controllerManager: namespace: "" manager: + defaultExtraPodSpecMergeStrategy: override containerSecurityContext: allowPrivilegeEscalation: false capabilities: @@ -90,6 +101,8 @@ controllerManager: requests: cpu: 512m memory: 1Gi + volumeMounts: [] + volumes: [] replicas: 1 serviceAccount: annotations: {} diff --git a/deploy/helm/charts/platform/templates/validate-values.yaml b/deploy/helm/charts/platform/templates/validate-values.yaml index 63154dc1504e..fb530ecf85c7 100644 --- a/deploy/helm/charts/platform/templates/validate-values.yaml +++ b/deploy/helm/charts/platform/templates/validate-values.yaml @@ -146,6 +146,26 @@ kai-scheduler: {{- end }} {{- end }} +{{- if and + .Values.global.grove.install + (index .Values "dynamo-operator").lpx.enabled +}} + {{- fail ` +CONFIGURATION ERROR: The bundled Grove v0.1.0-alpha.13 does not support +the scheduler-scoped PodGangs required by LPX workloads. + +Install the scheduler-scoped Grove build separately, then configure: + + global: + grove: + install: false + enabled: true + dynamo-operator: + lpx: + enabled: true +` }} +{{- end }} + {{/* NOTE: dynamo-operator subchart values (e.g. dynamo-operator.nats.enabled, dynamo-operator.webhook.enabled) are validated in the operator subchart's diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml new file mode 100644 index 000000000000..1bbaf7af5d34 --- /dev/null +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: LPX operator configuration and RBAC +templates: + - charts/dynamo-operator/templates/operator-config.yaml + - charts/dynamo-operator/templates/manager-rbac.yaml + - charts/dynamo-operator/templates/validate-values.yaml +tests: + - it: keeps LPX disabled by default + template: charts/dynamo-operator/templates/operator-config.yaml + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpx:\n enabled: false$' + + - it: renders only Dynamo operator LPX configuration when enabled + template: charts/dynamo-operator/templates/operator-config.yaml + set: + dynamo-operator.lpx.enabled: true + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpx:\n enabled: true$' + + - it: renders canonical LPX model registry configuration + template: charts/dynamo-operator/templates/operator-config.yaml + set: + dynamo-operator.lpx.enabled: true + dynamo-operator.lpx.modelRegistryURL: gs://models/canonical + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpx:\n enabled: true\n modelRegistryURL: "gs://models/canonical"$' + - notMatchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpu:' + + - it: renders the exact inert LpuPipelineRequest manager grant + template: charts/dynamo-operator/templates/manager-rbac.yaml + asserts: + - contains: + path: rules + content: + apiGroups: + - scheduling.lpu.nvidia.com + resources: + - lpupipelinerequests + verbs: + - create + - delete + - get + - list + - update + - watch + documentIndex: 0 diff --git a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml index 51662fcc3895..b7ccc00058bc 100644 --- a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml +++ b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml @@ -97,6 +97,22 @@ tests: asserts: - notFailedTemplate: {} + - it: should reject bundled Grove with LPX + set: + global.grove.install: true + dynamo-operator.lpx.enabled: true + asserts: + - failedTemplate: + errorPattern: "bundled Grove v0.1.0-alpha.13 does not support" + + - it: should allow externally installed scheduler-scoped Grove with LPX + set: + global.grove.install: false + global.grove.enabled: true + dynamo-operator.lpx.enabled: true + asserts: + - notFailedTemplate: {} + # --- Default values should render cleanly --- - it: should render successfully with default values diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 32f072db7223..4fe26587fba2 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -49,10 +49,13 @@ global: grove: # -- Whether this chart should install the bundled Grove subchart. # When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. + # The bundled version cannot be used with LPX; LPX requires an externally installed + # scheduler-scoped Grove build that supports disjoint schedulers in one PodCliqueSet. # NOTE: For production environments, it is recommended to install Grove separately. install: false # -- Whether to enable Grove integration (multinode orchestration via PodCliqueSets). # Set to true when Grove is available in the cluster (installed externally). + # LPX users must provide the scheduler-scoped Grove build and enable its lpx-scheduler profile. # Automatically true when install=true. The operator uses this to decide whether to # create PodCliqueSets for multinode deployments. enabled: false @@ -87,6 +90,17 @@ dynamo-operator: # -- URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). modelExpressURL: "" + + # LPX is installed and operated separately. These values only opt the + # Dynamo operator into producing node-local scheduling requests. + lpx: + # -- Enable LPX integration. Startup verifies that the externally installed + # LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest + # resources and fails otherwise. + enabled: false + # -- Configure the LPU model registry used by the LPX integration. + modelRegistryURL: "" + # -- DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. namespaceRestriction: # -- DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 5383594d2a0f..ac8d8db5c160 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -4,10 +4,10 @@ IMG ?= controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 -ENVTEST_PACKAGES ?= ./internal/controller ./internal/webhook/validation +ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/validation ./cmd/crd-apply ENVTEST_ASSETS_CMD = $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path -CLUSTERTEST_PACKAGES ?= ./internal/testing/clusterenv ./internal/controller +CLUSTERTEST_PACKAGES ?= ./internal/testing/clusterenv ./internal/controller/... CLUSTERTEST_TIMEOUT ?= 30m CLUSTERTEST_KIND_NAME ?= dynamo-clustertest-v136 CLUSTERTEST_CONTEXT = kind-$(CLUSTERTEST_KIND_NAME) @@ -282,6 +282,11 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C $(YQ) eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \ fi; \ done + # Compact LPX schemas last: subsequent schema edits could change an aliased role. + for file in config/crd/bases/nvidia.com_dynamographdeployments.yaml config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml; do \ + $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 $$file || exit 1; \ + done + .PHONY: generate generate: controller-gen generate-pydantic ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt_" paths="./..." @@ -304,10 +309,10 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet setup-envtest ## Run tests. - KUBEBUILDER_ASSETS="$$($(ENVTEST_ASSETS_CMD))" go test $$(go list ./... | grep -vE '/cmd($$|/)') -coverprofile cover.out + KUBEBUILDER_ASSETS="$$($(ENVTEST_ASSETS_CMD))" go test $$(go list ./... | grep -vE '/cmd($$|/)') ./cmd/crd-apply -coverprofile cover.out .PHONY: envtest -envtest: setup-envtest ## Run envtest-backed operator tests. +envtest: setup-envtest yq ## Run envtest-backed operator tests. KUBEBUILDER_ASSETS="$$($(ENVTEST_ASSETS_CMD))" go test $(ENVTEST_PACKAGES) .PHONY: setup-clustertest diff --git a/deploy/operator/api/config/v1alpha1/lpx.go b/deploy/operator/api/config/v1alpha1/lpx.go new file mode 100644 index 000000000000..2a6ba00e94bf --- /dev/null +++ b/deploy/operator/api/config/v1alpha1/lpx.go @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +// LPXConfiguration holds LPX scheduler integration and model-registry settings. +type LPXConfiguration struct { + // Enabled opts the operator into LPX request production. When true, startup + // verifies that the separately installed LPX API serves + // scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails + // otherwise. + // +kubebuilder:default=false + Enabled bool `json:"enabled"` + + // ModelRegistryURL configures the location of the LPU model registry used by LPX. + // Supported values are an absolute local path (or file:// URL) and gs:// URLs. + ModelRegistryURL string `json:"modelRegistryURL"` +} diff --git a/deploy/operator/api/config/v1alpha1/types.go b/deploy/operator/api/config/v1alpha1/types.go index 8064a7b2eb64..5f295000edb9 100644 --- a/deploy/operator/api/config/v1alpha1/types.go +++ b/deploy/operator/api/config/v1alpha1/types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -61,6 +62,9 @@ type OperatorConfiguration struct { // Checkpoint/restore configuration Checkpoint CheckpointConfiguration `json:"checkpoint"` + // Pod generation defaults + PodGeneration PodGenerationConfiguration `json:"podGeneration"` + // Discovery backend configuration Discovery DiscoveryConfiguration `json:"discovery"` @@ -72,6 +76,9 @@ type OperatorConfiguration struct { // HTTP/2 and TLS settings Security SecurityConfiguration `json:"security"` + + // LPX scheduler integration and model-registry configuration. + LPX LPXConfiguration `json:"lpx"` } // ServerConfiguration holds server bind addresses and ports. @@ -389,6 +396,14 @@ func (c *CheckpointConfiguration) EffectiveSeccompProfile() string { return c.Seccomp.Profile } +// PodGenerationConfiguration holds pod generation defaults applied by the operator. +type PodGenerationConfiguration struct { + // DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a + // component leaves spec.extraPodSpecMergeStrategy unset. + // +kubebuilder:default="override" + DefaultExtraPodSpecMergeStrategy nvidiacomv1alpha1.ExtraPodSpecMergeStrategy `json:"defaultExtraPodSpecMergeStrategy,omitempty"` +} + // DiscoveryConfiguration holds discovery backend settings. type DiscoveryConfiguration struct { // Backend is the discovery backend: "kubernetes" or "etcd" diff --git a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go index 8cb75d886f7e..42a171870f91 100644 --- a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go @@ -222,6 +222,21 @@ func (in *KaiSchedulerConfiguration) DeepCopy() *KaiSchedulerConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXConfiguration) DeepCopyInto(out *LPXConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXConfiguration. +func (in *LPXConfiguration) DeepCopy() *LPXConfiguration { + if in == nil { + return nil + } + out := new(LPXConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LWSConfiguration) DeepCopyInto(out *LWSConfiguration) { *out = *in @@ -356,10 +371,12 @@ func (in *OperatorConfiguration) DeepCopyInto(out *OperatorConfiguration) { out.RBAC = in.RBAC out.MPI = in.MPI in.Checkpoint.DeepCopyInto(&out.Checkpoint) + out.PodGeneration = in.PodGeneration out.Discovery = in.Discovery in.GPU.DeepCopyInto(&out.GPU) out.Logging = in.Logging out.Security = in.Security + out.LPX = in.LPX } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfiguration. @@ -399,6 +416,21 @@ func (in *OrchestratorConfiguration) DeepCopy() *OrchestratorConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGenerationConfiguration) DeepCopyInto(out *PodGenerationConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGenerationConfiguration. +func (in *PodGenerationConfiguration) DeepCopy() *PodGenerationConfiguration { + if in == nil { + return nil + } + out := new(PodGenerationConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RBACConfiguration) DeepCopyInto(out *RBACConfiguration) { *out = *in diff --git a/deploy/operator/api/config/validation/validation.go b/deploy/operator/api/config/validation/validation.go index 67020970d175..02cd25dd8d47 100644 --- a/deploy/operator/api/config/validation/validation.go +++ b/deploy/operator/api/config/validation/validation.go @@ -40,6 +40,13 @@ func ValidateOperatorConfiguration(config *configv1alpha1.OperatorConfiguration) allErrs = append(allErrs, validateDiscovery(&config.Discovery, field.NewPath("discovery"))...) allErrs = append(allErrs, validateRBAC(config)...) allErrs = append(allErrs, validateOrchestrators(&config.Orchestrators, field.NewPath("orchestrators"))...) + if strategy := config.PodGeneration.DefaultExtraPodSpecMergeStrategy; strategy != "" && !strategy.IsValid() { + allErrs = append(allErrs, field.NotSupported( + field.NewPath("podGeneration").Child("defaultExtraPodSpecMergeStrategy"), + strategy, + []string{"override", "strategic"}, + )) + } allErrs = append(allErrs, validateIngress(&config.Ingress, field.NewPath("ingress"))...) allErrs = append(allErrs, validateServiceMesh(&config.ServiceMesh, field.NewPath("serviceMesh"))...) diff --git a/deploy/operator/api/config/validation/validation_test.go b/deploy/operator/api/config/validation/validation_test.go index e713c8a68dc3..947deb8a7edd 100644 --- a/deploy/operator/api/config/validation/validation_test.go +++ b/deploy/operator/api/config/validation/validation_test.go @@ -362,3 +362,38 @@ func TestValidateInfrastructure_TLS(t *testing.T) { } }) } + +func TestValidateOperatorConfiguration_InvalidDefaultExtraPodSpecMergeStrategy(t *testing.T) { + cfg := validConfig() + cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy = "invalid" + + errs := ValidateOperatorConfiguration(cfg) + if len(errs) != 1 { + t.Errorf("expected 1 error for invalid extraPodSpec merge strategy, got %d: %v", len(errs), errs) + } +} + +func TestValidateOperatorConfiguration_PodGenerationCamelCaseConfigKey(t *testing.T) { + cfg := validConfig() + rawConfig := []byte(`{ + "podGeneration": { + "defaultExtraPodSpecMergeStrategy": "strategic" + } + }`) + + if err := json.Unmarshal(rawConfig, cfg); err != nil { + t.Fatalf("failed to unmarshal podGeneration config: %v", err) + } + + if cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy != "strategic" { + t.Fatalf( + "expected strategic merge strategy from podGeneration key, got %q", + cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy, + ) + } + + errs := ValidateOperatorConfiguration(cfg) + if len(errs) != 0 { + t.Errorf("expected no errors for camelCase podGeneration config, got: %v", errs) + } +} diff --git a/deploy/operator/api/conversion_webhook_test.go b/deploy/operator/api/conversion_webhook_test.go index 9cb377001b64..660b4a279c1d 100644 --- a/deploy/operator/api/conversion_webhook_test.go +++ b/deploy/operator/api/conversion_webhook_test.go @@ -33,16 +33,20 @@ func TestConversionWebhook_PartialObjectsRetainMetadata(t *testing.T) { } handler := webhookconversion.NewWebhookHandler(scheme, webhookconversion.NewRegistry()) - for _, kind := range []string{ - "DynamoGraphDeployment", - "DynamoComponentDeployment", - "DynamoGraphDeploymentRequest", + alphaVersion, betaVersion := v1alpha1.GroupVersion.String(), v1beta1.GroupVersion.String() + for _, test := range []struct { + kind, sourceVersion, destinationVersion string + }{ + {"DynamoGraphDeployment", alphaVersion, betaVersion}, + {"DynamoComponentDeployment", alphaVersion, betaVersion}, + {"DynamoGraphDeploymentRequest", alphaVersion, betaVersion}, + {"DynamoGraphDeployment", betaVersion, alphaVersion}, } { - t.Run(kind, func(t *testing.T) { - t.Log("Build a partial v1alpha1 object with an empty metadata envelope") + t.Run(test.kind+"/"+test.destinationVersion, func(t *testing.T) { + t.Logf("Build a partial %s object with an empty metadata envelope", test.sourceVersion) source := map[string]any{ - "apiVersion": v1alpha1.GroupVersion.String(), - "kind": kind, + "apiVersion": test.sourceVersion, + "kind": test.kind, "metadata": map[string]any{}, } sourceRaw, err := json.Marshal(source) @@ -57,8 +61,8 @@ func TestConversionWebhook_PartialObjectsRetainMetadata(t *testing.T) { Kind: "ConversionReview", }, Request: &apiextensionsv1.ConversionRequest{ - UID: types.UID("partial-" + kind), - DesiredAPIVersion: v1beta1.GroupVersion.String(), + UID: types.UID("partial-" + test.kind), + DesiredAPIVersion: test.destinationVersion, Objects: []runtime.RawExtension{ {Raw: sourceRaw}, }, @@ -76,7 +80,7 @@ func TestConversionWebhook_PartialObjectsRetainMetadata(t *testing.T) { t.Fatalf("conversion webhook status = %d, body = %s", recorder.Code, recorder.Body) } - t.Log("Verify the converted v1beta1 object retains an empty metadata envelope") + t.Logf("Verify the converted %s object retains an empty metadata envelope", test.destinationVersion) var response apiextensionsv1.ConversionReview if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatalf("unmarshal conversion response: %v", err) @@ -95,6 +99,10 @@ func TestConversionWebhook_PartialObjectsRetainMetadata(t *testing.T) { if err := json.Unmarshal(response.Response.ConvertedObjects[0].Raw, &converted); err != nil { t.Fatalf("unmarshal converted object: %v", err) } + if converted["apiVersion"] != test.destinationVersion || converted["kind"] != test.kind { + t.Fatalf("converted type = %v %v, want %s %s", + converted["apiVersion"], converted["kind"], test.destinationVersion, test.kind) + } metadata, found := converted["metadata"] if !found { t.Fatalf("converted object is missing metadata: %s", response.Response.ConvertedObjects[0].Raw) diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index c096459fe8c5..fa1050ecab67 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -182,7 +182,7 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { fuzzBetaDGDRSpec, fuzzBetaDGDRStatus, fuzzBetaDGDROverrides, - // PlacementStatus (v1alpha1 + v1beta1): pick admissible values so the + // Shared PlacementStatus: pick admissible values so the // round-trip fuzzer exercises Placement without producing shapes the CRD // schema rejects. State draws from the enum; Score draws either nil or a // value inside the [0, 1] bounds enforced by the CRD. @@ -195,15 +195,6 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { ) p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) }, - func(p *v1alpha1.PlacementStatus, c randfill.Continue) { - p.State = oneOf(c, - v1alpha1.PlacementScoreStateReported, - v1alpha1.PlacementScoreStatePartial, - v1alpha1.PlacementScoreStateUnsupported, - v1alpha1.PlacementScoreStateUnknown, - ) - p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) - }, // v1beta1 Components: the listMapKey marker requires name // to be non-empty and unique; MaxItems caps the length at 25. // Enforce both so the input is admissible. diff --git a/deploy/operator/api/scripts/lpx-crd-schema.yq b/deploy/operator/api/scripts/lpx-crd-schema.yq new file mode 100644 index 000000000000..f57a06163060 --- /dev/null +++ b/deploy/operator/api/scripts/lpx-crd-schema.yq @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# LPX is DGD-only. Preserve ordinary DCD roles while removing LPX inputs and +# their dependent rules. Retain the rejection of standalone type: lpx. +with(select(.metadata.name == "dynamocomponentdeployments.nvidia.com"); + (.spec.versions[].schema.openAPIV3Schema.properties.spec) |= ( + del(.properties.lpx) | + (select(has("x-kubernetes-validations"))."x-kubernetes-validations") |= + map(select(.rule | test("\\b(self|oldSelf)\\.(lpx|roles)\\b") | not)) + ) +) | + +# Keep native validation and merge semantics; omit only LPX template descriptions. +# Ordinary PodTemplate and alpha extraPodSpec documentation remains unchanged. +del(.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | .. | select(has("description")) | .description) | + +# Compact only LPX templates, after every other schema transformation has finished. +# YAML requires the anchor before every alias: traversal order is not document order. +with(select([.. | select(has("lpx") and has("roles"))] | length > 0); + ([.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + {"path": (path | to_json), "line": line, "column": column}] | sort_by(.line, .column))[0].path as $first | + [.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) == $first) | to_json][0] as $schema | + (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) == $first)) anchor = "lpx-pod-template" | + + # Different role-specific constraints must never be replaced by an alias. + (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) != $first) | select((to_json) == $schema)) alias = "lpx-pod-template" +) diff --git a/deploy/operator/api/v1alpha1/common.go b/deploy/operator/api/v1alpha1/common.go index 8f263e706b95..0525fad35502 100644 --- a/deploy/operator/api/v1alpha1/common.go +++ b/deploy/operator/api/v1alpha1/common.go @@ -92,7 +92,7 @@ type ComponentRoleSpec struct { // PodTemplate defines the Pod configuration for this role. Admission permits // it only when the enclosing component type explicitly supports role-specific - // Pod templates. No component type supports it in this release. + // Pod templates. LPX components support role-specific Pod templates. // +optional PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } @@ -472,10 +472,10 @@ type DynamoCheckpointIdentity struct { // +kubebuilder:validation:Required Model string `json:"model"` - // BackendFramework is the runtime framework (vllm, sglang, trtllm). + // BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). // Deprecated: legacy identity only. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=vllm;sglang;trtllm + // +kubebuilder:validation:Enum=vllm;sglang;trtllm;lpu BackendFramework string `json:"backendFramework"` // DynamoVersion is the Dynamo platform version. diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index bf0df76aadb3..1c5faa52b50a 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -59,8 +59,11 @@ DynamoComponentDeploymentSpec.experimental.gpuMemoryService.extraClientPods.name DynamoComponentDeploymentSpec.experimental.gpuMemoryService.extraClientPods.podTemplate DynamoComponentDeploymentSpec.experimental.gpuMemoryService.mode DynamoComponentDeploymentSpec.experimental.grove.forceScalingGroup +DynamoComponentDeploymentSpec.extraPodSpecMergeStrategy DynamoComponentDeploymentSpec.frontendSidecar DynamoComponentDeploymentSpec.globalDynamoNamespace +DynamoComponentDeploymentSpec.lpx.buildId +DynamoComponentDeploymentSpec.lpx.settings DynamoComponentDeploymentSpec.minAvailable DynamoComponentDeploymentSpec.modelRef.name DynamoComponentDeploymentSpec.modelRef.revision @@ -87,6 +90,7 @@ DynamoComponentDeploymentStatus.component.componentKind DynamoComponentDeploymentStatus.component.componentNames DynamoComponentDeploymentStatus.component.gpusPerEngine DynamoComponentDeploymentStatus.component.gpusPerReplica +DynamoComponentDeploymentStatus.component.ready DynamoComponentDeploymentStatus.component.readyReplicas DynamoComponentDeploymentStatus.component.replicas DynamoComponentDeploymentStatus.component.runtimeNamespace @@ -130,8 +134,11 @@ DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.extraClientPo DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.extraClientPods.podTemplate DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.mode DynamoGraphDeploymentSpec.components.experimental.grove.forceScalingGroup +DynamoGraphDeploymentSpec.components.extraPodSpecMergeStrategy DynamoGraphDeploymentSpec.components.frontendSidecar DynamoGraphDeploymentSpec.components.globalDynamoNamespace +DynamoGraphDeploymentSpec.components.lpx.buildId +DynamoGraphDeploymentSpec.components.lpx.settings DynamoGraphDeploymentSpec.components.minAvailable DynamoGraphDeploymentSpec.components.modelRef.name DynamoGraphDeploymentSpec.components.modelRef.revision @@ -167,6 +174,7 @@ DynamoGraphDeploymentSpec.providerOverride.value DynamoGraphDeploymentSpec.restart.id DynamoGraphDeploymentSpec.restart.strategy.order DynamoGraphDeploymentSpec.restart.strategy.type +DynamoGraphDeploymentSpec.scheduling.attemptDeadlineSeconds DynamoGraphDeploymentSpec.topologyConstraint.clusterTopologyName DynamoGraphDeploymentSpec.topologyConstraint.packDomain DynamoGraphDeploymentStatus.checkpoints.checkpointID @@ -178,15 +186,26 @@ DynamoGraphDeploymentStatus.components.componentKind DynamoGraphDeploymentStatus.components.componentNames DynamoGraphDeploymentStatus.components.gpusPerEngine DynamoGraphDeploymentStatus.components.gpusPerReplica +DynamoGraphDeploymentStatus.components.ready DynamoGraphDeploymentStatus.components.readyReplicas DynamoGraphDeploymentStatus.components.replicas DynamoGraphDeploymentStatus.components.runtimeNamespace DynamoGraphDeploymentStatus.components.scheduledReplicas DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions +DynamoGraphDeploymentStatus.lpx.modelDownload.builds +DynamoGraphDeploymentStatus.lpx.modelDownload.lastCheckedAt +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.deadlineAt +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.disarmedAt +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.exceededAt +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.observedGeneration +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.podCliqueSetUID +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.attemptDigest +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.name +DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.uid +DynamoGraphDeploymentStatus.lpx.placement.score +DynamoGraphDeploymentStatus.lpx.placement.state DynamoGraphDeploymentStatus.observedGeneration -DynamoGraphDeploymentStatus.placement.score -DynamoGraphDeploymentStatus.placement.state DynamoGraphDeploymentStatus.restart.inProgress DynamoGraphDeploymentStatus.restart.observedID DynamoGraphDeploymentStatus.restart.phase diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go index 7c43dc144d1e..ef7cb855e980 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go @@ -804,6 +804,7 @@ func TestDCD_RoundTrip_Status(t *testing.T) { ComponentNames: []string{"dcd-0"}, GPUsPerEngine: ptr.To(int64(2)), GPUsPerReplica: ptr.To(int64(3)), + Ready: true, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index f8832fa5c324..ea8d2d33c319 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -23,6 +23,7 @@ import ( "fmt" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,12 +35,44 @@ const ( DynamoGraphDeploymentConditionTypeDynamoComponentReady = "DynamoComponentReady" ) +// +kubebuilder:validation:Enum=override;strategic +type ExtraPodSpecMergeStrategy string + +const ( + ExtraPodSpecMergeStrategyOverride ExtraPodSpecMergeStrategy = "override" + ExtraPodSpecMergeStrategyStrategic ExtraPodSpecMergeStrategy = "strategic" + DefaultExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy = ExtraPodSpecMergeStrategyOverride +) + +func (s ExtraPodSpecMergeStrategy) IsValid() bool { + return s == ExtraPodSpecMergeStrategyOverride || s == ExtraPodSpecMergeStrategyStrategic +} + +func ResolveExtraPodSpecMergeStrategy( + explicitStrategy ExtraPodSpecMergeStrategy, + defaultStrategy ExtraPodSpecMergeStrategy, +) (ExtraPodSpecMergeStrategy, error) { + resolved := explicitStrategy + if resolved == "" { + resolved = defaultStrategy + } + if resolved == "" { + resolved = DefaultExtraPodSpecMergeStrategy + } + if !resolved.IsValid() { + return "", fmt.Errorf("invalid extraPodSpec merge strategy %q", resolved) + } + return resolved, nil +} + // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. // DynamoComponentDeploymentSpec defines the desired state of DynamoComponentDeployment +// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx')",message="standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment" type DynamoComponentDeploymentSpec struct { - // BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") + // BackendFramework specifies the GPU backend framework (for example, + // "sglang", "vllm", or "trtllm"). // +kubebuilder:validation:Enum=sglang;vllm;trtllm BackendFramework string `json:"backendFramework,omitempty"` @@ -51,6 +84,11 @@ type DynamoComponentDeploymentSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)",message="componentType is immutable after it is set" +// +kubebuilder:validation:XValidation:rule="!has(self.componentType) || self.componentType != 'lpu'",message="componentType lpu is not supported; use lpx" +// +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.componentType) && self.componentType == 'lpx')",message="lpx may only be set when componentType is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx') || has(self.lpx)",message="lpx is required when componentType is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))",message="replicas must be between 1 and 9 when componentType is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)",message="scalingAdapter is not supported when componentType is lpx" type DynamoComponentDeploymentSharedSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file @@ -132,6 +170,10 @@ type DynamoComponentDeploymentSharedSpec struct { // extraPodSpec and provide a non-empty mainContainer image. Existing components created // without extraPodSpec may remain unchanged. ExtraPodSpec *ExtraPodSpec `json:"extraPodSpec,omitempty"` + // ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the + // operator-generated pod and container defaults. + // +optional + ExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy `json:"extraPodSpecMergeStrategy,omitempty"` // LivenessProbe to detect and restart unhealthy containers. LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` @@ -168,6 +210,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. + // LPX components require a worker role and may declare a leader role. // +optional // +listType=map // +listMapKey=name @@ -195,6 +238,11 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional FrontendSidecar *FrontendSidecarSpec `json:"frontendSidecar,omitempty"` + // LPX holds LPX integration configuration. Only meaningful when + // ComponentType is "lpx". + // +optional + LPX *v1beta1.LPXConfig `json:"lpx,omitempty"` + // Checkpoint configures container checkpointing for this service. // When enabled, pods can be restored from a checkpoint files for faster cold start. // +optional @@ -381,7 +429,7 @@ func (s *DynamoComponentDeployment) SetDynamoDeploymentConfig(config []byte) { } func (s *DynamoComponentDeployment) IsMultinode() bool { - return s.GetNumberOfNodes() > 1 + return s.Spec.IsMultinode() } func (s *DynamoComponentDeployment) GetNumberOfNodes() int32 { diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 4a46a4b110c2..6f533cb244f1 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -107,6 +107,7 @@ func (src *DynamoGraphDeployment) ConvertTo(dstRaw conversion.Hub) error { // v1beta1. func ConvertFromDynamoGraphDeploymentSpec(src *DynamoGraphDeploymentSpec, dst *v1beta1.DynamoGraphDeploymentSpec, restored *v1beta1.DynamoGraphDeploymentSpec, save *DynamoGraphDeploymentSpec, ctx DynamoGraphDeploymentConversionContext) error { // Convert fields represented by both versions from the live source. + dst.Scheduling = src.Scheduling dst.Annotations = src.Annotations dst.Labels = src.Labels dst.PriorityClassName = src.PriorityClassName @@ -420,6 +421,7 @@ func (dst *DynamoGraphDeployment) ConvertFrom(srcRaw conversion.Hub) error { // v1alpha1. func ConvertToDynamoGraphDeploymentSpec(src *v1beta1.DynamoGraphDeploymentSpec, dst *DynamoGraphDeploymentSpec, restored *DynamoGraphDeploymentSpec, save *v1beta1.DynamoGraphDeploymentSpec, ctx DynamoGraphDeploymentConversionContext) error { // Convert fields represented by both versions from the live source. + dst.Scheduling = src.Scheduling dst.Annotations = src.Annotations dst.Labels = src.Labels dst.PriorityClassName = src.PriorityClassName @@ -652,16 +654,7 @@ func ConvertToSpecTopologyConstraint(src *v1beta1.SpecTopologyConstraint, dst *S func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, dst *v1beta1.DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = v1beta1.DGDState(src.State) - if src.Placement != nil { - dst.Placement = &v1beta1.PlacementStatus{ - State: v1beta1.PlacementScoreState(src.Placement.State), - } - if src.Placement.Score != nil { - dst.Placement.Score = ptr.To(*src.Placement.Score) - } - } else { - dst.Placement = nil - } + dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { @@ -699,16 +692,7 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStatus, dst *DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = DGDState(src.State) - if src.Placement != nil { - dst.Placement = &PlacementStatus{ - State: PlacementScoreState(src.Placement.State), - } - if src.Placement.Score != nil { - dst.Placement.Score = ptr.To(*src.Placement.Score) - } - } else { - dst.Placement = nil - } + dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { @@ -810,6 +794,7 @@ func ConvertFromServiceReplicaStatus(src *ServiceReplicaStatus, dst *v1beta1.Com ComponentKind: v1beta1.ComponentKind(src.ComponentKind), ComponentNames: componentNamesToHub(src), RuntimeNamespace: src.RuntimeNamespace, + Ready: src.Ready, Replicas: src.Replicas, UpdatedReplicas: src.UpdatedReplicas, } @@ -839,6 +824,7 @@ func ConvertToServiceReplicaStatus(src *v1beta1.ComponentReplicaStatus, dst *Ser ComponentKind: ComponentKind(src.ComponentKind), ComponentNames: componentNames, RuntimeNamespace: src.RuntimeNamespace, + Ready: src.Ready, Replicas: src.Replicas, UpdatedReplicas: src.UpdatedReplicas, } diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index 4fab6a9b827d..2a5a8c9a1e6b 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -127,6 +127,15 @@ func TestDGD_RoundTrip_Empty(t *testing.T) { if diff := cmp.Diff(src, got); diff != "" { t.Errorf("round-trip mismatch (-want +got):\n%s", diff) } + + t.Log("Omit LPX status when the deployment has no LPX payload") + raw, err := json.Marshal(got.Status) + if err != nil { + t.Fatal(err) + } + if string(raw) != `{"state":""}` { + t.Fatalf("empty status JSON = %s", raw) + } } func TestDGD_RoundTrip_Minimal(t *testing.T) { @@ -724,6 +733,7 @@ func TestDGD_FromV1alpha1_GMSExtraClientsRoundTripsThroughHub(t *testing.T) { } func TestDGD_RoundTrip_PodTemplate(t *testing.T) { + t.Log("Define native Pod metadata, scheduling fields and main-container settings") shm := resource.MustParse("4Gi") src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "pt", Namespace: "ns"}, @@ -739,6 +749,20 @@ func TestDGD_RoundTrip_PodTemplate(t *testing.T) { Labels: map[string]string{"tier": "gpu"}, }, Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"node-pool": "gpu"}, + ServiceAccountName: "dynamo-sa", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "ghcr-creds"}}, + Tolerations: []corev1.Toleration{ + {Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + }, + Volumes: []corev1.Volume{ + { + Name: "cache", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, Containers: []corev1.Container{ { Name: "main", @@ -746,6 +770,30 @@ func TestDGD_RoundTrip_PodTemplate(t *testing.T) { Env: []corev1.EnvVar{ {Name: "DYN_COMPONENT", Value: "worker"}, }, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-secret"}, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstrFromInt32(8080)}, + }, + InitialDelaySeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/ready", Port: intstrFromInt32(8080)}, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/startup", Port: intstrFromInt32(8080)}, + }, + FailureThreshold: 30, + }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), @@ -760,13 +808,10 @@ func TestDGD_RoundTrip_PodTemplate(t *testing.T) { }, }, } + + t.Log("Round-trip through alpha and compare the complete native template and shared memory") got := roundTripFromV1beta1(t, src) - // corev1.ResourceList equality can be quantity-representation-sensitive; - // use cmpopts to compare canonical forms. - opts := cmp.Options{ - cmpopts.EquateEmpty(), - } - if diff := cmp.Diff(src, got, opts); diff != "" { + if diff := cmp.Diff(src, got, cmpopts.EquateEmpty()); diff != "" { t.Errorf("round-trip mismatch (-want +got):\n%s", diff) } } @@ -904,7 +949,7 @@ func TestDGD_FromV1alpha1_SubComponentType(t *testing.T) { // TestDGD_RoundTrip_Status exercises every populated Status sub-struct so that // the ConvertTo / ConvertFrom status paths are covered (conditions, services -// map, restart, checkpoints, rollingUpdate). +// map, restart, checkpoints, LPX placement, rollingUpdate). func TestDGD_RoundTrip_Status(t *testing.T) { now := metav1.NewTime(metav1.Now().Rfc3339Copy().Time) later := metav1.NewTime(now.Time.Add(60 * time.Second)) @@ -913,9 +958,11 @@ func TestDGD_RoundTrip_Status(t *testing.T) { Status: v1beta1.DynamoGraphDeploymentStatus{ ObservedGeneration: 7, State: v1beta1.DGDStateSuccessful, - Placement: &v1beta1.PlacementStatus{ - Score: ptr.To(0.87), - State: v1beta1.PlacementScoreStateReported, + LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ + Placement: &v1beta1.PlacementStatus{ + Score: ptr.To(0.87), + State: v1beta1.PlacementScoreStateReported, + }, }, Conditions: []metav1.Condition{ { @@ -933,6 +980,7 @@ func TestDGD_RoundTrip_Status(t *testing.T) { RuntimeNamespace: "ns-status-worker-abc123", GPUsPerEngine: ptr.To(int64(2)), GPUsPerReplica: ptr.To(int64(3)), + Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), @@ -1026,101 +1074,6 @@ func TestDGD_RoundTrip_FullSharedSpec(t *testing.T) { } } -// TestDGD_RoundTrip_PodTemplateProbesAndEnvFrom covers the main-container -// fields that decomposePodTemplate preserves through ExtraPodSpec.MainContainer: -// EnvFrom, LivenessProbe, ReadinessProbe, StartupProbe. -func TestDGD_RoundTrip_PodTemplateProbesAndEnvFrom(t *testing.T) { - src := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: "probes", Namespace: "ns"}, - Spec: v1beta1.DynamoGraphDeploymentSpec{ - Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - { - ComponentName: "worker", - ComponentType: v1beta1.ComponentTypeWorker, - PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main", - Image: "dynamo:latest", - EnvFrom: []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "aws-secret"}, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstrFromInt32(8080)}, - }, - InitialDelaySeconds: 5, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{Path: "/ready", Port: intstrFromInt32(8080)}, - }, - }, - StartupProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{Path: "/startup", Port: intstrFromInt32(8080)}, - }, - FailureThreshold: 30, - }, - }, - }, - }, - }}, - }, - }, - } - got := roundTripFromV1beta1(t, src) - if diff := cmp.Diff(src, got, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("round-trip mismatch (-want +got):\n%s", diff) - } -} - -// TestDGD_RoundTrip_PodSpecExtras covers the non-main-container PodSpec fields -// that flow through ExtraPodSpec.PodSpec: NodeSelector, Tolerations, -// ServiceAccountName, ImagePullSecrets, Volumes. -func TestDGD_RoundTrip_PodSpecExtras(t *testing.T) { - src := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: "extras", Namespace: "ns"}, - Spec: v1beta1.DynamoGraphDeploymentSpec{ - Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - { - ComponentName: "worker", - ComponentType: v1beta1.ComponentTypeWorker, - PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - NodeSelector: map[string]string{"node-pool": "gpu"}, - ServiceAccountName: "dynamo-sa", - ImagePullSecrets: []corev1.LocalObjectReference{{Name: "ghcr-creds"}}, - Tolerations: []corev1.Toleration{ - {Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, - }, - Volumes: []corev1.Volume{ - { - Name: "cache", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - Containers: []corev1.Container{ - {Name: "main", Image: "dynamo:latest"}, - }, - }, - }}, - }, - }, - } - got := roundTripFromV1beta1(t, src) - if diff := cmp.Diff(src, got, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("round-trip mismatch (-want +got):\n%s", diff) - } -} - // TestDGD_RoundTrip_FrontendSidecar starts from v1beta1 (hub) with the // FrontendSidecar string naming a sidecar container in podTemplate.containers. func TestDGD_RoundTrip_FrontendSidecar(t *testing.T) { diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 162cff121dfc..c49bcd08410b 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) @@ -54,34 +55,6 @@ const ( DGDStateFailed DGDState = "failed" ) -// PlacementScoreState describes whether placement score is available and how -// complete the reported score is for a graph deployment. See the v1beta1 -// PlacementScoreState for the authoritative semantics of each value. -// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;Unknown -type PlacementScoreState string - -const ( - PlacementScoreStateReported PlacementScoreState = "Reported" - PlacementScoreStatePartial PlacementScoreState = "Partial" - PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" - PlacementScoreStateUnknown PlacementScoreState = "Unknown" -) - -// PlacementStatus groups DGD-level scheduler placement fields under a single -// status object so future placement signals can be added without a schema -// break. See the v1beta1 PlacementStatus for the authoritative field docs. -type PlacementStatus struct { - // Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] - // where higher is better and 1.0 is the best possible placement. - // +optional - // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=1 - Score *float64 `json:"score,omitempty"` - // State indicates placement score reporting state. - // +optional - State PlacementScoreState `json:"state,omitempty"` -} - // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. // +kubebuilder:validation:XValidation:rule="oldSelf.hasValue() || !has(self.restart)",message="spec.restart must be unset on create; set spec.restart.id after creation to request a restart",optionalOldSelf=true type DynamoGraphDeploymentSpec struct { @@ -92,6 +65,10 @@ type DynamoGraphDeploymentSpec struct { // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` + // Scheduling configures attempts for LPX components. Omission means no deadline. + // +optional + Scheduling *v1beta1.SchedulingSpec `json:"scheduling,omitempty"` + // Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). // Service-level annotations take precedence over these values. // +optional @@ -200,14 +177,13 @@ type DynamoGraphDeploymentStatus struct { // The map key is the service name from spec.services. // +optional Checkpoints map[string]ServiceCheckpointStatus `json:"checkpoints,omitempty"` + // LPX contains the status of the graph's LPX workload, when present. + // +optional + LPX *v1beta1.DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` // RollingUpdate tracks the progress of operator manged rolling updates. // Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` - // Placement groups DGD-level scheduler placement signals (score, reporting - // state, and any future placement fields). - // +optional - Placement *PlacementStatus `json:"placement,omitempty"` } // ServiceCheckpointStatus contains checkpoint information for a single service. @@ -324,6 +300,11 @@ type ServiceReplicaStatus struct { // +kubebuilder:validation:Minimum=0 GPUsPerReplica *int64 `json:"gpusPerReplica,omitempty"` + // Ready is the binary determination of whether the correct number of replicas + // are scheduled and available. + // +optional + Ready bool `json:"ready"` + // Replicas is the total number of non-terminated replicas. // Required for all component kinds. // +kubebuilder:validation:Minimum=0 diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go new file mode 100644 index 000000000000..a207164175ea --- /dev/null +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package v1alpha1 + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtraPodSpecMergeStrategy_RoundTripAndValidation(t *testing.T) { + dgd := DynamoGraphDeployment{ + Spec: DynamoGraphDeploymentSpec{ + Services: map[string]*DynamoComponentDeploymentSharedSpec{ + "frontend": { + ComponentType: "frontend", + ExtraPodSpecMergeStrategy: ExtraPodSpecMergeStrategyStrategic, + }, + }, + }, + } + + data, err := json.Marshal(dgd) + require.NoError(t, err) + + var restored DynamoGraphDeployment + require.NoError(t, json.Unmarshal(data, &restored)) + + assert.Equal( + t, + ExtraPodSpecMergeStrategyStrategic, + restored.Spec.Services["frontend"].ExtraPodSpecMergeStrategy, + ) + assert.True(t, ExtraPodSpecMergeStrategyOverride.IsValid()) + assert.True(t, ExtraPodSpecMergeStrategyStrategic.IsValid()) + assert.False(t, ExtraPodSpecMergeStrategy("invalid").IsValid()) +} + +func TestResolveExtraPodSpecMergeStrategy_DefaultsToOverride(t *testing.T) { + got, err := ResolveExtraPodSpecMergeStrategy("", "") + require.NoError(t, err) + assert.Equal(t, ExtraPodSpecMergeStrategyOverride, got) +} diff --git a/deploy/operator/api/v1alpha1/groupversion_info.go b/deploy/operator/api/v1alpha1/groupversion_info.go index 681adaaa7a69..dbaffb991bc0 100644 --- a/deploy/operator/api/v1alpha1/groupversion_info.go +++ b/deploy/operator/api/v1alpha1/groupversion_info.go @@ -38,6 +38,9 @@ var ( // DynamoGraphDeploymentGVK is the v1alpha1 DynamoGraphDeployment kind. DynamoGraphDeploymentGVK = GroupVersion.WithKind("DynamoGraphDeployment") + // LPXGraphDeploymentGVK identifies the operator-generated LPX child. + LPXGraphDeploymentGVK = GroupVersion.WithKind("LPXGraphDeployment") + // DynamoGraphDeploymentRequestGVK is the v1alpha1 DynamoGraphDeploymentRequest kind. DynamoGraphDeploymentRequestGVK = GroupVersion.WithKind("DynamoGraphDeploymentRequest") @@ -54,6 +57,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &DynamoComponentDeploymentList{}, &DynamoGraphDeployment{}, &DynamoGraphDeploymentList{}, + &LPXGraphDeployment{}, + &LPXGraphDeploymentList{}, &DynamoGraphDeploymentRequest{}, &DynamoGraphDeploymentRequestList{}, &DynamoGraphDeploymentScalingAdapter{}, diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go new file mode 100644 index 000000000000..e7db570b6040 --- /dev/null +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -0,0 +1,232 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package v1alpha1 + +import ( + "encoding/json" + "testing" + "time" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const editedThroughAlphaBuildID = "model/edited-through-alpha" + +func testCanonicalLPXConfig() *v1beta1.LPXConfig { + return &v1beta1.LPXConfig{ + BuildID: "model/build", + Settings: &apiextensionsv1.JSON{Raw: []byte(`{"scheduler":{"num_chips":4}}`)}, + } +} + +func testLPXRoleTemplate(image string) corev1.PodTemplateSpec { + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"role": image}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: image}}}, + } +} + +func TestDynamoGraphDeploymentRoleShapesRoundTrip(t *testing.T) { + t.Log("Cover absent, empty, and explicit roles without defaulting optional fields") + tests := []struct { + name string + alpha []ComponentRoleSpec + beta []v1beta1.ComponentRoleSpec + }{ + {name: "absent"}, + {name: "empty", alpha: []ComponentRoleSpec{}, beta: []v1beta1.ComponentRoleSpec{}}, + { + name: "native fields", + alpha: []ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleLeader}, + { + Name: v1beta1.ComponentRoleWorker, + Replicas: ptr.To[int32](3), + PodTemplate: ptr.To(testLPXRoleTemplate("agent")), + }, + }, + beta: []v1beta1.ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleLeader}, + { + Name: v1beta1.ComponentRoleWorker, + Replicas: ptr.To[int32](3), + PodTemplate: ptr.To(testLPXRoleTemplate("agent")), + }, + }, + }, + } + + // Conversion preserves the live representation independently of admission rules. + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Convert authored alpha roles to beta without changing their shape") + alpha := &DynamoGraphDeployment{ + Spec: DynamoGraphDeploymentSpec{Services: map[string]*DynamoComponentDeploymentSharedSpec{ + "lpx": {ComponentType: string(v1beta1.ComponentTypeLPX), Roles: tt.alpha}, + }}, + } + beta := &v1beta1.DynamoGraphDeployment{} + if err := alpha.ConvertTo(beta); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.beta, beta.Spec.Components[0].Roles); diff != "" { + t.Fatalf("role conversion mismatch (-want +got):\n%s", diff) + } + + t.Log("Convert beta roles back to alpha with the same order and optional values") + roundTrip := &DynamoGraphDeployment{} + if err := roundTrip.ConvertFrom(beta); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.alpha, roundTrip.Spec.Services["lpx"].Roles); diff != "" { + t.Fatalf("role round-trip mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { + checkedAt := metav1.NewTime(time.Unix(1_800_000_000, 0)) + deadlineAt := metav1.NewTime(time.Unix(1_800_000_900, 0)) + src := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "lpx", Namespace: "ns"}, + Spec: v1beta1.DynamoGraphDeploymentSpec{ + Scheduling: &v1beta1.SchedulingSpec{ + AttemptDeadlineSeconds: ptr.To[int64](900), + }, + Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "lpx", + ComponentType: v1beta1.ComponentTypeLPX, + LPX: testCanonicalLPXConfig(), + Roles: []v1beta1.ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleLeader, Replicas: ptr.To[int32](2)}, + {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + }, + }}, + }, + Status: v1beta1.DynamoGraphDeploymentStatus{ + LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: &v1beta1.ModelDownloadStatus{ + Builds: []string{"gs://models/draft", "gs://models/target"}, + LastCheckedAt: &checkedAt, + }, + Placement: &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ + ObservedGeneration: 2, + PodCliqueSetUID: "pcs-uid", + DeadlineAt: &deadlineAt, + Requests: []v1beta1.LPXAttemptRequestStatus{{ + Name: "request", AttemptDigest: "sha256:attempt", UID: "request-uid", + }}, + }}, + }, + }, + } + + t.Log("Convert the complete LPX configuration, scheduling, and status to alpha") + original := src.DeepCopy() + alpha := &DynamoGraphDeployment{} + if err := alpha.ConvertFrom(src); err != nil { + t.Fatalf("ConvertFrom: %v", err) + } + service := alpha.Spec.Services["lpx"] + if service == nil || service.ComponentType != string(v1beta1.ComponentTypeLPX) { + t.Fatalf("canonical LPX component did not convert to alpha: %#v", service) + } + if diff := cmp.Diff(testCanonicalLPXConfig(), service.LPX, cmpopts.EquateEmpty()); diff != "" { + t.Fatalf("LPX config conversion mismatch (-want +got):\n%s", diff) + } + wantAlphaRoles := []ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleLeader, Replicas: ptr.To[int32](2)}, + {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + } + if diff := cmp.Diff(wantAlphaRoles, service.Roles); diff != "" { + t.Fatalf("LPX role order or optional fields changed in alpha (-want +got):\n%s", diff) + } + if diff := cmp.Diff(src.Spec.Scheduling, alpha.Spec.Scheduling); diff != "" { + t.Fatalf("LPX scheduling conversion mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(src.Status.LPX, alpha.Status.LPX); diff != "" { + t.Fatalf("LPX status conversion mismatch (-want +got):\n%s", diff) + } + + t.Log("Serialize the native LPX payload under one status group in both versions") + payload, err := json.Marshal(src.Status.LPX) + if err != nil { + t.Fatal(err) + } + for _, status := range []any{src.Status, alpha.Status} { + raw, err := json.Marshal(status) + if err != nil { + t.Fatal(err) + } + if want := `{"state":"","lpx":` + string(payload) + `}`; string(raw) != want { + t.Fatalf("%T JSON = %s, want %s", status, raw, want) + } + } + + t.Log("Edit the alpha configuration and round-trip the complete LPX payload") + service.LPX = service.LPX.DeepCopy() + service.LPX.BuildID = editedThroughAlphaBuildID + service.Roles = []ComponentRoleSpec{ + *service.Roles[1].DeepCopy(), + {Name: v1beta1.ComponentRoleLeader, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, + } + + got := &v1beta1.DynamoGraphDeployment{} + if err := alpha.ConvertTo(got); err != nil { + t.Fatalf("ConvertTo: %v", err) + } + component := got.GetComponentByName("lpx") + if component == nil || component.LPX == nil || component.LPX.BuildID != editedThroughAlphaBuildID { + t.Fatalf("alpha LPX edit was lost: %#v", component) + } + wantBetaRoles := []v1beta1.ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + {Name: v1beta1.ComponentRoleLeader, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, + } + if diff := cmp.Diff(wantBetaRoles, component.Roles); diff != "" { + t.Fatalf("authored alpha role edits did not survive conversion (-want +got):\n%s", diff) + } + if diff := cmp.Diff(original, src); diff != "" { + t.Fatalf("alpha edit mutated the source LPX deployment (-want +got):\n%s", diff) + } + if diff := cmp.Diff(src.Spec.Scheduling, got.Spec.Scheduling); diff != "" { + t.Fatalf("LPX scheduling round-trip mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(src.Status, got.Status); diff != "" { + t.Fatalf("LPX status round-trip mismatch (-want +got):\n%s", diff) + } + if _, exists := got.Annotations[annDGDSpec]; exists { + t.Fatalf("canonical LPX spec must not use sparse conversion annotations: %v", got.Annotations) + } + if _, exists := got.Annotations[annDGDStatus]; exists { + t.Fatalf("canonical LPX status must not use sparse conversion annotations: %v", got.Annotations) + } + + t.Log("Preserve an explicitly empty LPX group through native conversion") + alpha.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{} + if err := alpha.ConvertTo(got); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(alpha.Status.LPX, got.Status.LPX); diff != "" { + t.Fatalf("empty LPX group changed (-want +got):\n%s", diff) + } + + t.Log("Clearing live alpha LPX status removes the previously populated group") + alpha.Status.LPX = nil + if err := alpha.ConvertTo(got); err != nil { + t.Fatal(err) + } + if got.Status.LPX != nil { + t.Fatalf("cleared LPX status was restored: %#v", got.Status.LPX) + } +} diff --git a/deploy/operator/api/v1alpha1/lpx_types.go b/deploy/operator/api/v1alpha1/lpx_types.go new file mode 100644 index 000000000000..a174bab26f1a --- /dev/null +++ b/deploy/operator/api/v1alpha1/lpx_types.go @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + +// IsLPX reports whether this shared spec uses the LPX integration. +func (s *DynamoComponentDeploymentSharedSpec) IsLPX() bool { + return s.ComponentType == commonconsts.ComponentTypeLPX +} diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go new file mode 100644 index 000000000000..04c6ee255294 --- /dev/null +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LPXGraphDeploymentSpec is the operator's handoff for all LPX components in a DGD. +// The owning DGD remains the only authored workload spec. The child reads that +// DGD only after verifying the controller owner and effective LPX input revision. +type LPXGraphDeploymentSpec struct { + // inputRevision hashes only effective LPX inputs, including its selected + // restart token. Unrelated component edits do not create an LPX revision. + // +kubebuilder:validation:Pattern="^sha256:[a-f0-9]{64}$" + InputRevision string `json:"inputRevision"` +} + +// LPXGraphDeploymentStatus is the LPX controller's durable lifecycle state. +// Dynamo consumes readiness only for the current generation and input revision. +type LPXGraphDeploymentStatus struct { + // observedGeneration is the child generation processed successfully. + // Every spec.inputRevision change advances the child generation. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // conditions report compilation, placement, and runtime readiness. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` + // components reports logical replicas by authored component name, never Agent Pods. + // +optional + Components map[string]v1beta1.ComponentReplicaStatus `json:"components,omitempty"` + // modelDownload retains the existing remote-build download progress. + // +optional + ModelDownload *v1beta1.ModelDownloadStatus `json:"modelDownload,omitempty"` + // placement retains the scheduling attempt fence, progress, and deadline. + // Its observed generation belongs to this child, not the source DGD. + // +optional + Placement *v1beta1.PlacementStatus `json:"placement,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:shortName=lpxgd +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// LPXGraphDeployment is generated and owned by the Dynamo operator, not authored +// by users. It owns one LPX PCS and its build/placement resources. +// It serves only v1alpha1 and has no conversion contract. +type LPXGraphDeployment struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec LPXGraphDeploymentSpec `json:"spec"` + // +optional + Status LPXGraphDeploymentStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LPXGraphDeploymentList contains operator-generated LPX deployments. +type LPXGraphDeploymentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LPXGraphDeployment `json:"items"` +} diff --git a/deploy/operator/api/v1alpha1/shared_spec_conversion.go b/deploy/operator/api/v1alpha1/shared_spec_conversion.go index cb429265280b..8208eb2c408d 100644 --- a/deploy/operator/api/v1alpha1/shared_spec_conversion.go +++ b/deploy/operator/api/v1alpha1/shared_spec_conversion.go @@ -160,6 +160,7 @@ func ConvertFromDynamoComponentDeploymentSharedSpec(src *DynamoComponentDeployme dst.GlobalDynamoNamespace = src.GlobalDynamoNamespace dst.Replicas = src.Replicas dst.MinAvailable = src.MinAvailable + dst.ExtraPodSpecMergeStrategy = v1beta1.ExtraPodSpecMergeStrategy(src.ExtraPodSpecMergeStrategy) if src.Multinode != nil { dst.Multinode = &v1beta1.MultinodeSpec{} @@ -186,6 +187,7 @@ func ConvertFromDynamoComponentDeploymentSharedSpec(src *DynamoComponentDeployme dst.EPPConfig = &v1beta1.EPPConfig{} ConvertFromEPPConfig(src.EPPConfig, dst.EPPConfig) } + dst.LPX = src.LPX // sharedMemory <-> sharedMemorySize (lossy struct flatten). if src.SharedMemory != nil && (src.SharedMemory.Disabled || !src.SharedMemory.Size.IsZero()) { @@ -248,6 +250,9 @@ func restoreSharedAlphaOnlySimpleFields(dst *DynamoComponentDeploymentSharedSpec cp := *preserved.Autoscaling dst.Autoscaling = &cp } + if dst.ExtraPodSpecMergeStrategy == "" && preserved.ExtraPodSpecMergeStrategy != "" { + dst.ExtraPodSpecMergeStrategy = preserved.ExtraPodSpecMergeStrategy + } if dst.Ingress == nil && preserved.Ingress != nil { cp := *preserved.Ingress dst.Ingress = &cp @@ -565,9 +570,10 @@ func ConvertToDynamoComponentDeploymentSharedSpec(src *v1beta1.DynamoComponentDe dst.EPPConfig = &EPPConfig{} ConvertToEPPConfig(src.EPPConfig, dst.EPPConfig) } - + dst.LPX = src.LPX dst.RuntimeVersionOverride = src.RuntimeVersionOverride dst.ServiceName = src.ComponentName + dst.ExtraPodSpecMergeStrategy = ExtraPodSpecMergeStrategy(src.ExtraPodSpecMergeStrategy) // sharedMemorySize -> SharedMemorySpec. if src.SharedMemorySize != nil { @@ -589,9 +595,7 @@ func ConvertToDynamoComponentDeploymentSharedSpec(src *v1beta1.DynamoComponentDe // podTemplate -> mainContainer + extraPodSpec + extraPodMetadata + // Resources + Envs + Probes (+ FrontendSidecar). - if err := decomposePodTemplateFromHub(src, dst, restored); err != nil { - return err - } + decomposePodTemplateFromHub(src, dst, restored) // Restore lossy cache flags before field-origin reconstruction so every // compilation-cache mount is excluded from main-container origin matching. @@ -1502,6 +1506,9 @@ func mergeExtraPodSpecMainContainer(src *DynamoComponentDeploymentSharedSpec, ma // appended the dedicated service-level mounts. Preserve that ordering rather // than allowing mergo to replace the VolumeMounts slice. mainBase.VolumeMounts = append(slices.Clone(main.VolumeMounts), dedicatedVolumeMounts...) + if main.Ports != nil { + mainBase.Ports = slices.Clone(main.Ports) + } // StartupProbe has no dedicated v1alpha1 field; take it verbatim. if main.StartupProbe != nil { mainBase.StartupProbe = main.StartupProbe @@ -1580,12 +1587,13 @@ func buildMainContainerFromDedicated(src *DynamoComponentDeploymentSharedSpec) c } // decomposePodTemplateFromHub inverts buildPodTemplateToHub. -func decomposePodTemplateFromHub(src *v1beta1.DynamoComponentDeploymentSharedSpec, dst, restored *DynamoComponentDeploymentSharedSpec) error { +func decomposePodTemplateFromHub(src *v1beta1.DynamoComponentDeploymentSharedSpec, dst, restored *DynamoComponentDeploymentSharedSpec) { if src.PodTemplate == nil { decomposeMissingPodTemplate(src, dst, restored) - return nil + return } + // Split one owned copy so projected fields can be consumed without changing src. podTpl := src.PodTemplate.DeepCopy() // ExtraPodMetadata from podTemplate.metadata. @@ -1596,8 +1604,8 @@ func decomposePodTemplateFromHub(src *v1beta1.DynamoComponentDeploymentSharedSpe other := make([]corev1.Container, 0, len(podTpl.Spec.Containers)) for i := range podTpl.Spec.Containers { if podTpl.Spec.Containers[i].Name == mainContainerName && main == nil { - m := podTpl.Spec.Containers[i].DeepCopy() - main = m + m := podTpl.Spec.Containers[i] + main = &m continue } other = append(other, podTpl.Spec.Containers[i]) @@ -1614,33 +1622,26 @@ func decomposePodTemplateFromHub(src *v1beta1.DynamoComponentDeploymentSharedSpe // that v1alpha1 can represent directly have already been moved into their // dedicated fields and cleared from main, so ExtraPodSpec only carries the // true escape-hatch remainder. - podSpecCopy := podTpl.Spec.DeepCopy() - podSpecCopy.Containers = other + podSpec := podTpl.Spec + podSpec.Containers = other // The forward path (buildPodTemplateToHub) always emits a "main" container, // even when v1alpha1 had no main-container fields set (e.g. only // FrontendSidecar triggered hasAny). Skip recording it on the v1alpha1 // side when every field other than Name is zero-valued, so that // ConvertFrom does not hallucinate an empty MainContainer. - var mainCopy *corev1.Container if main != nil { - m := main.DeepCopy() - m.Name = "" // v1alpha1 MainContainer has no Name (it is always "main"). - if !containerIsEmpty(m) { - mainCopy = m + main.Name = "" // v1alpha1 MainContainer has no Name (it is always "main"). + if containerIsEmpty(main) { + main = nil } } - if !podSpecIsZero(podSpecCopy) || mainCopy != nil { - eps := &ExtraPodSpec{} - if !podSpecIsZero(podSpecCopy) { - eps.PodSpec = podSpecCopy - } - if mainCopy != nil { - eps.MainContainer = mainCopy + if !podSpecIsZero(&podSpec) || main != nil { + eps := &ExtraPodSpec{MainContainer: main} + if !podSpecIsZero(&podSpec) { + eps.PodSpec = &podSpec } dst.ExtraPodSpec = eps } - - return nil } func decomposeMissingPodTemplate(src *v1beta1.DynamoComponentDeploymentSharedSpec, dst, restored *DynamoComponentDeploymentSharedSpec) { @@ -1702,6 +1703,7 @@ func restoreSharedHubOnlyFields(dst, preserved *v1beta1.DynamoComponentDeploymen return err } dst.PodTemplate = podTemplate + restoreSharedHubOnlyFrontendSidecar(dst, preserved) if dst.Experimental == nil && experimentalIsHubOnlyShape(preserved.Experimental) { dst.Experimental = preserved.Experimental.DeepCopy() diff --git a/deploy/operator/api/v1alpha1/shared_spec_conversion_bugs_test.go b/deploy/operator/api/v1alpha1/shared_spec_conversion_bugs_test.go index 766b2cac76ee..84cd5475d124 100644 --- a/deploy/operator/api/v1alpha1/shared_spec_conversion_bugs_test.go +++ b/deploy/operator/api/v1alpha1/shared_spec_conversion_bugs_test.go @@ -209,6 +209,99 @@ func TestBugDGD_HubVolumeMountOrderWithCompilationCacheRoundTrip(t *testing.T) { } } +func TestBugDGD_HubCarrierMountOrderRespectsLiveEdits(t *testing.T) { + t.Log("Define a compilation cache and native mounts with complete optional fields") + recursiveReadOnly := corev1.RecursiveReadOnlyEnabled + spoke := &DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "live-volume-mount-order", Namespace: "ns"}, + Spec: DynamoGraphDeploymentSpec{ + Services: map[string]*DynamoComponentDeploymentSharedSpec{ + "worker": { + ComponentType: "worker", + VolumeMounts: []VolumeMount{{ + Name: "compile-cache", + MountPoint: "/compile", + UseAsCompilationCache: true, + }}, + ExtraPodSpec: &ExtraPodSpec{ + MainContainer: &corev1.Container{ + Ports: []corev1.ContainerPort{}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config-a", + MountPath: "/config-a", + ReadOnly: true, + RecursiveReadOnly: &recursiveReadOnly, + SubPath: "worker", + }, + {Name: "config-b", MountPath: "/config-b", SubPath: "worker"}, + }, + }, + }, + }, + }, + }, + } + + t.Log("Create a hub carrier whose mount order is lossy in v1alpha1") + carrier := &v1beta1.DynamoGraphDeployment{} + if err := spoke.ConvertTo(carrier); err != nil { + t.Fatalf("ConvertTo() error = %v", err) + } + + t.Log("Renaming invalidates the alpha-origin save; the hub payload must preserve mount order") + carrier.Spec.Components[0].ComponentName = "renamed-worker" + originalMain, ok := findContainerByName(carrier.Spec.Components[0].PodTemplate.Spec.Containers, mainContainerName) + if !ok { + t.Fatalf("expected carrier main container, got %#v", carrier.Spec.Components[0].PodTemplate.Spec.Containers) + } + originalMounts := cloneNativeVolumeMounts(originalMain.VolumeMounts) + + t.Log("Preserve explicit empty ports in the native hub carrier") + if originalMain.Ports == nil { + t.Fatalf("main.Ports = nil, want explicit empty slice") + } + if len(originalMain.Ports) != 0 { + t.Fatalf("main.Ports = %#v, want empty", originalMain.Ports) + } + + t.Log("Convert to the cache-first spoke projection") + converted := &DynamoGraphDeployment{} + if err := converted.ConvertFrom(carrier); err != nil { + t.Fatalf("ConvertFrom() error = %v", err) + } + service := converted.Spec.Services["renamed-worker"] + if service == nil || len(service.VolumeMounts) != 3 { + t.Fatalf("expected three projected volume mounts, got %#v", service) + } + + t.Log("Restore the saved hub order while the live projection is unchanged") + unedited := &v1beta1.DynamoGraphDeployment{} + if err := converted.ConvertTo(unedited); err != nil { + t.Fatalf("ConvertTo() unedited error = %v", err) + } + if diff := cmp.Diff(originalMounts, unedited.Spec.Components[0].PodTemplate.Spec.Containers[0].VolumeMounts); diff != "" { + t.Fatalf("main volume-mount order changed across hub carrier round-trip (-want +got):\n%s", diff) + } + + t.Log("Reorder the live mounts without deleting the saved hub payload") + service.VolumeMounts[1], service.VolumeMounts[2] = service.VolumeMounts[2], service.VolumeMounts[1] + + t.Log("Convert back without allowing the stale hub save to override the live order") + restored := &v1beta1.DynamoGraphDeployment{} + if err := converted.ConvertTo(restored); err != nil { + t.Fatalf("ConvertTo() restored error = %v", err) + } + restoredMain, ok := findContainerByName(restored.Spec.Components[0].PodTemplate.Spec.Containers, mainContainerName) + if !ok { + t.Fatalf("expected restored main container, got %#v", restored.Spec.Components[0].PodTemplate.Spec.Containers) + } + want := []corev1.VolumeMount{originalMounts[2], originalMounts[1], originalMounts[0]} + if diff := cmp.Diff(want, restoredMain.VolumeMounts); diff != "" { + t.Fatalf("live main volume-mount order was overridden (-want +got):\n%s", diff) + } +} + func addGeneratedFrontendSidecarHubOnlySecurityContext(t *testing.T, podTemplate *corev1.PodTemplateSpec) { t.Helper() if podTemplate == nil { diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index bee15c5695c8..10736812e665 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,7 @@ limitations under the License. package v1alpha1 import ( + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "k8s.io/api/autoscaling/v2" "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -456,6 +457,11 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent *out = new(FrontendSidecarSpec) (*in).DeepCopyInto(*out) } + if in.LPX != nil { + in, out := &in.LPX, &out.LPX + *out = new(v1beta1.LPXConfig) + (*in).DeepCopyInto(*out) + } if in.Checkpoint != nil { in, out := &in.Checkpoint, &out.Checkpoint *out = new(ServiceCheckpointConfig) @@ -851,6 +857,11 @@ func (in *DynamoGraphDeploymentSpec) DeepCopyInto(out *DynamoGraphDeploymentSpec *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } + if in.Scheduling != nil { + in, out := &in.Scheduling, &out.Scheduling + *out = new(v1beta1.SchedulingSpec) + (*in).DeepCopyInto(*out) + } if in.Annotations != nil { in, out := &in.Annotations, &out.Annotations *out = make(map[string]string, len(*in)) @@ -951,16 +962,16 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.LPX != nil { + in, out := &in.LPX, &out.LPX + *out = new(v1beta1.DynamoGraphDeploymentLPXStatus) + (*in).DeepCopyInto(*out) + } if in.RollingUpdate != nil { in, out := &in.RollingUpdate, &out.RollingUpdate *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } - if in.Placement != nil { - in, out := &in.Placement, &out.Placement - *out = new(PlacementStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. @@ -1352,6 +1363,119 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXGraphDeployment) DeepCopyInto(out *LPXGraphDeployment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXGraphDeployment. +func (in *LPXGraphDeployment) DeepCopy() *LPXGraphDeployment { + if in == nil { + return nil + } + out := new(LPXGraphDeployment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LPXGraphDeployment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXGraphDeploymentList) DeepCopyInto(out *LPXGraphDeploymentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LPXGraphDeployment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXGraphDeploymentList. +func (in *LPXGraphDeploymentList) DeepCopy() *LPXGraphDeploymentList { + if in == nil { + return nil + } + out := new(LPXGraphDeploymentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LPXGraphDeploymentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXGraphDeploymentSpec) DeepCopyInto(out *LPXGraphDeploymentSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXGraphDeploymentSpec. +func (in *LPXGraphDeploymentSpec) DeepCopy() *LPXGraphDeploymentSpec { + if in == nil { + return nil + } + out := new(LPXGraphDeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make(map[string]v1beta1.ComponentReplicaStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.ModelDownload != nil { + in, out := &in.ModelDownload, &out.ModelDownload + *out = new(v1beta1.ModelDownloadStatus) + (*in).DeepCopyInto(*out) + } + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(v1beta1.PlacementStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXGraphDeploymentStatus. +func (in *LPXGraphDeploymentStatus) DeepCopy() *LPXGraphDeploymentStatus { + if in == nil { + return nil + } + out := new(LPXGraphDeploymentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LabelItemSchema) DeepCopyInto(out *LabelItemSchema) { *out = *in @@ -1438,26 +1562,6 @@ func (in *PVC) DeepCopy() *PVC { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { - *out = *in - if in.Score != nil { - in, out := &in.Score, &out.Score - *out = new(float64) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. -func (in *PlacementStatus) DeepCopy() *PlacementStatus { - if in == nil { - return nil - } - out := new(PlacementStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProfilingConfigSpec) DeepCopyInto(out *ProfilingConfigSpec) { *out = *in diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index dd8aca2e162d..6bdf8e9319ec 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -29,7 +29,7 @@ import ( // was used as a workaround for disaggregated serving), `prefill` and `decode` // are first-class values: users can set them directly and downstream consumers // (e.g., the EPP) can filter on the pod label `nvidia.com/dynamo-component-type`. -// +kubebuilder:validation:Enum=frontend;worker;prefill;decode;planner;epp +// +kubebuilder:validation:Enum=frontend;worker;prefill;decode;planner;epp;lpx type ComponentType string const ( @@ -39,6 +39,7 @@ const ( ComponentTypeDecode ComponentType = "decode" ComponentTypePlanner ComponentType = "planner" ComponentTypeEPP ComponentType = "epp" + ComponentTypeLPX ComponentType = "lpx" ) const ( @@ -136,11 +137,23 @@ type ComponentRoleSpec struct { // podTemplate defines the Pod configuration for this role. Admission permits // it only when the enclosing component type explicitly supports role-specific - // Pod templates. No component type supports it in this release. + // Pod templates. LPX components support role-specific Pod templates. // +optional PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } +// ComponentRole returns the authored role, or nil when the component does not declare it. +// The shared spec must not be nil. +func (s *DynamoComponentDeploymentSharedSpec) ComponentRole(name string) *ComponentRoleSpec { + // Resolve the role by its stable authored name. + for i := range s.Roles { + if s.Roles[i].Name == name { + return &s.Roles[i] + } + } + return nil +} + // MultinodeSpec configures a multinode component. type MultinodeSpec struct { // nodeCount is the number of nodes to deploy for the multinode component. @@ -509,10 +522,10 @@ type DynamoCheckpointIdentity struct { // +kubebuilder:validation:MinLength=1 Model string `json:"model"` - // backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). + // backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). // Deprecated: legacy identity only. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=vllm;sglang;trtllm + // +kubebuilder:validation:Enum=vllm;sglang;trtllm;lpu BackendFramework string `json:"backendFramework"` // dynamoVersion is the Dynamo platform version. @@ -714,6 +727,10 @@ type PlacementStatus struct { // for the semantics of each value. // +optional State PlacementScoreState `json:"state,omitempty"` + + // lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + // +optional + LPXAttempt *LPXAttemptStatus `json:"lpxAttempt,omitempty"` } // RestartPhase enumerates phases of a graph-level restart. @@ -825,6 +842,11 @@ type ComponentReplicaStatus struct { // +kubebuilder:validation:Minimum=0 GPUsPerReplica *int64 `json:"gpusPerReplica,omitempty"` + // ready is the binary determination of whether the correct number of replicas + // are scheduled and available. + // +optional + Ready bool `json:"ready"` + // replicas is the total number of non-terminated replicas. // +kubebuilder:validation:Minimum=0 Replicas int32 `json:"replicas"` diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 082e7d81df3c..28cdb65dfa2c 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -29,6 +29,18 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) +// +kubebuilder:validation:Enum=override;strategic +type ExtraPodSpecMergeStrategy string + +const ( + ExtraPodSpecMergeStrategyOverride ExtraPodSpecMergeStrategy = "override" + ExtraPodSpecMergeStrategyStrategic ExtraPodSpecMergeStrategy = "strategic" +) + +func (s ExtraPodSpecMergeStrategy) IsValid() bool { + return s == ExtraPodSpecMergeStrategyOverride || s == ExtraPodSpecMergeStrategyStrategic +} + const ( // DynamoComponentDeploymentConditionTypeAvailable indicates the component is // available and serving traffic. @@ -47,8 +59,10 @@ const ( ) // DynamoComponentDeploymentSpec defines the desired state of a DynamoComponentDeployment. +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx')",message="standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment" type DynamoComponentDeploymentSpec struct { - // backendFramework specifies the backend framework. + // backendFramework specifies the GPU backend framework (for example, + // "sglang", "vllm", or "trtllm"). // +kubebuilder:validation:Enum=sglang;vllm;trtllm BackendFramework string `json:"backendFramework,omitempty"` @@ -65,13 +79,19 @@ type DynamoComponentDeploymentSpec struct { // volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are // replaced with a single `podTemplate` field holding a native // `corev1.PodTemplateSpec`. The operator injects its defaults into the -// container named `"main"` and merges user overrides using strategic-merge-by-name -// semantics. Users can add sidecars, init containers, and pod-level configuration -// directly in `podTemplate` without any `extraPodSpec`-style escape hatch. +// container named `"main"` and merges user overrides according to +// `extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and +// pod-level configuration directly in `podTemplate` without any +// `extraPodSpec`-style escape hatch. // +kubebuilder:validation:XValidation:rule="!has(self.eppConfig) || (has(self.type) && self.type == 'epp')",message="eppConfig may only be set when type is epp" // +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.type) || (has(self.type) && self.type == oldSelf.type)",message="type is immutable after it is set" +// +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.type) && self.type == 'lpx')",message="lpx may only be set when type is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx') || has(self.lpx)",message="lpx is required when type is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx') || !has(self.podTemplate)",message="LPX Pod templates belong to roles" +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))",message="replicas must be between 1 and 9 when type is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx' && has(self.scalingAdapter))",message="scalingAdapter is not supported when type is lpx" type DynamoComponentDeploymentSharedSpec struct { // providerOverride configures the primary Grove unit representing this DGD // component. With apiVersion `grove.io/v1alpha1`, target is @@ -135,6 +155,13 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` + // extraPodSpecMergeStrategy controls how podTemplate is merged with the + // operator-generated pod and container defaults. When omitted, the operator + // uses its configured defaultExtraPodSpecMergeStrategy, which defaults to + // "override". + // +optional + ExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy `json:"extraPodSpecMergeStrategy,omitempty"` + // replicas is the desired number of Pods for this component. When // `scalingAdapter` is set on this component, this field is managed by // the DynamoGraphDeploymentScalingAdapter and should not be modified @@ -172,6 +199,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. + // LPX components require a worker role and may declare a leader role. // +optional // +listType=map // +listMapKey=name @@ -207,6 +235,11 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional EPPConfig *EPPConfig `json:"eppConfig,omitempty"` + // lpx holds LPX integration configuration. Only meaningful when + // `type` is `lpx`. + // +optional + LPX *LPXConfig `json:"lpx,omitempty"` + // frontendSidecar optionally designates a container in // `podTemplate.spec.containers` as the frontend sidecar. The value must // match the `name` of a container in that list; the operator merges its @@ -275,7 +308,7 @@ type DynamoComponentDeploymentStatus struct { // +kubebuilder:storageversion // +kubebuilder:resource:shortName=dcd // +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.conditions[?(@.type=='Available')].status",description="Available" -// +kubebuilder:printcolumn:name="Backend",type="string",JSONPath=`.spec.backendFramework`,description="Backend framework (sglang, vllm, trtllm)" +// +kubebuilder:printcolumn:name="Backend",type="string",JSONPath=`.spec.backendFramework`,description="GPU backend framework" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // DynamoComponentDeployment is the Schema for the dynamocomponentdeployments API. diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 583742860002..6dbd2204c80b 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -33,6 +33,10 @@ type DynamoGraphDeploymentSpec struct { // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` + // scheduling configures attempts for LPX components. Omission means no deadline. + // +optional + Scheduling *SchedulingSpec `json:"scheduling,omitempty"` + // annotations to propagate to all child resources (PCS, DCD, Deployments, // and pod templates). Component-level (`podTemplate`) values take precedence // on conflict. @@ -129,15 +133,14 @@ type DynamoGraphDeploymentStatus struct { // +optional Checkpoints map[string]ComponentCheckpointStatus `json:"checkpoints,omitempty"` + // lpx contains the status of the graph's LPX workload, when present. + // +optional + LPX *DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` + // rollingUpdate tracks the progress of operator-managed rolling updates. // Currently only supported for single-node, non-Grove deployments (DCD/Deployment). // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` - - // placement groups DGD-level scheduler placement signals (score, reporting - // state, and any future placement fields). - // +optional - Placement *PlacementStatus `json:"placement,omitempty"` } // DGD Ready condition reasons used to classify Grove-backed not-ready diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go new file mode 100644 index 000000000000..927f51097bb7 --- /dev/null +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1beta1 + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// LPXConfig identifies the component's compiled model and runtime settings. +type LPXConfig struct { + // buildId references the immutable model build. + // +kubebuilder:validation:MinLength=1 + BuildID string `json:"buildId"` + // settings override the existing model configuration. + // +optional + // +kubebuilder:validation:Type=object + // +nullable + Settings *apiextensionsv1.JSON `json:"settings,omitempty"` +} + +// SchedulingSpec configures LPX scheduling attempts. +type SchedulingSpec struct { + // attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + // Omission means unlimited; the value is not a solver budget. + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=9223372036 + AttemptDeadlineSeconds *int64 `json:"attemptDeadlineSeconds,omitempty"` +} + +// LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. +type LPXAttemptStatus struct { + ObservedGeneration int64 `json:"observedGeneration"` + PodCliqueSetUID types.UID `json:"podCliqueSetUID,omitempty"` + DeadlineAt *metav1.Time `json:"deadlineAt,omitempty"` + ExceededAt *metav1.Time `json:"exceededAt,omitempty"` + DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=9 + Requests []LPXAttemptRequestStatus `json:"requests"` +} + +// LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. +type LPXAttemptRequestStatus struct { + Name string `json:"name"` + AttemptDigest string `json:"attemptDigest"` + UID types.UID `json:"uid,omitempty"` +} + +// DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. +// Both served DGD versions use this type, so conversion preserves its live fields natively. +type DynamoGraphDeploymentLPXStatus struct { + // modelDownload contains the model download status for remote builds. + // +optional + ModelDownload *ModelDownloadStatus `json:"modelDownload,omitempty"` + // placement contains LPX scheduling progress and placement signals. + // +optional + Placement *PlacementStatus `json:"placement,omitempty"` +} + +// ModelDownloadStatus contains the status of remote LPU model downloads. +type ModelDownloadStatus struct { + // builds is the sorted set of resolved remote LPU build URLs whose artifacts + // were successfully downloaded into model-storage. + // +optional + Builds []string `json:"builds,omitempty"` + + // lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. + // +optional + LastCheckedAt *metav1.Time `json:"lastCheckedAt,omitempty"` +} + +// HasLPXComponent returns true if any component uses the LPX integration. +func (s *DynamoGraphDeployment) HasLPXComponent() bool { + for i := range s.Spec.Components { + if s.Spec.Components[i].IsLPX() { + return true + } + } + return false +} + +// IsLPX reports whether this shared spec uses the LPX integration. +func (s *DynamoComponentDeploymentSharedSpec) IsLPX() bool { + return s.ComponentType == ComponentTypeLPX +} + +// ManagedByExternalController reports whether a dedicated controller, rather than +// the ordinary DGD workload path, manages this component. Ownership is determined +// by the component type and does not change when its integration is disabled. +func (s *DynamoComponentDeploymentSharedSpec) ManagedByExternalController() bool { + return s.ComponentType == ComponentTypeLPX +} diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 93415e44d010..295593c91078 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -40,6 +40,7 @@ package v1beta1 import ( batchv1 "k8s.io/api/batch/v1" "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1" @@ -366,6 +367,11 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent *out = new(EPPConfig) (*in).DeepCopyInto(*out) } + if in.LPX != nil { + in, out := &in.LPX, &out.LPX + *out = new(LPXConfig) + (*in).DeepCopyInto(*out) + } if in.FrontendSidecar != nil { in, out := &in.FrontendSidecar, &out.FrontendSidecar *out = new(string) @@ -503,6 +509,31 @@ func (in *DynamoGraphDeploymentExperimentalSpec) DeepCopy() *DynamoGraphDeployme return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DynamoGraphDeploymentLPXStatus) DeepCopyInto(out *DynamoGraphDeploymentLPXStatus) { + *out = *in + if in.ModelDownload != nil { + in, out := &in.ModelDownload, &out.ModelDownload + *out = new(ModelDownloadStatus) + (*in).DeepCopyInto(*out) + } + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentLPXStatus. +func (in *DynamoGraphDeploymentLPXStatus) DeepCopy() *DynamoGraphDeploymentLPXStatus { + if in == nil { + return nil + } + out := new(DynamoGraphDeploymentLPXStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DynamoGraphDeploymentList) DeepCopyInto(out *DynamoGraphDeploymentList) { *out = *in @@ -778,6 +809,11 @@ func (in *DynamoGraphDeploymentSpec) DeepCopyInto(out *DynamoGraphDeploymentSpec *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } + if in.Scheduling != nil { + in, out := &in.Scheduling, &out.Scheduling + *out = new(SchedulingSpec) + (*in).DeepCopyInto(*out) + } if in.Annotations != nil { in, out := &in.Annotations, &out.Annotations *out = make(map[string]string, len(*in)) @@ -862,16 +898,16 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.LPX != nil { + in, out := &in.LPX, &out.LPX + *out = new(DynamoGraphDeploymentLPXStatus) + (*in).DeepCopyInto(*out) + } if in.RollingUpdate != nil { in, out := &in.RollingUpdate, &out.RollingUpdate *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } - if in.Placement != nil { - in, out := &in.Placement, &out.Placement - *out = new(PlacementStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. @@ -1122,6 +1158,73 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXAttemptRequestStatus) DeepCopyInto(out *LPXAttemptRequestStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptRequestStatus. +func (in *LPXAttemptRequestStatus) DeepCopy() *LPXAttemptRequestStatus { + if in == nil { + return nil + } + out := new(LPXAttemptRequestStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXAttemptStatus) DeepCopyInto(out *LPXAttemptStatus) { + *out = *in + if in.DeadlineAt != nil { + in, out := &in.DeadlineAt, &out.DeadlineAt + *out = (*in).DeepCopy() + } + if in.ExceededAt != nil { + in, out := &in.ExceededAt, &out.ExceededAt + *out = (*in).DeepCopy() + } + if in.DisarmedAt != nil { + in, out := &in.DisarmedAt, &out.DisarmedAt + *out = (*in).DeepCopy() + } + if in.Requests != nil { + in, out := &in.Requests, &out.Requests + *out = make([]LPXAttemptRequestStatus, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptStatus. +func (in *LPXAttemptStatus) DeepCopy() *LPXAttemptStatus { + if in == nil { + return nil + } + out := new(LPXAttemptStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXConfig) DeepCopyInto(out *LPXConfig) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = new(apiextensionsv1.JSON) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXConfig. +func (in *LPXConfig) DeepCopy() *LPXConfig { + if in == nil { + return nil + } + out := new(LPXConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MockerSpec) DeepCopyInto(out *MockerSpec) { *out = *in @@ -1152,6 +1255,30 @@ func (in *ModelCacheSpec) DeepCopy() *ModelCacheSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelDownloadStatus) DeepCopyInto(out *ModelDownloadStatus) { + *out = *in + if in.Builds != nil { + in, out := &in.Builds, &out.Builds + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LastCheckedAt != nil { + in, out := &in.LastCheckedAt, &out.LastCheckedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelDownloadStatus. +func (in *ModelDownloadStatus) DeepCopy() *ModelDownloadStatus { + if in == nil { + return nil + } + out := new(ModelDownloadStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelReference) DeepCopyInto(out *ModelReference) { *out = *in @@ -1231,6 +1358,11 @@ func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { *out = new(float64) **out = **in } + if in.LPXAttempt != nil { + in, out := &in.LPXAttempt, &out.LPXAttempt + *out = new(LPXAttemptStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. @@ -1424,6 +1556,26 @@ func (in *ScalingAdapter) DeepCopy() *ScalingAdapter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingSpec) DeepCopyInto(out *SchedulingSpec) { + *out = *in + if in.AttemptDeadlineSeconds != nil { + in, out := &in.AttemptDeadlineSeconds, &out.AttemptDeadlineSeconds + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingSpec. +func (in *SchedulingSpec) DeepCopy() *SchedulingSpec { + if in == nil { + return nil + } + out := new(SchedulingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SpecTopologyConstraint) DeepCopyInto(out *SpecTopologyConstraint) { *out = *in diff --git a/deploy/operator/cmd/crd-apply/main.go b/deploy/operator/cmd/crd-apply/main.go index fcd6d08661a1..8f87809c8e67 100644 --- a/deploy/operator/cmd/crd-apply/main.go +++ b/deploy/operator/cmd/crd-apply/main.go @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "flag" "fmt" "os" @@ -100,7 +101,8 @@ func main() { os.Exit(1) } - patchData, err := yaml.Marshal(crd) + // Compact JSON avoids expanding large CRD schemas into oversized YAML requests. + patchData, err := json.Marshal(crd) if err != nil { log.Error(err, "unable to marshal CRD for patch", "crd", crd.Name) os.Exit(1) diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go new file mode 100644 index 000000000000..fe6c3fb65d0a --- /dev/null +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -0,0 +1,137 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { + t.Log("Start a local API server with its default request and storage limits") + env := &envtest.Environment{UseExistingCluster: ptr.To(false)} + config, err := env.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := env.Stop(); err != nil { + t.Error(err) + } + }) + + t.Log("Give the installer credentials for only this ephemeral API server") + dir := t.TempDir() + kubeconfig := filepath.Join(dir, "kubeconfig") + if err := clientcmd.WriteToFile(clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{"test": { + Server: config.Host, CertificateAuthorityData: config.CAData, + }}, + AuthInfos: map[string]*clientcmdapi.AuthInfo{"test": { + ClientCertificateData: config.CertData, ClientKeyData: config.KeyData, + }}, + Contexts: map[string]*clientcmdapi.Context{"test": {Cluster: "test", AuthInfo: "test"}}, + CurrentContext: "test", + }, kubeconfig); err != nil { + t.Fatal(err) + } + + t.Log("Build the production CRD installer") + installer := filepath.Join(dir, "crd-apply") + if output, err := exec.CommandContext(t.Context(), "go", "build", "-o", installer, ".").CombinedOutput(); err != nil { + t.Fatalf("build installer: %v\n%s", err, output) + } + + t.Log("Install and reapply every generated CRD, optionally upgrading an external baseline") + crdDirectories := []string{"../../config/crd/bases", "../../config/crd/bases"} + if baseline := os.Getenv("CRD_APPLY_BASELINE_DIR"); baseline != "" { + crdDirectories = append([]string{baseline}, crdDirectories...) + } + for _, crdDirectory := range crdDirectories { + command := exec.CommandContext(t.Context(), installer, + "--crds-dir", crdDirectory, + "--version", "schema-test", + "--conversion-webhook-service-name", "dynamo-operator-webhook-service", + "--conversion-webhook-service-namespace", "dynamo-system", + ) + command.Env = append(os.Environ(), "KUBECONFIG="+kubeconfig) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("apply CRDs: %v\n%s", err, output) + } + } + + t.Log("Verify installed schemas keep public LPX status grouped and private child status flat") + client, err := apiextensionsclient.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{ + "dynamographdeployments.nvidia.com", "dynamocomponentdeployments.nvidia.com", "lpxgraphdeployments.nvidia.com", + } { + crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if crd.Annotations[versionAnnotation] != "schema-test" { + t.Errorf("CRD %s is missing the installer version", name) + } + if name == "lpxgraphdeployments.nvidia.com" { + t.Log("Keep the private LGD exclusively alpha, without conversion") + require.Len(t, crd.Spec.Versions, 1) + version := crd.Spec.Versions[0] + require.Equal(t, "v1alpha1", version.Name) + require.True(t, version.Served) + require.True(t, version.Storage) + require.Equal(t, []string{version.Name}, crd.Status.StoredVersions) + require.Equal(t, + &apiextensionsv1.CustomResourceConversion{Strategy: apiextensionsv1.NoneConverter}, + crd.Spec.Conversion) + } + if name == "dynamocomponentdeployments.nvidia.com" { + t.Log("Retain ordinary template schemas and documentation in both served DCD versions") + for _, version := range crd.Spec.Versions { + templateField := "podTemplate" + if version.Name == "v1alpha1" { + templateField = "extraPodSpec" + } + template := version.Schema.OpenAPIV3Schema.Properties["spec"].Properties[templateField] + if template.Type != "object" || template.Description == "" { + t.Errorf("%s %s is missing the documented %s schema", name, version.Name, templateField) + } + } + continue + } + for _, version := range crd.Spec.Versions { + status := version.Schema.OpenAPIV3Schema.Properties["status"] + if name == "dynamographdeployments.nvidia.com" { + for _, field := range []string{"modelDownload", "placement"} { + if _, exists := status.Properties[field]; exists { + t.Errorf("%s %s retains status.%s", name, version.Name, field) + } + } + status = status.Properties["lpx"] + } else if _, exists := status.Properties["lpx"]; exists { + t.Errorf("%s %s unexpectedly groups private child status", name, version.Name) + } + for _, field := range []string{"modelDownload", "placement"} { + if status.Properties[field].Type != "object" { + t.Errorf("%s %s is missing the typed %s projection", name, version.Name, field) + } + } + } + } +} diff --git a/deploy/operator/cmd/crd-apply/schema_test.go b/deploy/operator/cmd/crd-apply/schema_test.go new file mode 100644 index 000000000000..252002f6980e --- /dev/null +++ b/deploy/operator/cmd/crd-apply/schema_test.go @@ -0,0 +1,168 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package main + +import ( + "bytes" + "encoding/json" + "os/exec" + "reflect" + "testing" + + "sigs.k8s.io/yaml" +) + +func TestLPXSchemaCompaction(t *testing.T) { + // Exercise native validation and merge metadata without copying an entire generated CRD. + const podSchema = ` +description: A role-specific Pod template. +type: object +required: [spec] +properties: + metadata: + type: object + properties: + labels: + description: Pod labels. + type: object + additionalProperties: {type: string} + spec: + description: Native PodSpec. + type: object + required: [containers] + properties: + containers: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [name] + items: + type: object + required: [name] + x-kubernetes-validations: + - rule: self.name != '' + message: Name must not be empty. + properties: + name: {type: string} + image: {type: string} + restartPolicy: {type: string, default: Always, enum: [Always, Never]} +` + // Keep absence, equal schemas, and role-specific constraints in one behavioral matrix. + for _, test := range []struct { + name string + withLPX bool + different bool + flow bool + aliases int + }{ + {name: "no LPX"}, + {name: "identical schemas", withLPX: true, aliases: 1}, + {name: "identical schemas on one line", withLPX: true, flow: true, aliases: 1}, + {name: "different constraints", withLPX: true, different: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Render role templates in two API versions alongside unchanged ordinary schemas") + var alphaRoles, betaRoles, ordinary, undocumented map[string]any + for _, target := range []*map[string]any{&alphaRoles, &betaRoles, &ordinary, &undocumented} { + if err := yaml.Unmarshal([]byte(podSchema), target); err != nil { + t.Fatal(err) + } + } + if test.different { + betaRoles["required"] = []any{"metadata", "spec"} + } + removeSchemaDescriptions(undocumented) + input := map[string]any{ + "podTemplate": ordinary, + "extraPodSpec": map[string]any{"description": "Alpha documentation.", "properties": ordinary}, + "unrelated": map[string]any{"podTemplate": undocumented}, + } + if test.withLPX { + input["versions"] = []any{ + map[string]any{"properties": map[string]any{ + "lpx": map[string]any{"type": "object"}, + "roles": map[string]any{"items": map[string]any{"properties": map[string]any{ + "podTemplate": alphaRoles, + }}}, + }}, + map[string]any{"properties": map[string]any{ + "lpx": map[string]any{"type": "object"}, + "roles": map[string]any{"items": map[string]any{"properties": map[string]any{ + "podTemplate": betaRoles, + }}}, + }}, + } + } + original, err := yaml.Marshal(input) + if test.flow { + original, err = json.Marshal(input) + } + if err != nil { + t.Fatal(err) + } + + t.Log("Run the production transformation and decode its aliases using the installer YAML library") + output := compactLPXSchema(t, original) + var actual map[string]any + if err := yaml.Unmarshal(output, &actual); err != nil { + t.Fatalf("installer cannot decode compacted schema: %v\n%s", err, output) + } + if test.withLPX { + removeSchemaDescriptions(alphaRoles) + removeSchemaDescriptions(betaRoles) + } + if !reflect.DeepEqual(actual, input) { + t.Fatalf("schema semantics changed beyond LPX descriptions:\n%s", output) + } + + t.Log("Check alias scope and physical order, including an ordinary identical schema") + if got := bytes.Count(output, []byte("*lpx-pod-template")); got != test.aliases { + t.Errorf("alias count = %d, want %d", got, test.aliases) + } + anchor := bytes.Index(output, []byte("&lpx-pod-template")) + alias := bytes.Index(output, []byte("*lpx-pod-template")) + if test.aliases > 0 && anchor > alias { + t.Error("alias precedes its anchor") + } + if !test.withLPX && bytes.Contains(output, []byte("&lpx-pod-template")) { + t.Error("ordinary schema gained an LPX anchor") + } + + t.Log("Require a stable second transformation") + if again := compactLPXSchema(t, output); !bytes.Equal(output, again) { + t.Errorf("second transformation changed the YAML:\n%s", again) + } + }) + } +} + +func compactLPXSchema(t *testing.T, input []byte) []byte { + t.Helper() + + // Use the same pinned yq binary and expression as make manifests. + command := exec.CommandContext(t.Context(), "../../bin/yq", + "--from-file", "../../api/scripts/lpx-crd-schema.yq", "--indent", "2") + command.Stdin = bytes.NewReader(input) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("compact LPX schema (run make yq first): %v\n%s", err, output) + } + return output +} + +func removeSchemaDescriptions(value any) { + // Remove documentation recursively without interpreting or changing any validation marker. + switch value := value.(type) { + case map[string]any: + delete(value, "description") + for _, child := range value { + removeSchemaDescriptions(child) + } + case []any: + for _, child := range value { + removeSchemaDescriptions(child) + } + } +} diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 3da73d70bd38..b0d3cd1d3d43 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -73,8 +73,10 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/rbac" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" istioclientsetscheme "istio.io/client-go/pkg/clientset/versioned/scheme" gaiev1 "sigs.k8s.io/gateway-api-inference-extension/api/v1" @@ -126,6 +128,10 @@ func initCRDSchemes() { // operator only consumes them (creates/reads), it does not reconcile them. utilruntime.Must(snapshotv1alpha1.AddToScheme(crdScheme)) + utilruntime.Must(groveschedulerv1alpha1.AddToScheme(crdScheme)) + + utilruntime.Must(lpxv1alpha1.AddToScheme(crdScheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(crdScheme)) utilruntime.Must(admissionregistrationv1.AddToScheme(crdScheme)) diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 737b042a2471..8657d375c1a4 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -667,7 +667,9 @@ spec: type: integer type: object backendFramework: - description: BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") + description: |- + BackendFramework specifies the GPU backend framework (for example, + "sglang", "vllm", or "trtllm"). enum: - sglang - vllm @@ -711,12 +713,13 @@ spec: properties: backendFramework: description: |- - BackendFramework is the runtime framework (vllm, sglang, trtllm). + BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm + - lpu type: string dtype: description: |- @@ -10700,6 +10703,14 @@ spec: x-kubernetes-list-type: map required: [] type: object + extraPodSpecMergeStrategy: + description: |- + ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the + operator-generated pod and container defaults. + enum: + - override + - strategic + type: string failover: description: |- Failover configures GMS (GPU Memory Service) failover for this service. @@ -11465,6 +11476,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components require a worker role and may declare a leader role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -11482,7 +11494,7 @@ spec: description: |- PodTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific - Pod templates. No component type supports it in this release. + Pod templates. LPX components support role-specific Pod templates. properties: metadata: description: |- @@ -19706,12 +19718,20 @@ spec: type: array type: object x-kubernetes-validations: + - message: standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment + rule: '!(has(self.componentType) && self.componentType == ''lpx'')' - message: minAvailable must be less than or equal to replicas unless replicas is 0 rule: '!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' - message: minAvailable is immutable after creation rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set rule: '!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)' + - message: componentType lpu is not supported; use lpx + rule: '!has(self.componentType) || self.componentType != ''lpu''' + - message: replicas must be between 1 and 9 when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: scalingAdapter is not supported when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)' status: description: Status reflects the current observed state of the component deployment. properties: @@ -19839,6 +19859,11 @@ spec: format: int64 minimum: 0 type: integer + ready: + description: |- + Ready is the binary determination of whether the correct number of replicas + are scheduled and available. + type: boolean readyReplicas: description: |- ReadyReplicas is the number of ready replicas. @@ -19897,7 +19922,7 @@ spec: jsonPath: .status.conditions[?(@.type=='Available')].status name: Available type: string - - description: Backend framework (sglang, vllm, trtllm) + - description: GPU backend framework jsonPath: .spec.backendFramework name: Backend type: string @@ -19935,7 +19960,9 @@ spec: description: spec defines the desired state for this Dynamo component deployment. properties: backendFramework: - description: backendFramework specifies the backend framework. + description: |- + backendFramework specifies the GPU backend framework (for example, + "sglang", "vllm", or "trtllm"). enum: - sglang - vllm @@ -20180,12 +20207,13 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). + backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm + - lpu type: string dtype: description: |- @@ -20404,6 +20432,16 @@ spec: type: boolean type: object type: object + extraPodSpecMergeStrategy: + description: |- + extraPodSpecMergeStrategy controls how podTemplate is merged with the + operator-generated pod and container defaults. When omitted, the operator + uses its configured defaultExtraPodSpecMergeStrategy, which defaults to + "override". + enum: + - override + - strategic + type: string frontendSidecar: description: |- frontendSidecar optionally designates a container in @@ -28641,6 +28679,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components require a worker role and may declare a leader role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -28658,7 +28697,7 @@ spec: description: |- podTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific - Pod templates. No component type supports it in this release. + Pod templates. LPX components support role-specific Pod templates. properties: metadata: description: |- @@ -36846,10 +36885,13 @@ spec: - decode - planner - epp + - lpx type: string required: [] type: object x-kubernetes-validations: + - message: standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment + rule: '!(has(self.type) && self.type == ''lpx'')' - message: eppConfig may only be set when type is epp rule: '!has(self.eppConfig) || (has(self.type) && self.type == ''epp'')' - message: minAvailable must be less than or equal to replicas unless replicas is 0 @@ -36858,6 +36900,12 @@ spec: rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: type is immutable after it is set rule: '!has(oldSelf.type) || (has(self.type) && self.type == oldSelf.type)' + - message: LPX Pod templates belong to roles + rule: '!(has(self.type) && self.type == ''lpx'') || !has(self.podTemplate)' + - message: replicas must be between 1 and 9 when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: scalingAdapter is not supported when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.scalingAdapter))' status: description: status reflects the current observed state of the component deployment. properties: @@ -36910,6 +36958,11 @@ spec: format: int64 minimum: 0 type: integer + ready: + description: |- + ready is the binary determination of whether the correct number of replicas + are scheduled and available. + type: boolean readyReplicas: description: |- readyReplicas is the number of ready replicas. Populated for diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 8cde37c3d564..f1de4853be29 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -398,6 +398,18 @@ spec: required: - id type: object + scheduling: + description: Scheduling configures attempts for LPX components. Omission means no deadline. + properties: + attemptDeadlineSeconds: + description: |- + attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + Omission means unlimited; the value is not a solver budget. + format: int64 + maximum: 9223372036 + minimum: 1 + type: integer + type: object services: additionalProperties: properties: @@ -1042,12 +1054,13 @@ spec: properties: backendFramework: description: |- - BackendFramework is the runtime framework (vllm, sglang, trtllm). + BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm + - lpu type: string dtype: description: |- @@ -11031,6 +11044,14 @@ spec: x-kubernetes-list-type: map required: [] type: object + extraPodSpecMergeStrategy: + description: |- + ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the + operator-generated pod and container defaults. + enum: + - override + - strategic + type: string failover: description: |- Failover configures GMS (GPU Memory Service) failover for this service. @@ -11508,6 +11529,23 @@ spec: format: int32 type: integer type: object + lpx: + description: |- + LPX holds LPX integration configuration. Only meaningful when + ComponentType is "lpx". + properties: + buildId: + description: buildId references the immutable model build. + minLength: 1 + type: string + settings: + description: settings override the existing model configuration. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - buildId + type: object minAvailable: description: |- MinAvailable maps to Grove PodClique minAvailable for single-node and @@ -11828,6 +11866,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components require a worker role and may declare a leader role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -11841,16 +11880,9 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: - description: |- - PodTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. No component type supports it in this release. + podTemplate: &lpx-pod-template properties: metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata properties: annotations: additionalProperties: @@ -11870,64 +11902,27 @@ spec: type: string type: object spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. format: int64 type: integer affinity: - description: If specified, the pod's scheduling constraints properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -11939,27 +11934,13 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -11973,7 +11954,6 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -11983,43 +11963,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -12031,27 +11986,13 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -12072,51 +12013,22 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12130,70 +12042,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12207,38 +12078,20 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. format: int32 type: integer required: @@ -12248,49 +12101,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12304,70 +12126,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12381,30 +12162,15 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12413,51 +12179,22 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12471,70 +12208,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12548,38 +12244,20 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. format: int32 type: integer required: @@ -12589,49 +12267,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12645,70 +12292,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12722,30 +12328,15 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12755,138 +12346,62 @@ spec: type: object type: object automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. type: boolean containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. items: - description: A single application container that you want to run within a pod. properties: args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -12895,44 +12410,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -12947,52 +12447,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13000,67 +12473,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13069,104 +12506,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13175,130 +12566,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13307,129 +12643,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. type: string ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -13440,71 +12709,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13513,99 +12750,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -13614,34 +12803,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -13657,9 +12825,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -13668,69 +12833,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -13745,261 +12861,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14008,134 +12974,61 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -14146,64 +13039,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -14214,11 +13064,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -14228,203 +13073,87 @@ spec: - name x-kubernetes-list-type: map dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. properties: nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. items: type: string type: array x-kubernetes-list-type: atomic options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. items: - description: PodDNSConfigOption defines DNS resolver options of a pod. properties: name: - description: |- - Name is this DNS resolver option's name. - Required. type: string value: - description: Value is this DNS resolver option's value. type: string type: object type: array x-kubernetes-list-type: atomic searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. items: type: string type: array x-kubernetes-list-type: atomic type: object dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. type: boolean ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. properties: args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -14433,44 +13162,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -14485,52 +13199,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -14538,63 +13225,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: Lifecycle is not allowed for ephemeral containers. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14603,104 +13258,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14709,126 +13318,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14837,121 +13395,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. type: string ports: - description: Ports are not allowed for ephemeral containers. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -14962,67 +13461,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -15031,97 +13502,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: Resources resize policy for the container. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -15130,33 +13555,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -15172,9 +13577,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -15183,49 +13585,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - You cannot set this field on ephemeral containers. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. You cannot set this field on - ephemeral containers. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -15240,253 +13613,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -15495,143 +13726,63 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. type: string terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -15642,64 +13793,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -15710,11 +13818,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -15724,22 +13827,14 @@ spec: - name x-kubernetes-list-type: map hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. items: type: string type: array x-kubernetes-list-type: atomic ip: - description: IP address of the host file entry. type: string required: - ip @@ -15749,70 +13844,22 @@ spec: - ip x-kubernetes-list-type: map hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. type: boolean hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - When using HostNetwork you should specify ports so the scheduler is aware. - When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, - and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. - Default to false. type: boolean hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. type: boolean hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. type: boolean hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. type: string hostnameOverride: - description: |- - HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. - This field only specifies the pod's hostname and does not affect its DNS records. - When this field is set to a non-empty string: - - It takes precedence over the values set in `hostname` and `subdomain`. - - The Pod's hostname will be set to this value. - - `setHostnameAsFQDN` must be nil or set to false. - - `hostNetwork` must be set to false. - - This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. - Requires the HostnameOverride feature gate to be enabled. type: string imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -15821,144 +13868,60 @@ spec: - name x-kubernetes-list-type: map initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ items: - description: A single application container that you want to run within a pod. properties: args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -15967,44 +13930,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -16019,52 +13967,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -16072,67 +13993,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16141,104 +14026,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16247,130 +14086,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16379,129 +14163,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. type: string ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -16512,71 +14229,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16585,99 +14270,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -16686,34 +14323,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -16729,9 +14345,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -16740,69 +14353,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -16817,261 +14381,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -17080,134 +14494,61 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -17218,64 +14559,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -17286,11 +14584,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -17300,63 +14593,15 @@ spec: - name x-kubernetes-list-type: map nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename type: string nodeSelector: additionalProperties: type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.resources - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup properties: name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null type: string required: - name @@ -17368,50 +14613,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md type: object preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. type: string priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. format: int32 type: integer priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. type: string readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates items: - description: PodReadinessGate contains the reference to a pod condition properties: conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. type: string required: - conditionType @@ -17419,73 +14632,13 @@ spec: type: array x-kubernetes-list-type: atomic resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is a stable field but requires that the - DynamicResourceAllocation feature gate is enabled. - - This field is immutable. items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - - When the DRAWorkloadResourceClaims feature gate is enabled and this Pod - belongs to a PodGroup, a PodResourceClaim is matched to a - PodGroupResourceClaim if all of their fields are equal (Name, - ResourceClaimName, and ResourceClaimTemplateName). A matched claim references - a single ResourceClaim shared across all Pods in the PodGroup, reserved for - the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual - Pods. properties: name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. type: string resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. type: string resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - When the DRAWorkloadResourceClaims feature gate is enabled and the pod - belongs to a PodGroup that defines a PodGroupResourceClaim with the same - Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the - ResourceClaim generated for the PodGroup. All pods in the group that - define an equivalent PodResourceClaim matching the - PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same - generated ResourceClaim. ResourceClaims generated for a PodGroup are - owned by the PodGroup and their lifecycles are tied to the PodGroup - instead of any individual pod. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. type: string required: - name @@ -17495,40 +14648,13 @@ spec: - name x-kubernetes-list-type: map resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -17544,9 +14670,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -17555,47 +14678,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy type: string runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class type: string schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. type: string schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. properties: name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. type: string required: - name @@ -17605,217 +14699,70 @@ spec: - name x-kubernetes-list-type: map schedulingGroup: - description: |- - SchedulingGroup provides a reference to the immediate scheduling runtime - grouping object that this Pod belongs to. - This field is used by the scheduler to identify the group and apply the - correct group scheduling policies. The association with a group also - impacts other lifecycle aspects of a Pod that are relevant in a wider context - of scheduling like preemption, resource attachment, etc. If not specified, - the Pod is treated as a single unit in all of these aspects. - The group object referenced by this field may not exist at the time the - Pod is created. - This field is immutable, but a group object with the same name may be - recreated with different policies. Doing this during pod scheduling - may result in the placement not conforming to the expected policies. properties: podGroupName: - description: |- - PodGroupName specifies the name of the standalone PodGroup object - that represents the runtime instance of this group. - Must be a DNS subdomain. type: string type: object securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. properties: appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. type: string runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. type: string seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. type: string sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. items: - description: Sysctl defines a kernel parameter to be set properties: name: - description: Name of a property to set type: string value: - description: Value of a property to set type: string required: - name @@ -17824,156 +14771,60 @@ spec: type: array x-kubernetes-list-type: atomic windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. type: string serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ type: string setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. type: boolean shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. type: boolean subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. type: string terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. format: int64 type: integer tolerations: - description: If specified, the pod's tolerations. items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . properties: effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array x-kubernetes-list-type: atomic topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -17987,125 +14838,27 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string type: array x-kubernetes-list-type: atomic maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. format: int32 type: integer nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. type: string nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. type: string topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. type: string whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. type: string required: - maxSkew @@ -18118,244 +14871,111 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore properties: fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). format: int32 type: integer readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: boolean volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string required: - volumeID type: object azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. properties: cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' type: string diskName: - description: diskName is the Name of the data disk in the blob storage type: string diskURI: - description: diskURI is the URI of data disk in the blob storage type: string fsType: default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. properties: readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key type: string shareName: - description: shareName is the azure share Name type: string required: - secretName - shareName type: object cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. properties: monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: boolean secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string required: - monitors type: object cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: boolean secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string required: - volumeID type: object configMap: - description: configMap represents a configMap that should populate this volume properties: defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -18365,129 +14985,67 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. properties: driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. type: string fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. type: string nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: Items is a list of downward API volume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource @@ -18500,87 +15058,21 @@ spec: x-kubernetes-list-type: atomic type: object emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir properties: medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string sizeLimit: anyOf: - type: integer - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. properties: volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. properties: metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. properties: annotations: additionalProperties: @@ -18600,42 +15092,19 @@ spec: type: string type: object spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. properties: accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 items: type: string type: array x-kubernetes-list-type: atomic dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. type: string kind: - description: Kind is the type of resource being referenced type: string name: - description: Name is the name of resource being referenced type: string required: - kind @@ -18643,60 +15112,20 @@ spec: type: object x-kubernetes-map-type: atomic dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. type: string kind: - description: Kind is the type of resource being referenced type: string name: - description: Name is the name of resource being referenced type: string namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. type: string required: - kind - name type: object resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources properties: limits: additionalProperties: @@ -18705,9 +15134,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -18716,37 +15142,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object selector: - description: selector is a label query over volumes to consider for binding. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -18760,38 +15167,16 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ type: string volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. type: string volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object required: @@ -18799,79 +15184,41 @@ spec: type: object type: object fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string lun: - description: 'lun is Optional: FC target lun number' format: int32 type: integer readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' items: type: string type: array x-kubernetes-list-type: atomic wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. items: type: string type: array x-kubernetes-list-type: atomic type: object flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. properties: driver: - description: driver is the name of the driver to use for this volume. type: string fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string - description: 'options is Optional: this field holds extra command options if any.' type: object readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -18879,230 +15226,98 @@ spec: - driver type: object flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. properties: datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated type: string datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk properties: fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk format: int32 type: integer pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: boolean required: - pdName type: object gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. properties: directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. type: string repository: - description: repository is the URL type: string revision: - description: revision is the commit hash for the specified revision. type: string required: - repository type: object glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. properties: endpoints: - description: endpoints is the endpoint name that details Glusterfs topology. type: string path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: string readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: boolean required: - endpoints - path type: object hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath properties: path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string required: - path type: object image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. properties: pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. type: string reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string type: object iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication type: boolean fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi type: string initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. type: string iqn: - description: iqn is the target iSCSI Qualified Name. type: string iscsiInterface: default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). type: string lun: - description: lun represents iSCSI Target Lun number. format: int32 type: integer portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). items: type: string type: array x-kubernetes-list-type: atomic readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. type: boolean secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). type: string required: - iqn @@ -19110,161 +15325,68 @@ spec: - targetPortal type: object name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs properties: path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: boolean server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string required: - path - server type: object persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. type: boolean required: - claimName type: object photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: - description: pdID is the ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver. properties: fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean volumeID: - description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: - description: projected items for all in one resources secrets, configmaps, and downward API properties: defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. properties: clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. properties: labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -19278,72 +15400,31 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. type: string optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. type: boolean path: - description: Relative path from the volume root to write the bundle. type: string signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. type: string required: - path type: object configMap: - description: configMap information about the configMap data to project properties: items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19353,69 +15434,42 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic downwardAPI: - description: downwardAPI information about the downwardAPI data to project properties: items: - description: Items is a list of DownwardAPIVolume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource @@ -19428,161 +15482,39 @@ spec: x-kubernetes-list-type: atomic type: object podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. properties: certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. type: string credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. type: string keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. type: string keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". type: string maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. format: int32 type: integer signerName: - description: Kubelet's generated CSRs will be addressed to this signer. type: string userAnnotations: additionalProperties: type: string - description: |- - userAnnotations allow pod authors to pass additional information to - the signer implementation. Kubernetes does not restrict or validate this - metadata in any way. - - These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of - the PodCertificateRequest objects that Kubelet creates. - - Entries are subject to the same validation as object metadata annotations, - with the addition that all keys must be domain-prefixed. No restrictions - are placed on values, except an overall size limitation on the entire field. - - Signers should document the keys and values they support. Signers should - deny requests that contain keys they do not recognize. type: object required: - keyType - signerName type: object secret: - description: secret information about the secret data to project properties: items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19592,42 +15524,19 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional field specify whether the Secret or its key must be defined type: boolean type: object x-kubernetes-map-type: atomic serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project properties: audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. type: string expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. format: int64 type: integer path: - description: |- - path is the path relative to the mount point of the file to project the - token into. type: string required: - path @@ -19637,176 +15546,84 @@ spec: x-kubernetes-list-type: atomic type: object quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. properties: group: - description: |- - group to map volume access to - Default is no group type: string readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. type: boolean registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes type: string tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: - description: |- - user to map volume access to - Defaults to serivceaccount user type: string volume: - description: volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd type: string image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string keyring: default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic pool: default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: boolean secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string required: - image - monitors type: object scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. properties: fsType: default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". type: string gateway: - description: gateway is the host address of the ScaleIO API Gateway. type: string protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false type: boolean storageMode: default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. type: string storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. type: string system: - description: system is the name of the storage system as configured in ScaleIO. type: string volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. type: string required: - gateway @@ -19814,52 +15631,19 @@ spec: - system type: object secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret properties: defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19868,81 +15652,37 @@ spec: type: array x-kubernetes-list-type: atomic optional: - description: optional field specify whether the Secret or its keys must be defined type: boolean secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret type: string type: object storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. type: string volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. properties: fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. type: string volumePath: - description: volumePath is the path that identifies vSphere volume vmdk type: string required: - volumePath @@ -20104,6 +15844,16 @@ spec: rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set rule: '!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)' + - message: componentType lpu is not supported; use lpx + rule: '!has(self.componentType) || self.componentType != ''lpu''' + - message: lpx may only be set when componentType is lpx + rule: '!has(self.lpx) || (has(self.componentType) && self.componentType == ''lpx'')' + - message: lpx is required when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'') || has(self.lpx)' + - message: replicas must be between 1 and 9 when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: scalingAdapter is not supported when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)' description: Services are the services to deploy as part of this deployment. maxProperties: 25 type: object @@ -20220,31 +15970,103 @@ spec: - type type: object type: array + lpx: + description: LPX contains the status of the graph's LPX workload, when present. + properties: + modelDownload: + description: modelDownload contains the model download status for remote builds. + properties: + builds: + description: |- + builds is the sorted set of resolved remote LPU build URLs whose artifacts + were successfully downloaded into model-storage. + items: + type: string + type: array + lastCheckedAt: + description: lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. + format: date-time + type: string + type: object + placement: + description: placement contains LPX scheduling progress and placement signals. + properties: + lpxAttempt: + description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + properties: + deadlineAt: + format: date-time + type: string + disarmedAt: + format: date-time + type: string + exceededAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + podCliqueSetUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + requests: + items: + description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + properties: + attemptDigest: + type: string + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - attemptDigest + - name + type: object + maxItems: 9 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - observedGeneration + - requests + type: object + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object + type: object observedGeneration: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placement: - description: |- - Placement groups DGD-level scheduler placement signals (score, reporting - state, and any future placement fields). - properties: - score: - description: |- - Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] - where higher is better and 1.0 is the best possible placement. - maximum: 1 - minimum: 0 - type: number - state: - description: State indicates placement score reporting state. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string - type: object restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -20349,6 +16171,11 @@ spec: format: int64 minimum: 0 type: integer + ready: + description: |- + Ready is the binary determination of whether the correct number of replicas + are scheduled and available. + type: boolean readyReplicas: description: |- ReadyReplicas is the number of ready replicas. @@ -20488,9 +16315,10 @@ spec: volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the - container named `"main"` and merges user overrides using strategic-merge-by-name - semantics. Users can add sidecars, init containers, and pod-level configuration - directly in `podTemplate` without any `extraPodSpec`-style escape hatch. + container named `"main"` and merges user overrides according to + `extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and + pod-level configuration directly in `podTemplate` without any + `extraPodSpec`-style escape hatch. properties: compilationCache: description: |- @@ -20731,12 +16559,13 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). + backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm + - lpu type: string dtype: description: |- @@ -20955,6 +16784,16 @@ spec: type: boolean type: object type: object + extraPodSpecMergeStrategy: + description: |- + extraPodSpecMergeStrategy controls how podTemplate is merged with the + operator-generated pod and container defaults. When omitted, the operator + uses its configured defaultExtraPodSpecMergeStrategy, which defaults to + "override". + enum: + - override + - strategic + type: string frontendSidecar: description: |- frontendSidecar optionally designates a container in @@ -20974,6 +16813,23 @@ spec: namespace rather than the per-deployment namespace derived from the DGD name. type: boolean + lpx: + description: |- + lpx holds LPX integration configuration. Only meaningful when + `type` is `lpx`. + properties: + buildId: + description: buildId references the immutable model build. + minLength: 1 + type: string + settings: + description: settings override the existing model configuration. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - buildId + type: object minAvailable: description: |- minAvailable maps to Grove PodCliqueScalingGroup minAvailable for @@ -29224,6 +25080,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components require a worker role and may declare a leader role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -29237,8135 +25094,19 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: + podTemplate: *lpx-pod-template + providerOverride: description: |- - podTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. No component type supports it in this release. + providerOverride configures the provider workload unit generated for this + role. It is supported only for components embedded in a DGD. properties: - metadata: + apiVersion: description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options of a pod. - properties: - name: - description: |- - Name is this DNS resolver option's name. - Required. - type: string - value: - description: Value is this DNS resolver option's value. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral containers. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - You cannot set this field on ephemeral containers. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. You cannot set this field on - ephemeral containers. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - When using HostNetwork you should specify ports so the scheduler is aware. - When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, - and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - hostnameOverride: - description: |- - HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. - This field only specifies the pod's hostname and does not affect its DNS records. - When this field is set to a non-empty string: - - It takes precedence over the values set in `hostname` and `subdomain`. - - The Pod's hostname will be set to this value. - - `setHostnameAsFQDN` must be nil or set to false. - - `hostNetwork` must be set to false. - - This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. - Requires the HostnameOverride feature gate to be enabled. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.resources - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is a stable field but requires that the - DynamicResourceAllocation feature gate is enabled. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - - When the DRAWorkloadResourceClaims feature gate is enabled and this Pod - belongs to a PodGroup, a PodResourceClaim is matched to a - PodGroupResourceClaim if all of their fields are equal (Name, - ResourceClaimName, and ResourceClaimTemplateName). A matched claim references - a single ResourceClaim shared across all Pods in the PodGroup, reserved for - the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual - Pods. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - When the DRAWorkloadResourceClaims feature gate is enabled and the pod - belongs to a PodGroup that defines a PodGroupResourceClaim with the same - Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the - ResourceClaim generated for the PodGroup. All pods in the group that - define an equivalent PodResourceClaim matching the - PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same - generated ResourceClaim. ResourceClaims generated for a PodGroup are - owned by the PodGroup and their lifecycles are tied to the PodGroup - instead of any individual pod. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - schedulingGroup: - description: |- - SchedulingGroup provides a reference to the immediate scheduling runtime - grouping object that this Pod belongs to. - This field is used by the scheduler to identify the group and apply the - correct group scheduling policies. The association with a group also - impacts other lifecycle aspects of a Pod that are relevant in a wider context - of scheduling like preemption, resource attachment, etc. If not specified, - the Pod is treated as a single unit in all of these aspects. - The group object referenced by this field may not exist at the time the - Pod is created. - This field is immutable, but a group object with the same name may be - recreated with different policies. Doing this during pod scheduling - may result in the placement not conforming to the expected policies. - properties: - podGroupName: - description: |- - PodGroupName specifies the name of the standalone PodGroup object - that represents the runtime instance of this group. - Must be a DNS subdomain. - type: string - type: object - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. - properties: - driver: - description: driver is the name of the driver to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - properties: - endpoints: - description: endpoints is the endpoint name that details Glusterfs topology. - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver. - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. - properties: - certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. - type: string - keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". - type: string - maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. - format: int32 - type: integer - signerName: - description: Kubelet's generated CSRs will be addressed to this signer. - type: string - userAnnotations: - additionalProperties: - type: string - description: |- - userAnnotations allow pod authors to pass additional information to - the signer implementation. Kubernetes does not restrict or validate this - metadata in any way. - - These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of - the PodCertificateRequest objects that Kubelet creates. - - Entries are subject to the same validation as object metadata annotations, - with the addition that all keys must be domain-prefixed. No restrictions - are placed on values, except an overall size limitation on the entire field. - - Signers should document the keys and values they support. Signers should - deny requests that contain keys they do not recognize. - type: object - required: - - keyType - - signerName - type: object - secret: - description: secret information about the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify whether the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - providerOverride: - description: |- - providerOverride configures the provider workload unit generated for this - role. It is supported only for components embedded in a DGD. - properties: - apiVersion: - description: |- - apiVersion is the Kubernetes API group and version of the provider schema. - Grove requires `grove.io/v1alpha1`. - minLength: 1 - type: string - target: + apiVersion is the Kubernetes API group and version of the provider schema. + Grove requires `grove.io/v1alpha1`. + minLength: 1 + type: string + target: description: |- target identifies the provider resource kind or embedded provider schema. It may be omitted on input when the DGD location has one unambiguous target; @@ -37458,6 +25199,7 @@ spec: - decode - planner - epp + - lpx type: string required: - name @@ -37471,6 +25213,16 @@ spec: rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: type is immutable after it is set rule: '!has(oldSelf.type) || (has(self.type) && self.type == oldSelf.type)' + - message: lpx may only be set when type is lpx + rule: '!has(self.lpx) || (has(self.type) && self.type == ''lpx'')' + - message: lpx is required when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'') || has(self.lpx)' + - message: LPX Pod templates belong to roles + rule: '!(has(self.type) && self.type == ''lpx'') || !has(self.podTemplate)' + - message: replicas must be between 1 and 9 when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: scalingAdapter is not supported when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.scalingAdapter))' maxItems: 25 type: array x-kubernetes-list-map-keys: @@ -37776,6 +25528,18 @@ spec: required: - id type: object + scheduling: + description: scheduling configures attempts for LPX components. Omission means no deadline. + properties: + attemptDeadlineSeconds: + description: |- + attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + Omission means unlimited; the value is not a solver budget. + format: int64 + maximum: 9223372036 + minimum: 1 + type: integer + type: object topologyConstraint: description: |- topologyConstraint is the deployment-level topology constraint. When @@ -37880,6 +25644,11 @@ spec: format: int64 minimum: 0 type: integer + ready: + description: |- + ready is the binary determination of whether the correct number of replicas + are scheduled and available. + type: boolean readyReplicas: description: |- readyReplicas is the number of ready replicas. Populated for @@ -37989,37 +25758,103 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + lpx: + description: lpx contains the status of the graph's LPX workload, when present. + properties: + modelDownload: + description: modelDownload contains the model download status for remote builds. + properties: + builds: + description: |- + builds is the sorted set of resolved remote LPU build URLs whose artifacts + were successfully downloaded into model-storage. + items: + type: string + type: array + lastCheckedAt: + description: lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. + format: date-time + type: string + type: object + placement: + description: placement contains LPX scheduling progress and placement signals. + properties: + lpxAttempt: + description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + properties: + deadlineAt: + format: date-time + type: string + disarmedAt: + format: date-time + type: string + exceededAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + podCliqueSetUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + requests: + items: + description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + properties: + attemptDigest: + type: string + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - attemptDigest + - name + type: object + maxItems: 9 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - observedGeneration + - requests + type: object + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object + type: object observedGeneration: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placement: - description: |- - placement groups DGD-level scheduler placement signals (score, reporting - state, and any future placement fields). - properties: - score: - description: |- - score is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation - uses the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - state: - description: |- - state indicates placement score reporting state. See PlacementScoreState - for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string - type: object restart: description: restart contains the status of a graph-level restart. properties: diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml new file mode 100644 index 000000000000..3e8681d4e41b --- /dev/null +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + helm.sh/resource-policy: keep + name: lpxgraphdeployments.nvidia.com +spec: + group: nvidia.com + names: + kind: LPXGraphDeployment + listKind: LPXGraphDeploymentList + plural: lpxgraphdeployments + shortNames: + - lpxgd + singular: lpxgraphdeployment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + LPXGraphDeployment is generated and owned by the Dynamo operator, not authored + by users. It owns one LPX PCS and its build/placement resources. + It serves only v1alpha1 and has no conversion contract. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + LPXGraphDeploymentSpec is the operator's handoff for all LPX components in a DGD. + The owning DGD remains the only authored workload spec. The child reads that + DGD only after verifying the controller owner and effective LPX input revision. + properties: + inputRevision: + description: |- + inputRevision hashes only effective LPX inputs, including its selected + restart token. Unrelated component edits do not create an LPX revision. + pattern: ^sha256:[a-f0-9]{64}$ + type: string + required: + - inputRevision + type: object + status: + description: |- + LPXGraphDeploymentStatus is the LPX controller's durable lifecycle state. + Dynamo consumes readiness only for the current generation and input revision. + properties: + components: + additionalProperties: + description: ComponentReplicaStatus contains replica information for a single component. + properties: + availableReplicas: + description: |- + availableReplicas is the number of available replicas. Populated for + `Deployment` and `PodCliqueScalingGroup`; not available for + `PodClique` or `LeaderWorkerSet`. + format: int32 + minimum: 0 + type: integer + componentKind: + description: |- + componentKind is the underlying resource kind (e.g. `PodClique`, + `Deployment`, `LeaderWorkerSet`). + enum: + - PodClique + - PodCliqueScalingGroup + - Deployment + - LeaderWorkerSet + type: string + componentNames: + description: |- + componentNames is the list of underlying Kubernetes resource names for + this Dynamo component. During normal operation this contains a single + name; during rolling updates it contains both old and new resource names. + items: + type: string + type: array + gpusPerEngine: + description: |- + gpusPerEngine is the number of GPUs assigned to one inference engine in a + component replica, across all of its nodes. Independent auxiliary GPU + allocations are excluded. A present zero means the engine itself has no + GPUs; consult gpusPerReplica for auxiliary allocations. + format: int64 + minimum: 0 + type: integer + gpusPerReplica: + description: |- + gpusPerReplica is the unique GPU allocation added when this component + scales by one replica, across all nodes, application and initialization + phases, and provider-owned Pods. Scalar GPUs use the Kubernetes effective + Pod scheduling footprint; shared DRA claims are counted once. A present + zero records a successful non-GPU resolution; omission means no current + shape is available. + format: int64 + minimum: 0 + type: integer + ready: + description: |- + ready is the binary determination of whether the correct number of replicas + are scheduled and available. + type: boolean + readyReplicas: + description: |- + readyReplicas is the number of ready replicas. Populated for + `PodClique`, `Deployment`, and `LeaderWorkerSet`; not available for + `PodCliqueScalingGroup`. + format: int32 + minimum: 0 + type: integer + replicas: + description: replicas is the total number of non-terminated replicas. + format: int32 + minimum: 0 + type: integer + runtimeNamespace: + description: |- + runtimeNamespace is the effective Dynamo runtime namespace for this + component. Worker components may include a generation suffix; non-workers + use the base namespace. During rolling updates, worker status keeps the old + active revision namespace until cutover completes. + type: string + scheduledReplicas: + description: |- + scheduledReplicas is the number of replicas the backend scheduler has + scheduled, expressed strictly in Dynamo component-replica units (not + raw backend pod counts). It is a diagnostic aid for distinguishing + capacity/scheduling shortfalls from runtime readiness. + + It is optional and omitted (nil) when the active backend cannot derive + it reliably in component-replica units — for example before the backing + resource's status has been observed, or for backends that do not report + a scheduling count. A nil value therefore means "not reported", never + "zero scheduled"; consumers must not treat absence as a scheduling + failure. + format: int32 + minimum: 0 + type: integer + updatedReplicas: + description: updatedReplicas is the number of replicas at the current/desired revision. + format: int32 + minimum: 0 + type: integer + required: + - componentKind + - replicas + - updatedReplicas + type: object + description: components reports logical replicas by authored component name, never Agent Pods. + type: object + conditions: + description: conditions report compilation, placement, and runtime readiness. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + modelDownload: + description: modelDownload retains the existing remote-build download progress. + properties: + builds: + description: |- + builds is the sorted set of resolved remote LPU build URLs whose artifacts + were successfully downloaded into model-storage. + items: + type: string + type: array + lastCheckedAt: + description: lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. + format: date-time + type: string + type: object + observedGeneration: + description: |- + observedGeneration is the child generation processed successfully. + Every spec.inputRevision change advances the child generation. + format: int64 + type: integer + placement: + description: |- + placement retains the scheduling attempt fence, progress, and deadline. + Its observed generation belongs to this child, not the source DGD. + properties: + lpxAttempt: + description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + properties: + deadlineAt: + format: date-time + type: string + disarmedAt: + format: date-time + type: string + exceededAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + podCliqueSetUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + requests: + items: + description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + properties: + attemptDigest: + type: string + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - attemptDigest + - name + type: object + maxItems: 9 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - observedGeneration + - requests + type: object + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/operator/docs/crd-ref-docs-config.yaml b/deploy/operator/docs/crd-ref-docs-config.yaml index 67017cc2485f..517302928762 100644 --- a/deploy/operator/docs/crd-ref-docs-config.yaml +++ b/deploy/operator/docs/crd-ref-docs-config.yaml @@ -35,6 +35,8 @@ processor: # Keep SharedSpec so embedded fields are documented once # - "DynamoComponentDeploymentOverridesSpec$" - "ConversionContext$" + # LPXGraphDeployment is an operator-generated private child, not a public API. + - "LPXGraphDeployment.*$" - "DynamoComponentDeploymentStatus$" - "BaseStatus$" - "ExcludedNamespacesChecker" diff --git a/deploy/operator/docs/fix-api-anchors.py b/deploy/operator/docs/fix-api-anchors.py index 3a4fd8c55033..87b2bb4e6e5c 100644 --- a/deploy/operator/docs/fix-api-anchors.py +++ b/deploy/operator/docs/fix-api-anchors.py @@ -38,6 +38,7 @@ "DynamoComponentDeploymentSharedSpec", "ComponentRoleSpec", "ProviderOverride", + "LPXConfig", } DCD_REFERENCE_RE = re.compile(r"^- \[DynamoComponentDeploymentSpec\]\(#[^)]+\)\s*$") @@ -52,7 +53,9 @@ def project_standalone_dcd_schema(markdown: str) -> str: if heading: current_type = heading.group("name") - if current_type == DCD_SPEC and line.startswith("| `providerOverride` "): + if current_type == DCD_SPEC and line.startswith( + ("| `providerOverride` ", "| `lpx` ") + ): continue if current_type == DCD_SPEC and line.startswith("| `roles` "): diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index bcafaae4de01..78a5fe44244c 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -3,9 +3,12 @@ module github.com/ai-dynamo/dynamo/deploy/operator go 1.26.6 require ( + capnproto.org/go/capnp/v3 v3.1.0-alpha.2 emperror.dev/errors v0.8.1 github.com/Masterminds/semver/v3 v3.4.0 github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 + github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13 + github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f github.com/ai-dynamo/snapshot/api v0.1.0 github.com/bsm/gomega v1.27.10 github.com/evanphx/json-patch/v5 v5.9.11 @@ -16,6 +19,7 @@ require ( github.com/onsi/ginkgo/v2 v2.30.0 github.com/onsi/gomega v1.41.0 github.com/open-policy-agent/cert-controller v0.16.0 + github.com/pelletier/go-toml/v2 v2.3.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.71.2 github.com/prometheus/client_golang v1.23.2 @@ -26,7 +30,9 @@ require ( go.yaml.in/yaml/v3 v3.0.4 golang.org/x/crypto v0.55.0 golang.org/x/sync v0.22.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af + gotest.tools/v3 v3.5.2 helm.sh/helm/v3 v3.20.2 istio.io/api v1.23.1 istio.io/client-go v1.23.1 @@ -35,7 +41,7 @@ require ( k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.3 k8s.io/klog/v2 v2.140.0 - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api-inference-extension v1.2.0 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 @@ -56,6 +62,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect @@ -119,7 +126,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.83.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index b04b1c4ed76e..dd11984f9685 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -1,3 +1,5 @@ +capnproto.org/go/capnp/v3 v3.1.0-alpha.2 h1:93ISpqHf2/3WQlfrBP0tT8Dg/RQc3uq6DV36vN0ePWk= +capnproto.org/go/capnp/v3 v3.1.0-alpha.2/go.mod h1:2vT5D2dtG8sJGEoEKU17e+j7shdaYp1Myl8X03B3hmc= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= @@ -18,6 +20,10 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 h1:eVDVuyh5vDgWG6I2Mffs68g+25qW94VAG3EfDTxbhc0= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13/go.mod h1:csL4kgDc1MoD7Erjyu8QyVrbL/dv8fIZ3sYkJcc2nrQ= +github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13 h1:1pS3fH87Zr4IXTC8IR7KQxrLoChPrLv4OWOiquW/YVg= +github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13/go.mod h1:+JD/5QmNY0FViNWzx6EWMrLDdYbl4gkO5472mqfFeck= +github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f h1:PBmn3itAlMFHc0HgDOcr0IwV0uiUDQehcg7/aumIobU= +github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f/go.mod h1:i7tIkusSEcMQqN4IDd3cXZ6EL0Kfke6F00Nn5WbuH3Y= github.com/ai-dynamo/snapshot/api v0.1.0 h1:S8HG6K+YhloUc/usu03wiJ8fTo3Mhju/oWQVFDYVw/o= github.com/ai-dynamo/snapshot/api v0.1.0/go.mod h1:LPkn788YGMGlWN1hm1M3Jm82WVY6tUWpRfmE+8seA04= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= @@ -34,6 +40,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 h1:d5EKgQfRQvO97jnISfR89AiCCCJMwMFoSxUiU0OGCRU= +github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381/go.mod h1:OU76gHeRo8xrzGJU3F3I1CqX1ekM8dfJw0+wPeMwnp0= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= @@ -159,6 +167,10 @@ github.com/open-policy-agent/cert-controller v0.16.0 h1:/w+wAM9dylWGwhPnvfCgepyB github.com/open-policy-agent/cert-controller v0.16.0/go.mod h1:w5qBWYbc8HwyHI9VYAZ6YjWOcZtQ39A30I9W4X7pVVk= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a/go.mod h1:tI7nc6H6os2UYZRvSm9Y7bq4oMoXqhwA0WfnqKpoAgc= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -209,6 +221,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= +github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM= @@ -297,6 +313,8 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= helm.sh/helm/v3 v3.20.2 h1:binM4rvPx5DcNsa1sIt7UZi55lRbu3pZUFmQkSoRh48= helm.sh/helm/v3 v3.20.2/go.mod h1:Fl1kBaWCpkUrM6IYXPjQ3bdZQfFrogKArqptvueZ6Ww= istio.io/api v1.23.1 h1:bm2XF0j058FfzWVHUfpmMj4sFDkcD1X609qs5AU97Pc= @@ -325,8 +343,8 @@ k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hk k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= diff --git a/deploy/operator/internal/consts/lpx.go b/deploy/operator/internal/consts/lpx.go new file mode 100644 index 000000000000..49a6f3698411 --- /dev/null +++ b/deploy/operator/internal/consts/lpx.go @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package consts + +const ( + KubeAnnotationLPXSchedulerBackend = "scheduling.lpu.nvidia.com/scheduler-backend" + KubeAnnotationLPXExecutionBackend = "scheduling.lpu.nvidia.com/execution-backend" + LPXSchedulerBackend = "lpx-scheduler" + ComponentTypeLPX = "lpx" + ModelStorageVolumeName = "model-storage" + + // AnnotationExtraResourcesHash stores the generated LPX configuration hash. + AnnotationExtraResourcesHash = "nvidia.com/extra-resources-hash" +) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 6aca795da6de..77f0d1102805 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -20,9 +20,11 @@ package controller import ( "context" "fmt" + "time" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + apiequality "k8s.io/apimachinery/pkg/api/equality" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -36,6 +38,7 @@ type groveProgram struct { scalingAdapters *dgdScalingAdaptersReconciler topology *dgdGroveTopologyConditionReconciler gate features.Gate + lpx *dgdLPXHandoff } // newGroveProgram wires the Grove pathway at the DGD composition root. @@ -66,6 +69,7 @@ func (r *DynamoGraphDeploymentReconciler) newGroveProgram() *groveProgram { scalingAdapters: newDGDScalingAdaptersReconciler(r.Client, r.Recorder), topology: newDGDGroveTopologyConditionReconciler(r.Client), gate: r.RuntimeConfig.Gate, + lpx: &dgdLPXHandoff{Client: r.Client}, } } @@ -121,10 +125,15 @@ func (p *groveProgram) Reconcile( ctx, req.DGD, &programResult.Status, - p.restartProgress.Resolve, + p.resolveRestartProgress, ) recordRestartTransition(previousRestart, restart.Status, &programResult) programResult.Status.Restart = restart.Status + if req.DGD.HasLPXComponent() && !apiequality.Semantic.DeepEqual(req.DGD.Status.Restart, restart.Status) { + // Persist the selected restart before delivering its token to the child. + programResult.RequeueAfter = time.Nanosecond + return programResult, nil + } result, err := p.workloads.Reconcile( ctx, @@ -132,6 +141,7 @@ func (p *groveProgram) Reconcile( restart.State, checkpoints.Infos, ) + if err != nil { // Preserve newly observed component status while leaving the generation unobserved. if result.ComponentStatus != nil { @@ -139,7 +149,16 @@ func (p *groveProgram) Reconcile( } return programResult, fmt.Errorf("failed to reconcile Grove workloads: %w", err) } + child, err := p.lpx.Reconcile(ctx, req.DGD) + if err != nil { + return programResult, fmt.Errorf("reconcile LPX child: %w", err) + } + projectLPXChildStatus(req.DGD, child, &result, &programResult.Status) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) + if child != nil && !child.DeletionTimestamp.IsZero() { + programResult.RequeueAfter = 5 * time.Second + } + if result.State != nvidiacomv1beta1.DGDStatePending || result.Reason != reasonWaitingForCheckpoint { if err := p.scalingAdapters.Reconcile(ctx, req.DGD); err != nil { log.FromContext(ctx).Error(err, "Failed to reconcile scaling adapters") diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index 26a0966b8d4b..8b91cc9e3124 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -21,12 +21,11 @@ import ( "context" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -79,7 +78,7 @@ func podCliqueEventPredicates() predicate.Funcs { newPodClique, newOK := updateEvent.ObjectNew.(*grovev1alpha1.PodClique) return oldOK && newOK && - podCliqueStatusChangeIsSignificant(oldPodClique, newPodClique) + commoncontroller.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique) }, GenericFunc: func(event.GenericEvent) bool { return false }, } @@ -94,7 +93,7 @@ func pcsgEventPredicates() predicate.Funcs { newScalingGroup, newOK := updateEvent.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) return oldOK && newOK && - pcsgStatusChangeIsSignificant(oldScalingGroup, newScalingGroup) + commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup) }, GenericFunc: func(event.GenericEvent) bool { return false }, } @@ -105,7 +104,8 @@ func (s *groveWatchSetup) mapPodCliqueToRequests( obj client.Object, ) []ctrl.Request { podClique, ok := obj.(*grovev1alpha1.PodClique) - if !ok { + // LPX owns its role observations; the DGD watches the aggregate child status. + if !ok || podClique == nil || lpxcontroller.OwnsPodClique(ctx, s.reader, podClique) { return nil } @@ -181,62 +181,3 @@ func (s *groveWatchSetup) mapPodCliqueScalingGroupToRequests( }, }} } - -// groveScheduledConditionChanged reports whether the scheduling conditions -// consumed by Grove readiness changed. -func groveScheduledConditionChanged(oldConditions, newConditions []metav1.Condition) bool { - for _, conditionType := range []string{ - groveconstants.ConditionTypePodCliqueScheduled, - groveconstants.ConditionTypeMinAvailableBreached, - } { - oldCondition := meta.FindStatusCondition(oldConditions, conditionType) - newCondition := meta.FindStatusCondition(newConditions, conditionType) - if (oldCondition == nil) != (newCondition == nil) { - return true - } - if oldCondition != nil && - newCondition != nil && - (oldCondition.Status != newCondition.Status || - oldCondition.Reason != newCondition.Reason) { - return true - } - } - return false -} - -// podCliqueStatusChangeIsSignificant mirrors every PodClique field consumed by -// Grove readiness and capacity classification. -func podCliqueStatusChangeIsSignificant( - oldPodClique *grovev1alpha1.PodClique, - newPodClique *grovev1alpha1.PodClique, -) bool { - return oldPodClique.Status.ReadyReplicas != newPodClique.Status.ReadyReplicas || - oldPodClique.Status.UpdatedReplicas != newPodClique.Status.UpdatedReplicas || - oldPodClique.Status.Replicas != newPodClique.Status.Replicas || - oldPodClique.Status.ScheduledReplicas != newPodClique.Status.ScheduledReplicas || - oldPodClique.Status.ScheduleGatedReplicas != newPodClique.Status.ScheduleGatedReplicas || - oldPodClique.Spec.Replicas != newPodClique.Spec.Replicas || - !ptr.Equal(oldPodClique.Status.ObservedGeneration, newPodClique.Status.ObservedGeneration) || - !ptr.Equal(oldPodClique.Status.CurrentPodCliqueSetGenerationHash, newPodClique.Status.CurrentPodCliqueSetGenerationHash) || - (oldPodClique.Status.UpdateProgress != nil && oldPodClique.Status.UpdateProgress.UpdateEndedAt != nil) != - (newPodClique.Status.UpdateProgress != nil && newPodClique.Status.UpdateProgress.UpdateEndedAt != nil) || - groveScheduledConditionChanged(oldPodClique.Status.Conditions, newPodClique.Status.Conditions) -} - -// pcsgStatusChangeIsSignificant mirrors every PodCliqueScalingGroup field -// consumed by Grove readiness and capacity classification. -func pcsgStatusChangeIsSignificant( - oldScalingGroup *grovev1alpha1.PodCliqueScalingGroup, - newScalingGroup *grovev1alpha1.PodCliqueScalingGroup, -) bool { - return oldScalingGroup.Status.AvailableReplicas != newScalingGroup.Status.AvailableReplicas || - oldScalingGroup.Status.UpdatedReplicas != newScalingGroup.Status.UpdatedReplicas || - oldScalingGroup.Status.Replicas != newScalingGroup.Status.Replicas || - oldScalingGroup.Status.ScheduledReplicas != newScalingGroup.Status.ScheduledReplicas || - oldScalingGroup.Spec.Replicas != newScalingGroup.Spec.Replicas || - !ptr.Equal(oldScalingGroup.Status.ObservedGeneration, newScalingGroup.Status.ObservedGeneration) || - !ptr.Equal(oldScalingGroup.Status.CurrentPodCliqueSetGenerationHash, newScalingGroup.Status.CurrentPodCliqueSetGenerationHash) || - (oldScalingGroup.Status.UpdateProgress != nil && oldScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) != - (newScalingGroup.Status.UpdateProgress != nil && newScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) || - groveScheduledConditionChanged(oldScalingGroup.Status.Conditions, newScalingGroup.Status.Conditions) -} diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go new file mode 100644 index 000000000000..c7948d876459 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +func TestGroveWatchSkipsOnlyVerifiedLPXOwners(t *testing.T) { + t.Log("Distinguish LPX-owned Grove roles from ordinary DGD readiness events") + const wrongAPIVersion = "unrelated/v1" + scheme := runtime.NewScheme() + require.NoError(t, grovev1alpha1.AddToScheme(scheme)) + child := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "graph", UID: "child-uid"}} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: "engine-pcs", Namespace: "workloads", UID: "pcs-uid", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(child, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + }} + group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ + Name: "engine-group", Namespace: "workloads", UID: "group-uid", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }} + clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ + Name: "role", Namespace: "workloads", + Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: child.Name}, + Annotations: map[string]string{ + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleConductor, + dynamolpx.WorkloadDigestAnnotation: "sha256:workload", + dynamo.LPXDeploymentUIDAnnotation: string(child.UID), + }, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }} + ready := clique.DeepCopy() + ready.Status.ReadyReplicas++ + require.True(t, podCliqueEventPredicates().Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: ready})) + + t.Log("The LPX controller observes role readiness and the DGD observes aggregate child status") + parent := newGroveWatchSetup(fake.NewClientBuilder().WithScheme(scheme).WithObjects(pcs, group).Build()) + for _, role := range []string{lpxv1alpha1.PodRoleAgent, lpxv1alpha1.PodRoleConductor, lpxv1alpha1.PodRoleCyborgWorker} { + t.Run(role, func(t *testing.T) { + observed := ready.DeepCopy() + observed.Annotations[lpxv1alpha1.PodRoleAnnotation] = role + require.Empty(t, parent.mapPodCliqueToRequests(t.Context(), observed)) + }) + } + + t.Log("Authorable LPX hints cannot suppress ordinary readiness, nor can incomplete cached ownership") + for _, test := range []struct { + name string + mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) + }{ + {"ordinary unmarked", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + c.Annotations = nil + }}, + {"ordinary direct owner", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + c.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(p, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + }}, + {"ordinary grouped owner", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + p.OwnerReferences[0].Kind = "DynamoGraphDeployment" + }}, + {"missing group", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + g.Name = "not-the-owner" + }}, + {"replaced group", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + g.UID = "new-group-uid" + }}, + {"wrong group API", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + c.OwnerReferences[0].APIVersion = wrongAPIVersion + }}, + {"missing PCS", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + p.Name = "not-the-owner" + }}, + {"replaced PCS", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + p.UID = "new-pcs-uid" + }}, + {"wrong PCS API", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + g.OwnerReferences[0].APIVersion = wrongAPIVersion + }}, + {"wrong child API", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + p.OwnerReferences[0].APIVersion = wrongAPIVersion + }}, + {"different child", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { + p.OwnerReferences[0].Name = "other-graph" + }}, + {"stale child stamp", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + c.Annotations[dynamo.LPXDeploymentUIDAnnotation] = "old-child-uid" + }}, + } { + t.Run(test.name, func(t *testing.T) { + currentPCS, currentGroup := pcs.DeepCopy(), group.DeepCopy() + observed := ready.DeepCopy() + test.mutate(observed, currentGroup, currentPCS) + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(currentPCS, currentGroup).Build() + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: observed.Namespace, Name: child.Name}}}, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), observed)) + }) + } +} diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index 221c64ff884b..2ebe0c95fa08 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -20,6 +20,7 @@ package controller import ( "context" "fmt" + "slices" "strings" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -99,12 +100,24 @@ func (r *groveWorkloadRenderer) Render( if err != nil { return nil, err } - desired, err := r.renderPodCliqueSet( - ctx, renderDeployment, existingPodCliqueSet, restartState, checkpointInfos, + // Render ordinary workloads and retain the observed server-owned fields. + existingRestartAnnotations := restartAnnotationsFromPodCliqueSet(existingPodCliqueSet) + desired, err := dynamo.GenerateGrovePodCliqueSet( + ctx, renderDeployment, r.config, r.runtimeConfig, r.reader, + r.dockerSecretRetriever, restartState, existingRestartAnnotations, checkpointInfos, ) if err != nil { return nil, err } + prepareGroveTopologyConstraintUpgrade(desired, existingPodCliqueSet) + preserveGrovePodCliqueSetOrder(desired, existingPodCliqueSet) + preserveGrovePodCliqueSetReplicas( + desired, + existingPodCliqueSet, + checkpointInfos, + ) + + // Resolve capacity from the same rendered ordinary workload. gpuShapes, err := dynamo.ResolveGroveGPUShapes(ctx, r.reader, renderDeployment, desired) if err != nil { return nil, err @@ -117,41 +130,22 @@ func (r *groveWorkloadRenderer) Render( }, nil } -func (r *groveWorkloadRenderer) renderPodCliqueSet( - ctx context.Context, - renderDeployment *nvidiacomv1beta1.DynamoGraphDeployment, - existing *grovev1alpha1.PodCliqueSet, - restartState *dynamo.RestartState, - checkpointInfos map[string]*checkpoint.CheckpointInfo, -) (*grovev1alpha1.PodCliqueSet, error) { - existingRestartAnnotations := restartAnnotationsFromPodCliqueSet(existing) - desired, err := dynamo.GenerateGrovePodCliqueSet( - ctx, - renderDeployment, - r.config, - r.runtimeConfig, - r.reader, - r.dockerSecretRetriever, - restartState, - existingRestartAnnotations, - checkpointInfos, - ) - if err != nil { - return nil, err - } - - prepareGroveTopologyConstraintUpgrade(desired, existing) - preserveGrovePodCliqueSetOrder(desired, existing) - preserveGrovePodCliqueSetReplicas(desired, existing, checkpointInfos) - return desired, nil -} - func groveRenderDeployment( dgd *nvidiacomv1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, workerHashSuffix bool, ) (*nvidiacomv1beta1.DynamoGraphDeployment, error) { - renderDeployment := dgd.DeepCopy() + // Exclude externally managed templates before copying the ordinary render inputs. + ordinary := *dgd + externallyManaged := func(component nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.ManagedByExternalController() + } + if slices.ContainsFunc(dgd.Spec.Components, externallyManaged) { + ordinary.Spec.Components = slices.DeleteFunc(slices.Clone(dgd.Spec.Components), externallyManaged) + } + renderDeployment := ordinary.DeepCopy() + + // Compatibility and worker labels mutate only the independently owned copy. applyGroveCompatibility(renderDeployment, pcs) if !workerHashSuffix { return renderDeployment, nil @@ -407,26 +401,27 @@ func prepareLegacyGroveTopologyConstraintRepair( desired.Pack = &pack } -// Grove horizontal replicas are driven through scale subresources after -// creation; keep existing template values so DGD replica changes do not update -// the PodCliqueSet spec. +// Keep Grove-owned scaling state stable across parent PCS reconciliation. +// Horizontal replicas are driven through scale subresources. +// +//nolint:gocyclo // Grove scale fields share one ordered snapshot traversal. func preserveGrovePodCliqueSetReplicas( desired *grovev1alpha1.PodCliqueSet, existing *grovev1alpha1.PodCliqueSet, - checkpointInfoByComponent ...map[string]*checkpoint.CheckpointInfo, + checkpointInfoByComponent map[string]*checkpoint.CheckpointInfo, ) { if desired == nil || existing == nil { return } + + // Let checkpoint-gated components retain their generated zero replicas. replicaPreserveSkips := map[string]struct{}{} - if len(checkpointInfoByComponent) > 0 { - for componentName, info := range checkpointInfoByComponent[0] { - if info != nil && - info.Enabled && - info.StartupPolicy == nvidiacomv1alpha1.CheckpointStartupPolicyWaitForCheckpoint && - !info.Ready { - replicaPreserveSkips[strings.ToLower(componentName)] = struct{}{} - } + for componentName, info := range checkpointInfoByComponent { + if info != nil && + info.Enabled && + info.StartupPolicy == nvidiacomv1alpha1.CheckpointStartupPolicyWaitForCheckpoint && + !info.Ready { + replicaPreserveSkips[strings.ToLower(componentName)] = struct{}{} } } @@ -461,17 +456,15 @@ func preserveGrovePodCliqueSetReplicas( } } - scalingGroupReplicasByName := make( - map[string]*int32, - len(existing.Spec.Template.PodCliqueScalingGroupConfigs), - ) + // Index existing scaling-group replicas by name. + scalingGroupReplicasByName := make(map[string]*int32, len(existing.Spec.Template.PodCliqueScalingGroupConfigs)) for _, config := range existing.Spec.Template.PodCliqueScalingGroupConfigs { - if config.Name == "" { - // Defensive only; generated PCSG configs always have names. - continue + if config.Name != "" { + scalingGroupReplicasByName[config.Name] = config.Replicas } - scalingGroupReplicasByName[config.Name] = config.Replicas } + + // Preserve scale-owned fields in generated configuration order. for i := range desired.Spec.Template.PodCliqueScalingGroupConfigs { config := &desired.Spec.Template.PodCliqueScalingGroupConfigs[i] if _, skip := replicaPreserveSkips[strings.ToLower(config.Name)]; skip { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 44f299b1dc65..9ce745f1da48 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -28,6 +28,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" @@ -92,9 +93,27 @@ func (r *groveWorkloadsReconciler) Reconcile( logger.Error(err, "failed to generate the Grove GangSet") return ReconcileResult{}, fmt.Errorf("failed to generate the Grove GangSet: %w", err) } + if len(renderedPodCliqueSet.desired.Spec.Template.Cliques) == 0 { + if existing := renderedPodCliqueSet.existing; existing != nil { + if !metav1.IsControlledBy(existing, dgd) { + return ReconcileResult{}, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) + } + if existing.DeletionTimestamp.IsZero() { + uid, version := existing.UID, existing.ResourceVersion + if err := r.syncer.Delete(ctx, existing, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); client.IgnoreNotFound(err) != nil { + return ReconcileResult{}, err + } + } + return ReconcileResult{State: nvidiacomv1beta1.DGDStatePending, Reason: "RemovingEmptyPodCliqueSet", Message: "Waiting for the removed ordinary workload"}, nil + } + stableResources, err := r.stableResources.Reconcile(ctx, dgd, renderedPodCliqueSet.renderDeployment) + if err != nil { + return ReconcileResult{}, err + } + return checkResourcesReadiness(stableResources), nil + } syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, dgd, renderedPodCliqueSet) if err != nil { - logger.Error(err, "failed to reconcile the Grove PodCliqueSet") return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) } // Defer commit if the PCS was written this reconcile; the informer hasn't caught up yet. @@ -120,7 +139,7 @@ func (r *groveWorkloadsReconciler) Reconcile( } } - if err := r.scaler.Reconcile(ctx, dgd, checkpointInfos); err != nil { + if err := r.scaler.Reconcile(ctx, renderedPodCliqueSet.renderDeployment, checkpointInfos); err != nil { logger.Error(err, "failed to reconcile Grove scaling") return ReconcileResult{}, fmt.Errorf("failed to reconcile Grove scaling: %w", err) } @@ -132,7 +151,7 @@ func (r *groveWorkloadsReconciler) Reconcile( podCliqueSetResource, readiness, err := r.observePodCliqueSetReadiness( ctx, - dgd, + renderedPodCliqueSet.renderDeployment, syncedPodCliqueSet, ) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 6501586249f5..c296907406f7 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -49,7 +49,7 @@ import ( const updatedWorkerVersion = "new" func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { - t.Log("Build a ready frontend and its DGD") + t.Log("Build a ready frontend alongside an independently owned LPX component") dgd := betaDGD(t, &nvidiacomv1alpha1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "graph", Namespace: "default"}, Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{ @@ -62,6 +62,9 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { }, }, }) + lpxSource := newLPXHandoffSource(t, "node-local-v2-hybrid") + dgd.Spec.Components = append(dgd.Spec.Components, lpxSource.Spec.Components[0]) + wantSpec := dgd.Spec.DeepCopy() podClique := &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{ Name: "graph-0-frontend", @@ -126,6 +129,10 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { assert.Equal(t, nvidiacomv1beta1.DGDStateSuccessful, result.State) assert.Equal(t, 1, scaleUpdates) assert.Equal(t, 1, podCliqueReads) + require.Len(t, result.ComponentStatus, 1) + assert.Equal(t, nvidiacomv1beta1.ComponentKindPodClique, result.ComponentStatus["frontend"].ComponentKind) + assert.Equal(t, []string{"graph-0-frontend"}, result.ComponentStatus["frontend"].ComponentNames) + assert.Equal(t, *wantSpec, dgd.Spec) } func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFails(t *testing.T) { @@ -249,10 +256,13 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( Envs: []corev1.EnvVar{{Name: "WORKER_VERSION", Value: "old"}}, }, }) + lpxSource := newLPXHandoffSource(t, "node-local-v2-hybrid") + dgd.Spec.Components = append(dgd.Spec.Components, lpxSource.Spec.Components[0]) currentHash, err := dynamo.ComputeDGDWorkersSpecHash(dgd) require.NoError(t, err) dgd.Annotations = map[string]string{consts.AnnotationCurrentWorkerHashV2: currentHash} dgd.GetComponentByName("prefill").PodTemplate.Spec.Containers[0].Env[0].Value = updatedWorkerVersion + wantSpec := dgd.Spec.DeepCopy() wantHash, err := dynamo.ComputeDGDWorkersSpecHash(dgd) require.NoError(t, err) legacyPCS := &grovev1alpha1.PodCliqueSet{ @@ -325,6 +335,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( storedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), storedDGD)) assert.Equal(t, currentHash, storedDGD.Annotations[consts.AnnotationCurrentWorkerHashV2]) + assert.Equal(t, *wantSpec, storedDGD.Spec) assert.Equal(t, 1, pcsUpdateCalls) assert.Zero(t, dgdUpdateCalls) @@ -346,6 +357,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Verify the retry commits the target hash without rewriting the PCS") require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), storedDGD)) assert.Equal(t, wantHash, storedDGD.Annotations[consts.AnnotationCurrentWorkerHashV2]) + assert.Equal(t, *wantSpec, storedDGD.Spec) assert.Equal(t, 1, pcsUpdateCalls) assert.Equal(t, 2, dgdUpdateCalls) diff --git a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go new file mode 100644 index 000000000000..4f9a45741b3a --- /dev/null +++ b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go @@ -0,0 +1,110 @@ +//go:build !clustertest + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { + t.Log("Install the real CRDs in an isolated API server; no runtime controllers or cluster workloads") + env := operatorenv.New(operatorenv.Options{SetupWebhooks: setupProductionWebhooks}).RunT(t) + require.True(t, env.Client().Scheme().Recognizes(v1alpha1.LPXGraphDeploymentGVK)) + require.False(t, env.Client().Scheme().Recognizes(v1beta1.GroupVersion.WithKind("LPXGraphDeployment"))) + source := newLPXHandoffSource(t, "node-local-v2-lpu-only") + source.Namespace, source.UID, source.Generation = env.Namespace(), "", 0 + require.NoError(t, env.Client().Create(t.Context(), source)) + + t.Log("Persist the pending component projection using the real DGD status schema") + result := &ReconcileResult{} + projectLPXChildStatus(source, nil, result, &source.Status) + source.Status.Components = result.ComponentStatus + source.Status.State = result.State + require.NoError(t, env.Client().Status().Update(t.Context(), source)) + require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) + require.False(t, source.Status.Components["lpx"].Ready) + + t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") + handoff := &dgdLPXHandoff{Client: env.Client()} + child, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, int64(1), child.Generation) + require.NotEmpty(t, child.UID) + require.NotEqual(t, source.UID, child.UID) + require.Equal(t, metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK), metav1.GetControllerOf(child)) + require.NoError(t, dynamo.ValidateLPXSource(child, source)) + + t.Log("Project the observed download payload and its actionable failure") + child.Status.ObservedGeneration = child.Generation + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} + child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, + LastTransitionTime: metav1.Now(), Reason: "PublicationDenied", Message: "Check the namespace quota"}} + require.NoError(t, env.Client().Status().Update(t.Context(), child)) + projected := v1beta1.DynamoGraphDeploymentStatus{} + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, child, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, Reason(child.Status.Conditions[0].Reason), result.Reason) + require.Equal(t, Message(child.Status.Conditions[0].Message), result.Message) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + + t.Log("Ordinary source metadata does not advance the child generation or frozen source identity") + source.Labels = map[string]string{"unrelated": "metadata"} + require.NoError(t, env.Client().Update(t.Context(), source)) + unchanged, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, child.ResourceVersion, unchanged.ResourceVersion) + require.Equal(t, child.Annotations, unchanged.Annotations) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, unchanged, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + + t.Log("An LPX template edit advances the real child generation and invalidates the old observation") + lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" + require.NoError(t, env.Client().Update(t.Context(), source)) + updated, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, child.Generation+1, updated.Generation) + require.NotEqual(t, child.Spec.InputRevision, updated.Spec.InputRevision) + require.NoError(t, dynamo.ValidateLPXSource(updated, source)) + require.Equal(t, child.Status.ObservedGeneration, updated.Status.ObservedGeneration) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, updated, result, &projected) + require.Equal(t, v1beta1.DGDStatePending, result.State) + require.Equal(t, Reason("LPXChildPending"), result.Reason) + require.Nil(t, projected.LPX) + + t.Log("Reject malformed revision hashes in the API server") + invalid := updated.DeepCopy() + invalid.Spec.InputRevision = "unversioned" + require.True(t, apierrors.IsInvalid(env.Client().Update(t.Context(), invalid))) + + t.Log("The status subresource retains child observations without rewriting source DGD status") + updated.Status.ObservedGeneration = updated.Generation + updated.Status.Conditions[0].ObservedGeneration = updated.Generation + require.NoError(t, env.Client().Status().Update(t.Context(), updated)) + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(updated), stored)) + require.Equal(t, updated.Status, stored.Status) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, stored, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + storedSource := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) + require.Equal(t, source.Status, storedSource.Status) +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go new file mode 100644 index 000000000000..88091723b643 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -0,0 +1,234 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "strconv" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// dgdLPXHandoff owns only the generated child and its status projection. Build, +// Grove and scheduler lifecycles are exclusively the LPX controller's concern. +type dgdLPXHandoff struct{ client.Client } + +func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGraphDeployment) (*v1alpha1.LPXGraphDeployment, error) { + child := &v1alpha1.LPXGraphDeployment{} + err := r.Get(ctx, client.ObjectKeyFromObject(source), child) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + exists := err == nil + if exists && !exactLPXSourceOwner(child, source) { + return nil, fmt.Errorf("LPXGraphDeployment %q belongs to a different source; adoption is not supported", child.Name) + } + if !source.HasLPXComponent() { + if !exists { + return nil, nil + } + return child, r.deleteChild(ctx, child) + } + if exists && !child.DeletionTimestamp.IsZero() { + return child, nil + } + + restart := dynamo.LPXRestartToken(source, child.Annotations[dynamo.LPXRestartAnnotation]) + revision, err := dynamo.LPXInputRevision(source, restart) + if err != nil { + return nil, err + } + if exists && child.Spec.InputRevision == revision && + child.Annotations[dynamo.LPXRestartAnnotation] == restart && + child.Annotations[dynamo.LPXPCSNameAnnotation] == dynamo.PCSNameForLPX(source) { + return child, nil + } + if !exists { + child = &v1alpha1.LPXGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace}, + } + if err := ctrl.SetControllerReference(source, child, r.Scheme()); err != nil { + return nil, err + } + } + child.Spec.InputRevision = revision + metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXPCSNameAnnotation, dynamo.PCSNameForLPX(source)) + metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXRestartAnnotation, restart) + metav1.SetMetaDataAnnotation(&child.ObjectMeta, lpx.DGDGenerationAnnotation, strconv.FormatInt(source.Generation, 10)) + if exists { + err = r.Update(ctx, child) + } else { + err = r.Create(ctx, child) + } + return child, err +} + +func exactLPXSourceOwner(child *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) bool { + owner := metav1.GetControllerOf(child) + return child.Name == source.Name && child.Namespace == source.Namespace && + owner != nil && owner.Kind == "DynamoGraphDeployment" && owner.APIVersion == v1beta1.GroupVersion.String() && + owner.Name == source.Name && owner.UID == source.UID +} + +func (r *dgdLPXHandoff) deleteChild(ctx context.Context, child *v1alpha1.LPXGraphDeployment) error { + if !child.DeletionTimestamp.IsZero() { + return nil + } + uid, version := child.UID, child.ResourceVersion + return client.IgnoreNotFound(r.Delete(ctx, child, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}})) +} + +func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGraphDeployment) error { + child := &v1alpha1.LPXGraphDeployment{} + if err := r.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil { + return client.IgnoreNotFound(err) + } + if !exactLPXSourceOwner(child, source) { + return fmt.Errorf("refusing to delete a foreign LPXGraphDeployment %q", child.Name) + } + if err := r.deleteChild(ctx, child); err != nil { + return err + } + return fmt.Errorf("waiting for LPXGraphDeployment %q to finish cleanup", child.Name) +} + +// resolveRestartProgress composes child-owned LPX and ordinary Grove observations. +func (p *groveProgram) resolveRestartProgress(ctx context.Context, dgd *v1beta1.DynamoGraphDeployment, inProgress []string) []string { + ordinary := make([]string, 0, len(inProgress)) + pending := make(map[string]bool, len(inProgress)) + var childComponents map[string]v1beta1.ComponentReplicaStatus + childObserved := false + + // Partition live members and observe the shared LPX child only when requested. + for _, name := range inProgress { + component := dgd.GetComponentByName(name) + if component == nil { + continue + } + if !component.IsLPX() { + ordinary = append(ordinary, name) + continue + } + if !childObserved { + childComponents = observeLPXRestart(ctx, p.lpx.Client, dgd) + childObserved = true + } + if !childComponents[name].Ready { + pending[name] = true + } + } + + // Native pending or failed observations cannot prevent LPX progress. + if len(ordinary) > 0 { + for _, name := range p.restartProgress.Resolve(ctx, dgd, ordinary) { + pending[name] = true + } + } + + // Retain the original authored restart order across the two observations. + remaining := make([]string, 0, len(inProgress)) + for _, name := range inProgress { + if pending[name] { + remaining = append(remaining, name) + } + } + return remaining +} + +// observeLPXRestart returns component status from one current, ready restart observation. +// A failed read or incomplete child leaves every requested member pending. +func observeLPXRestart(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment) map[string]v1beta1.ComponentReplicaStatus { + child := &v1alpha1.LPXGraphDeployment{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || + child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || + dynamo.ValidateLPXSource(child, source) != nil || source.Spec.Restart == nil || + child.Annotations[dynamo.LPXRestartAnnotation] != source.Spec.Restart.ID { + return nil + } + ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { + return nil + } + return child.Status.Components +} + +// projectLPXChildStatus projects the child returned by a successful handoff of source. +// Its identity is already reconciled; results and conditions still require current generations. +// child may be nil or deleting; all other pointer inputs must be non-nil. +func projectLPXChildStatus(source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, result *ReconcileResult, status *v1beta1.DynamoGraphDeploymentStatus) { + status.LPX = nil + components := lpx.Components(source) + if len(components) == 0 && child == nil { + return + } + if result.ComponentStatus == nil { + result.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) + } + serving := lpx.ServingComponent(source) + for _, component := range components { + kind := v1beta1.ComponentKindPodCliqueScalingGroup + if component != serving { + kind = v1beta1.ComponentKindPodClique + } + result.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ + ComponentKind: kind, + } + } + // Result payloads belong only to a completely observed revision. + current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() + observed := current && child.Status.ObservedGeneration == child.Generation + if observed { + status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: child.Status.ModelDownload.DeepCopy(), + Placement: child.Status.Placement.DeepCopy(), + } + // Public attempt observation is relative to the DGD, not its private child. + if status.LPX.Placement != nil && status.LPX.Placement.LPXAttempt != nil { + status.LPX.Placement.LPXAttempt.ObservedGeneration = source.Generation + } + // Project only current authored members from the single observed child. + for _, component := range components { + if observedStatus, found := child.Status.Components[component.ComponentName]; found { + result.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() + } + } + } + + // Reconciliation errors advance their condition, not the complete result. + // Surface that current failure without exposing stale payloads or hiding an ordinary failure. + if result.State != v1beta1.DGDStateFailed && current { + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + if failed != nil && failed.Status == metav1.ConditionTrue && failed.ObservedGeneration == child.Generation { + result.State = v1beta1.DGDStateFailed + result.Reason, result.Message = Reason(failed.Reason), Message(failed.Message) + return + } + } + if observed { + // The child publishes aggregate readiness and member status atomically. + ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + if ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { + return + } + if result.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { + result.State = v1beta1.DGDStatePending + result.Reason, result.Message = Reason(ready.Reason), Message(ready.Message) + return + } + } + if result.State != v1beta1.DGDStateFailed { + result.State = v1beta1.DGDStatePending + result.Reason = "LPXChildPending" + result.Message = "Waiting for the current LPX engine revision and readiness" + } +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go new file mode 100644 index 000000000000..9fbe9c1977d6 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -0,0 +1,263 @@ +//go:build !clustertest + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "capnproto.org/go/capnp/v3" + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { + t.Log("Encode a real two-partition LPU build owned by this test") + buildDir := t.TempDir() + message, segment := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnpv2.NewRootManifest(segment) + require.NoError(t, err) + manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) + model, err := manifest.NewModel() + require.NoError(t, err) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stopTokens, err := tokenizer.NewStopTokens(1) + require.NoError(t, err) + stopTokens.Set(0, 1) + build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) + require.NoError(t, err) + require.NoError(t, build.SetText(11, "publication-test")) + require.NoError(t, manifest.SetReserved3(build.ToPtr())) + deployment, err := manifest.NewDeployment() + require.NoError(t, err) + deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpuOnly) + deployment.SetNumLpuNodes(4) + program, err := deployment.NewProgram() + require.NoError(t, err) + program.SetBatchSize(1) + program.SetSequenceLength(8192) + program.SetInputSize(1) + program.SetOutputSize(1) + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + runtimeIO, err := deployment.NewRuntimeIo() + require.NoError(t, err) + runtimeIO.SetProtocol(0) + runtimeIO.SetReserved1(1) + runtimeIO.SetIoFpgaCount(1) + runtimeIO.SetFanoutFactor(1) + chains, err := deployment.NewSelectedPropSyncChains(1) + require.NoError(t, err) + partitionIDs, err := chains.At(0).NewPartitionIds(2) + require.NoError(t, err) + partitionIDs.Set(0, 7) + partitionIDs.Set(1, 8) + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(2) + require.NoError(t, err) + for index := range partitions.Len() { + partition, err := partitions.At(index).NewPartition() + require.NoError(t, err) + partition.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + partition.SetPartitionId(uint32(index + 7)) + detail, err := partitions.At(index).Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", index+7))) + require.NoError(t, detail.SetTopology("URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA")) + detail.SetNumChips(16) + detail.SetDevicesPerNode(8) + } + payload, err := message.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), payload, 0o600)) + + t.Log("Start an isolated API server and register the external kinds needed only for watches") + config := &configv1alpha1.OperatorConfiguration{} + config.LPX.Enabled = true + config.MPI.SSHSecretName = "ssh-secret" + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}} + env := operatorenv.New(operatorenv.Options{ + Config: config, RuntimeConfig: runtimeConfig, + Admission: operatorenv.AdmissionWebhooks{Mutating: true, Validating: true}, + SetupWebhooks: func(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptions) error { + if err := lpxv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { + return err + } + if err := groveschedulerv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { + return err + } + return setupProductionWebhooks(mgr, opts) + }, + }).RunT(t) + dependencies := make([]*apiextensionsv1.CustomResourceDefinition, 0, 2) + for _, dependency := range []struct{ group, kind, plural string }{ + {lpxv1alpha1.APIGroup, "LpuPipelineRequest", "lpupipelinerequests"}, + {groveschedulerv1alpha1.SchemeGroupVersion.Group, "PodGang", "podgangs"}, + } { + dependencies = append(dependencies, &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: dependency.plural + "." + dependency.group}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: dependency.group, Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{Kind: dependency.kind, ListKind: dependency.kind + "List", Plural: dependency.plural}, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1alpha1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{Type: "object", XPreserveUnknownFields: ptr.To(true)}}, + }}, + }, + }) + } + _, err = envtest.InstallCRDs(env.RESTConfig(), envtest.CRDInstallOptions{CRDs: dependencies}) + require.NoError(t, err) + + t.Log("Deny Grove publication with real namespace quota admission, not a mocked client") + quotaResource := corev1.ResourceName("count/podcliquesets.grove.io") + quota := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "block-lpx-publication", Namespace: env.Namespace()}, + Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{quotaResource: resource.MustParse("0")}}, + } + require.NoError(t, env.Client().Create(t.Context(), quota)) + quota.Status = corev1.ResourceQuotaStatus{Hard: quota.Spec.Hard.DeepCopy(), Used: corev1.ResourceList{quotaResource: resource.MustParse("0")}} + require.NoError(t, env.Client().Status().Update(t.Context(), quota)) + config = env.OperatorConfig().DeepCopy() + config.Namespace.Restricted = env.Namespace() + env.StartManager(func(mgr ctrl.Manager) error { + return SetupDynamoGraphDeployment(mgr, DynamoGraphDeploymentSetupOptions{ + SetupOptions: SetupOptions{Config: config, RuntimeConfig: runtimeConfig}, + }) + }) + + t.Log("Create only the public DGD and let production setup reconcile both controllers") + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "publication-failure", Namespace: env.Namespace()}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "lpx", ComponentType: v1beta1.ComponentTypeLPX, + LPX: &v1beta1.LPXConfig{BuildID: (&url.URL{Scheme: "file", Path: buildDir}).String()}, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLeader}, { + Name: v1beta1.ComponentRoleWorker, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: consts.MainContainerName, Image: "example/lpu-runtime:1.4.0", + VolumeMounts: []corev1.VolumeMount{{Name: consts.ModelStorageVolumeName, MountPath: "/models"}}, + }}, + Volumes: []corev1.Volume{{Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + }}, + }}, + }}}, + } + require.NoError(t, env.Client().Create(t.Context(), source)) + key := client.ObjectKeyFromObject(source) + require.EventuallyWithT(t, func(c *assert.CollectT) { + if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) { + return + } + ready := meta.FindStatusCondition(source.Status.Conditions, "Ready") + if assert.NotNil(c, ready) { + assert.Contains(c, ready.Message, "exceeded quota: block-lpx-publication") + } + }, 20*time.Second, 50*time.Millisecond) + + t.Log("Change LPX intent so the failing child generation is newer than its completed observation") + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "example/lpu-runtime:1.4.1" + require.NoError(t, env.Client().Update(t.Context(), source)) + child := &v1alpha1.LPXGraphDeployment{} + require.EventuallyWithT(t, func(c *assert.CollectT) { + if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { + return + } + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + ready := meta.FindStatusCondition(source.Status.Conditions, "Ready") + if !assert.NotNil(c, failed) || !assert.NotNil(c, ready) { + return + } + assert.Greater(c, child.Generation, child.Status.ObservedGeneration) + assert.Equal(c, child.Generation, failed.ObservedGeneration) + assert.Equal(c, metav1.ConditionTrue, failed.Status) + assert.Contains(c, failed.Message, "exceeded quota: block-lpx-publication") + assert.Equal(c, v1beta1.DGDStateFailed, source.Status.State) + assert.Equal(c, source.Generation, ready.ObservedGeneration) + assert.Equal(c, metav1.ConditionFalse, ready.Status) + assert.Equal(c, failed.Reason, ready.Reason) + assert.Equal(c, failed.Message, ready.Message) + assert.NotNil(c, child.Status.ModelDownload) + assert.Nil(c, source.Status.LPX) + }, 20*time.Second, 50*time.Millisecond) + + t.Log("The rejected publication created neither Grove workloads nor scheduler requests") + cliques := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, env.Client().List(t.Context(), cliques, client.InNamespace(env.Namespace()))) + require.Empty(t, cliques.Items) + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, env.Client().List(t.Context(), requests, client.InNamespace(env.Namespace()))) + require.Empty(t, requests.Items) + + t.Log("Accept an LPX runtime-invalid edit and report its exact rejection on the public DGD") + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeName = "manual-placement" + require.NoError(t, env.Client().Update(t.Context(), source)) + require.EventuallyWithT(t, func(c *assert.CollectT) { + if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { + return + } + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + ready := meta.FindStatusCondition(source.Status.Conditions, "Ready") + if !assert.NotNil(c, failed) || !assert.NotNil(c, ready) { + return + } + assert.Equal(c, child.Generation, failed.ObservedGeneration) + assert.Equal(c, "LPXRejected", failed.Reason) + assert.Contains(c, failed.Message, "spec.components[0].roles[1].podTemplate.spec.nodeName") + assert.Contains(c, failed.Message, "LPX owns role addressing and placement") + assert.Equal(c, v1beta1.DGDStateFailed, source.Status.State) + assert.Equal(c, source.Generation, ready.ObservedGeneration) + assert.Equal(c, metav1.ConditionFalse, ready.Status) + assert.Equal(c, failed.Reason, ready.Reason) + assert.Equal(c, failed.Message, ready.Message) + }, 20*time.Second, 50*time.Millisecond) + + t.Log("Repair the accepted configuration so the same DGD and child can reconcile again") + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeName = "" + require.NoError(t, env.Client().Update(t.Context(), source)) + + t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") + require.NoError(t, env.Client().Delete(t.Context(), quota)) + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: child.Annotations[dynamo.LPXPCSNameAnnotation], Namespace: source.Namespace}} + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: source.Name + "-lpu", Namespace: source.Namespace}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: dynamo.GetDCDResourceName(source, "lpx", ""), Namespace: source.Namespace}} + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, resource := range []client.Object{pcs, configMap, service} { + if assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(resource), resource)) { + assert.Equal(c, metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK), metav1.GetControllerOf(resource)) + } + } + }, 20*time.Second, 50*time.Millisecond) +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go new file mode 100644 index 000000000000..f91a6c9dc122 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -0,0 +1,689 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/yaml" +) + +// newLPXHandoffSource loads a fresh authored workload without compiler or child-controller fixtures. +func newLPXHandoffSource(t *testing.T, fixture string) *v1beta1.DynamoGraphDeployment { + t.Helper() + data, err := os.ReadFile(filepath.Join("../dynamo/testdata/from_dgd_yaml", fixture+".input.yaml")) + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(data, source)) + lpx.ServingComponent(source).ComponentName = "lpx" + source.Generation = 3 + return source +} + +// newLPXHandoffFixture creates only the parent-owned handoff through its production path. +func newLPXHandoffFixture(t *testing.T, fixture string) (*v1alpha1.LPXGraphDeployment, *v1beta1.DynamoGraphDeployment, client.Client) { + t.Helper() + source := newLPXHandoffSource(t, fixture) + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). + WithRESTMapper(groveScaleRESTMapper()). + WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}, &v1alpha1.LPXGraphDeployment{}). + WithObjects(source).Build() + child, err := (&dgdLPXHandoff{Client: kube}).Reconcile(t.Context(), source) + require.NoError(t, err) + child.UID, child.Generation = "child-uid", 3 + require.NoError(t, kube.Update(t.Context(), child)) + return child, source, kube +} + +func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { + t.Log("Create the generated handoff from a real source DGD") + source := newLPXHandoffSource(t, "node-local-v2-lpu-only") + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(source).Build() + handoff := &dgdLPXHandoff{Client: kube} + child, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.True(t, exactLPXSourceOwner(child, source)) + require.NotEmpty(t, child.Spec.InputRevision) + require.NoError(t, dynamo.ValidateLPXSource(child, source)) + require.NotEqual(t, dynamo.PCSNameForDGD(source.Name, source.Spec.Components), child.Annotations[dynamo.LPXPCSNameAnnotation]) + pending := ReconcileResult{State: v1beta1.DGDStatePending} + projectLPXChildStatus(source, child, &pending, &v1beta1.DynamoGraphDeploymentStatus{}) + require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, pending.ComponentStatus["lpx"].ComponentKind) + + t.Log("Reject foreign references before returning a child for status projection") + for _, invalidIdentity := range []string{"foreign-owner", "missing-owner"} { + foreign := child.DeepCopy() + switch invalidIdentity { + case "foreign-owner": + foreign.OwnerReferences[0].UID = "another-owner" + case "missing-owner": + foreign.OwnerReferences = nil + } + foreignClient := fake.NewClientBuilder().WithScheme(kube.Scheme()).WithObjects(foreign).Build() + _, err := (&dgdLPXHandoff{Client: foreignClient}).Reconcile(t.Context(), source) + require.ErrorContains(t, err, "adoption is not supported", invalidIdentity) + } + + t.Log("Refuse adoption and deletion after the source name is reused with a different UID") + source.UID = "replacement-source" + _, err = handoff.Reconcile(t.Context(), source) + require.ErrorContains(t, err, "adoption is not supported") + require.ErrorContains(t, handoff.Finalize(t.Context(), source), "foreign") + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.True(t, stored.DeletionTimestamp.IsZero()) +} + +func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { + t.Log("Project exactly one complete LPX engine alongside an ordinary component") + child, source, _ := newLPXHandoffFixture(t, "node-local-v2-hybrid") + child.Status.ObservedGeneration = child.Generation + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": { + Replicas: 3, AvailableReplicas: ptr.To(int32(3)), Ready: true, + GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), + }} + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"build"}} + child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ + ObservedGeneration: child.Generation, + Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "request", UID: "request-uid"}}, + }} + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) + result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4)), Ready: true}}} + status := v1beta1.DynamoGraphDeploymentStatus{} + projectLPXChildStatus(source, child, &result, &status) + require.Equal(t, v1beta1.DGDStateSuccessful, result.State) + require.Len(t, result.ComponentStatus, 2) + require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) + status.LPX.ModelDownload.Builds[0] = "parent-copy" + status.LPX.Placement.LPXAttempt.Requests[0].UID = "parent-copy" + require.Equal(t, "build", child.Status.ModelDownload.Builds[0]) + require.Equal(t, "request-uid", string(child.Status.Placement.LPXAttempt.Requests[0].UID)) + + t.Log("Reject missing or stale children and stale Ready conditions") + for _, failure := range []string{"missing-child", "generation", "condition"} { + stale := child.DeepCopy() + switch failure { + case "missing-child": + stale = nil + case "generation": + stale.Status.ObservedGeneration-- + case "condition": + stale.Status.Conditions[0].ObservedGeneration-- + } + pending := ReconcileResult{State: v1beta1.DGDStateSuccessful} + previous := status + projectLPXChildStatus(source, stale, &pending, &previous) + require.Equal(t, v1beta1.DGDStatePending, pending.State, failure) + if failure == "missing-child" || failure == "generation" { + require.Nil(t, previous.LPX, failure) + } else { + require.NotNil(t, previous.LPX, failure) + } + } + + t.Log("A partially ready child retains logical replica counts and GPU capacity in public status") + partial := child.Status.Components["lpx"] + partial.AvailableReplicas, partial.Ready = ptr.To(int32(2)), false + child.Status.Components["lpx"] = partial + child.Status.ModelDownload, child.Status.Placement = nil, nil + result.Reason = "CheckpointReady" + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "LPXRetiring"}) + projectLPXChildStatus(source, child, &result, &status) + require.Equal(t, v1beta1.DGDStatePending, result.State) + require.Equal(t, Reason("LPXRetiring"), result.Reason) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{}, status.LPX) + require.Equal(t, partial, result.ComponentStatus["lpx"]) + source.Status.Components = result.ComponentStatus + request := &v1beta1.DynamoGraphDeploymentRequest{} + require.True(t, updateDeploymentInfo(request, source)) + require.Equal(t, ptr.To(int32(7)), request.Status.DeploymentInfo.Replicas) + require.Equal(t, ptr.To(int32(6)), request.Status.DeploymentInfo.AvailableReplicas) + + t.Log("Retirement remains pending in final DGD status after ordinary checkpoint readiness") + projected := workloadProgramResult{Status: status} + projected.applyReconcileResult(source.Generation, result) + require.Equal(t, v1beta1.DGDStatePending, projected.Status.State) + ready := meta.FindStatusCondition(projected.Status.Conditions, "Ready") + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, child.Status.Conditions[0].Reason, ready.Reason) + require.Equal(t, child.Status.Conditions[0].Message, ready.Message) + require.Equal(t, source.Generation, ready.ObservedGeneration) + + t.Log("Remove the complete LPX group when its components leave the source graph") + source.Spec.Components = nil + projectLPXChildStatus(source, child, &result, &status) + require.Nil(t, status.LPX) +} + +func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { + for _, test := range []struct { + name string + failed bool + }{ + {"unobserved-result", true}, + {"observed-result", true}, + {"old-condition", false}, + {"cleared-condition", false}, + {"deleting", false}, + {"ordinary-failure", true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Only a current child failure may bypass the complete-observation gate") + child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"stale-build"}} + child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: child.Generation}} + child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "PublicationDenied", Message: "Check the namespace quota"}} + result := ReconcileResult{State: v1beta1.DGDStateSuccessful} + status := v1beta1.DynamoGraphDeploymentStatus{LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: child.Status.ModelDownload.DeepCopy(), Placement: child.Status.Placement.DeepCopy(), + }} + switch test.name { + case "observed-result": + child.Status.ObservedGeneration = child.Generation + result = ReconcileResult{State: v1beta1.DGDStatePending, Reason: "RolloutInProgress"} + status.RollingUpdate = &v1beta1.RollingUpdateStatus{Phase: v1beta1.RollingUpdatePhaseInProgress} + case "old-condition": + child.Status.Conditions[0].ObservedGeneration-- + case "cleared-condition": + child.Status.Conditions[0].Status = metav1.ConditionFalse + case "deleting": + child.DeletionTimestamp = ptr.To(metav1.Now()) + case "ordinary-failure": + result = ReconcileResult{State: v1beta1.DGDStateFailed, Reason: "OrdinaryFailure", Message: "Existing ordinary failure"} + } + projectLPXChildStatus(source, child, &result, &status) + require.Equal(t, test.failed, result.State == v1beta1.DGDStateFailed) + if test.name == "ordinary-failure" { + require.Equal(t, Reason("OrdinaryFailure"), result.Reason) + require.Equal(t, Message("Existing ordinary failure"), result.Message) + } else if test.failed { + require.Equal(t, Reason(child.Status.Conditions[0].Reason), result.Reason) + require.Equal(t, Message(child.Status.Conditions[0].Message), result.Message) + } else { + require.Equal(t, Reason("LPXChildPending"), result.Reason) + } + + t.Log("A failure condition does not make stale result payloads current") + if test.name == "observed-result" { + require.NotNil(t, status.LPX) + require.NotNil(t, status.LPX.ModelDownload) + require.NotNil(t, status.LPX.Placement) + require.Equal(t, int32(1), result.ComponentStatus["lpx"].Replicas) + + t.Log("A retained ordinary rollout cannot hide the child failure in final DGD status") + projected := workloadProgramResult{Status: status} + projected.applyReconcileResult(source.Generation, result) + require.Equal(t, v1beta1.DGDStateFailed, projected.Status.State) + ready := meta.FindStatusCondition(projected.Status.Conditions, "Ready") + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, child.Status.Conditions[0].Reason, ready.Reason) + require.Equal(t, child.Status.Conditions[0].Message, ready.Message) + require.Equal(t, source.Generation, ready.ObservedGeneration) + } else { + require.Nil(t, status.LPX) + require.Zero(t, result.ComponentStatus["lpx"].Replicas) + } + }) + } +} + +func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { + for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeSequential, v1beta1.RestartStrategyTypeParallel} { + t.Run(string(strategy), func(t *testing.T) { + t.Log("Request an LPX-only restart without requiring an ordinary PCS") + _, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + source.Spec.Restart = &v1beta1.Restart{ID: "restart-1", Strategy: &v1beta1.RestartStrategy{Type: strategy}} + handoff := &dgdLPXHandoff{Client: kube} + unchanged, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Empty(t, unchanged.Annotations[dynamo.LPXRestartAnnotation]) + progress := (&DynamoGraphDeploymentReconciler{ + Client: kube, RuntimeConfig: &commoncontroller.RuntimeConfig{}, + }).newGroveProgram().resolveRestartProgress + restart := newDGDRestartReconciler().Resolve(t.Context(), source, &source.Status, progress) + require.Equal(t, []string{"lpx"}, restart.Status.InProgress) + + t.Log("Deliver the token only after the DGD has persisted its component selection") + source.Status.Restart = restart.Status + updated, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, "restart-1", updated.Annotations[dynamo.LPXRestartAnnotation]) + require.Equal(t, []string{"lpx"}, progress(t.Context(), source, []string{"lpx"})) + updated.Status.ObservedGeneration = updated.Generation + updated.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Ready: true, Replicas: 1, AvailableReplicas: ptr.To(int32(1))}} + meta.SetStatusCondition(&updated.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: updated.Generation, Reason: "Ready"}) + require.NoError(t, kube.Status().Update(t.Context(), updated)) + + t.Log("Complete the restart only from the current child's full-engine readiness") + require.Empty(t, progress(t.Context(), source, []string{"lpx"})) + completed := newDGDRestartReconciler().Resolve(t.Context(), source, &source.Status, progress) + require.Equal(t, v1beta1.RestartPhaseCompleted, completed.Status.Phase) + updated.Status.ObservedGeneration-- + require.NoError(t, kube.Status().Update(t.Context(), updated)) + require.Equal(t, []string{"lpx"}, progress(t.Context(), source, []string{"lpx"})) + + t.Log("A second request cannot borrow the completed first restart's selection") + source.Status.Restart = completed.Status + source.Spec.Restart.ID = "restart-2" + source.Generation++ + require.NoError(t, dynamo.ValidateLPXSource(updated, source)) + unchanged, err = handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, updated, unchanged) + + t.Log("Deliver the second restart only after its own selection is persisted") + source.Status.Restart = newDGDRestartReconciler().Resolve(t.Context(), source, &source.Status, progress).Status + next, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, "restart-2", next.Annotations[dynamo.LPXRestartAnnotation]) + require.NotEqual(t, updated.Spec.InputRevision, next.Spec.InputRevision) + }) + } +} + +func TestLPXHandoffOrdinaryScalingPreservesTheEntireAttempt(t *testing.T) { + t.Log("Seed the shared child's current readiness, published request identity and deadline") + child, source, kube := newLPXHandoffFixture(t, "node-local-v2-specdecode") + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1)), + }, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:0"}}}}, + }) + deadline := metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(time.Minute)) + child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ + ObservedGeneration: child.Generation, DeadlineAt: &deadline, + Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "published-request", UID: "request-uid"}}, + }} + child.Status.ObservedGeneration = child.Generation + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ + "lpx": {Replicas: 1, AvailableReplicas: ptr.To(int32(1)), Ready: true}, + "draft": {Replicas: 2, AvailableReplicas: ptr.To(int32(2)), Ready: true}, + } + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) + require.NoError(t, kube.Status().Update(t.Context(), child)) + beforeChild := child.DeepCopy() + handoff := &dgdLPXHandoff{Client: kube} + + t.Log("Reorder the pair and make five frontend-only edits without changing the child") + source.Spec.Components[0], source.Spec.Components[1] = source.Spec.Components[1], source.Spec.Components[0] + for edit := range 5 { + source.GetComponentByName("frontend").PodTemplate.Spec.Containers[0].Image = "frontend:" + strconv.Itoa(edit+1) + source.Generation++ + beforeSource := source.DeepCopy() + current, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, beforeChild, current) + ordinary := ReconcileResult{State: v1beta1.DGDStateSuccessful, Reason: "Ready"} + projected := newWorkloadProgramResult(source) + projectLPXChildStatus(source, current, &ordinary, &projected.Status) + projected.applyReconcileResult(source.Generation, ordinary) + require.Equal(t, v1beta1.DGDStateSuccessful, projected.Status.State) + require.True(t, meta.IsStatusConditionTrue(projected.Status.Conditions, "Ready")) + require.Equal(t, source.Generation, projected.Status.ObservedGeneration) + require.Equal(t, source.Generation, projected.Status.LPX.Placement.LPXAttempt.ObservedGeneration) + require.Equal(t, beforeChild.Status.Placement, current.Status.Placement) + require.True(t, deadline.Equal(projected.Status.LPX.Placement.LPXAttempt.DeadlineAt)) + require.Equal(t, beforeSource, source) + source.Status = projected.Status + } + require.Equal(t, int64(8), source.Generation) + require.Equal(t, int64(3), beforeChild.Status.Placement.LPXAttempt.ObservedGeneration) + + t.Log("Scale prefill without changing the child or its durable attempt") + source.GetComponentByName("prefill").Replicas = ptr.To(int32(4)) + source.Generation++ + current, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, beforeChild, current) + + t.Log("LPX capacity updates the handoff without modifying its child's durable attempt") + source.GetComponentByName("draft").Replicas = ptr.To(int32(3)) + source.Generation++ + updated, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.NotEqual(t, beforeChild.Spec.InputRevision, updated.Spec.InputRevision) + require.Equal(t, strconv.FormatInt(source.Generation, 10), updated.Annotations[lpx.DGDGenerationAnnotation]) + require.Equal(t, beforeChild.Status, updated.Status) + // Fake clients do not advance generations; the isolated API test covers that transition. + updated.Generation++ + projected := newWorkloadProgramResult(source) + require.NotNil(t, projected.Status.LPX) + pending := ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, updated, &pending, &projected.Status) + require.Equal(t, v1beta1.DGDStatePending, pending.State) + require.Nil(t, projected.Status.LPX) +} + +func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { + for _, scenario := range []struct { + name string + strategy v1beta1.RestartStrategy + }{ + {"draft first", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeSequential, Order: []string{"draft", "lpx"}}}, + {"target first", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeSequential, Order: []string{"lpx", "draft"}}}, + {"parallel", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeParallel}}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Persist a requested restart of the authored draft and target") + _, source, kube := newLPXHandoffFixture(t, "node-local-v2-specdecode") + source.Spec.Restart = &v1beta1.Restart{ID: "pair-restart", Strategy: &scenario.strategy} + var childReads, pcsReads int + var failChildRead bool + observed := interceptor.NewClient(kube.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, pcs := obj.(*grovev1alpha1.PodCliqueSet); pcs { + pcsReads++ + } + if _, child := obj.(*v1alpha1.LPXGraphDeployment); child { + childReads++ + if failChildRead { + return apierrors.NewServiceUnavailable("child observation unavailable") + } + } + return reader.Get(ctx, key, obj, opts...) + }, + }) + progress := (&DynamoGraphDeploymentReconciler{ + Client: observed, RuntimeConfig: &commoncontroller.RuntimeConfig{}, + }).newGroveProgram().resolveRestartProgress + restarter := newDGDRestartReconciler() + source.Status.Restart = restarter.Resolve(t.Context(), source, &source.Status, progress).Status + require.NotEmpty(t, source.Status.Restart.InProgress) + + t.Log("Either selected member delivers one token to the same child") + handoff := &dgdLPXHandoff{Client: kube} + child, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, "pair-restart", child.Annotations[dynamo.LPXRestartAnnotation]) + require.NotEmpty(t, progress(t.Context(), source, source.Status.Restart.InProgress)) + child.Status.ObservedGeneration = child.Generation + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ + "lpx": {Ready: true, Replicas: 1}, "draft": {Ready: true, Replicas: 2}, + } + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) + require.NoError(t, kube.Status().Update(t.Context(), child)) + before := child.DeepCopy() + + t.Log("One failed child observation keeps both ready members pending") + failChildRead, childReads = true, 0 + require.Equal(t, []string{"draft", "lpx"}, progress(t.Context(), source, []string{"draft", "lpx"})) + require.Equal(t, 1, childReads) + failChildRead = false + + t.Log("Finish each selected member without changing the shared child revision again") + for range 2 { + childReads = 0 + require.Empty(t, progress(t.Context(), source, source.Status.Restart.InProgress)) + require.Equal(t, min(len(source.Status.Restart.InProgress), 1), childReads) + source.Status.Restart = restarter.Resolve(t.Context(), source, &source.Status, progress).Status + current, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, before, current) + } + require.Equal(t, v1beta1.RestartPhaseCompleted, source.Status.Restart.Phase) + + t.Log("A missing draft status cannot borrow the target's readiness") + delete(child.Status.Components, "draft") + require.NoError(t, kube.Status().Update(t.Context(), child)) + require.Equal(t, []string{"draft"}, progress(t.Context(), source, []string{"draft", "lpx"})) + require.Zero(t, pcsReads) + if scenario.strategy.Type != v1beta1.RestartStrategyTypeParallel { + return + } + + t.Log("A missing ordinary PCS keeps only live ordinary restart members pending") + source.Spec.Components = append(source.Spec.Components, + v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1))}, + v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, + ) + child.Status.Components["draft"] = v1beta1.ComponentReplicaStatus{Ready: true, Replicas: 2} + require.NoError(t, kube.Status().Update(t.Context(), child)) + requested := []string{"draft", "frontend", "lpx", "prefill", "removed"} + beforeSource := source.DeepCopy() + childReads, pcsReads = 0, 0 + require.Equal(t, []string{"frontend", "prefill"}, progress(t.Context(), source, requested)) + require.Equal(t, 1, childReads) + require.Equal(t, 1, pcsReads) + + t.Log("An unobserved ordinary PCS cannot hold back either ready LPX member") + pcs := &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForDGD(source.Name, source.Spec.Components), Namespace: source.Namespace, Generation: 2}, + Status: grovev1alpha1.PodCliqueSetStatus{ObservedGeneration: ptr.To(int64(1))}, + } + require.NoError(t, kube.Create(t.Context(), pcs)) + childReads, pcsReads = 0, 0 + require.Equal(t, []string{"frontend", "prefill"}, progress(t.Context(), source, requested)) + require.Equal(t, 1, childReads) + require.Equal(t, 1, pcsReads) + + t.Log("A failed child read cannot hold back a ready ordinary member or reorder pending members") + pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) + require.NoError(t, kube.Update(t.Context(), pcs)) + prefill := &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(source, "prefill"), Namespace: source.Namespace, Generation: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, + } + require.NoError(t, kube.Create(t.Context(), prefill)) + failChildRead, childReads, pcsReads = true, 0, 0 + require.Equal(t, []string{"draft", "frontend", "lpx"}, progress(t.Context(), source, requested)) + require.Equal(t, 1, childReads) + require.Equal(t, 1, pcsReads) + require.Equal(t, beforeSource, source) + require.Equal(t, []string{"draft", "frontend", "lpx", "prefill", "removed"}, requested) + }) + } +} + +func TestLPXChildFinalizationAndDeselectionWaitForCleanup(t *testing.T) { + t.Log("Deselect LPX while the child reports unfinished cleanup through its finalizer") + child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + child.Finalizers = []string{"test.example/child-cleanup"} + require.NoError(t, kube.Update(t.Context(), child)) + handoff := &dgdLPXHandoff{Client: kube} + source.Spec.Components = nil + deleting, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.NotNil(t, deleting) + require.ErrorContains(t, handoff.Finalize(t.Context(), source), "finish cleanup") + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.False(t, stored.DeletionTimestamp.IsZero()) + + t.Log("The outer DGD finalizer must wait for LPX before inspecting checkpoint cleanup") + snapshotReads := 0 + observed := interceptor.NewClient(kube.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, snapshot := list.(*snapshotv1alpha1.SnapshotJobList); snapshot { + snapshotReads++ + } + return delegated.List(ctx, list, opts...) + }, + }) + parent := &DynamoGraphDeploymentReconciler{Client: observed} + require.ErrorContains(t, parent.FinalizeResource(t.Context(), source), "waiting for LPXGraphDeployment") + require.Zero(t, snapshotReads) + + t.Log("Finish parent finalization only after the child releases its cleanup finalizer") + stored.Finalizers = nil + require.NoError(t, kube.Update(t.Context(), stored)) + require.NoError(t, handoff.Finalize(t.Context(), source)) + require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored))) +} + +func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { + t.Log("Leave the child pending while the DGD desires an ordinary prefill workload") + _, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + require.NoError(t, rbacv1.AddToScheme(kube.Scheme())) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + source.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} + source.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} + source.Spec.Labels = map[string]string{"example.com/policy": "enabled"} + source.Spec.BackendFramework = string(dynamo.BackendFrameworkVLLM) + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "vllm-runtime"}}}}, + ProviderOverride: &v1beta1.ProviderOverride{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueTemplateSpec", + Value: apiextensionsv1.JSON{Raw: []byte(`{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}`)}, + }, + }) + require.NoError(t, kube.Update(t.Context(), source)) + child, err := (&dgdLPXHandoff{Client: kube}).Reconcile(t.Context(), source) + require.NoError(t, err) + child.Status.ObservedGeneration = child.Generation + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "ModelDownloadPending"}} + require.NoError(t, kube.Status().Update(t.Context(), child)) + config := &configv1alpha1.OperatorConfiguration{} + config.Namespace.Restricted = source.Namespace + parent := &DynamoGraphDeploymentReconciler{ + Client: kube, Config: config, Recorder: events.NewFakeRecorder(100), + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true, Checkpoint: true}}, + } + program := parent.newGroveProgram() + + t.Log("Publish ordinary resources independently of the child's pending download") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.DGDStatePending, result.Status.State) + pcs := &grovev1alpha1.PodCliqueSet{} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(source.Name, source.Spec.Components)} + require.NoError(t, kube.Get(t.Context(), key, pcs)) + require.True(t, metav1.IsControlledBy(pcs, source)) + require.Len(t, pcs.Spec.Template.Cliques, 1) + require.Equal(t, "prefill", pcs.Spec.Template.Cliques[0].Name) + require.Equal(t, &grovev1alpha1.TopologyConstraint{ + TopologyName: "fabric", Pack: &grovev1alpha1.TopologyPackConstraint{RequiredDomain: "rack"}, + }, pcs.Spec.Template.Cliques[0].TopologyConstraint) + + t.Log("Model discovery remains DGD-owned while the LPX child is downloading") + service := &corev1.Service{} + serviceKey := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.GenerateServiceName("test/model")} + require.NoError(t, kube.Get(t.Context(), serviceKey, service)) + require.Equal(t, corev1.ClusterIPNone, service.Spec.ClusterIP) + require.False(t, service.Spec.PublishNotReadyAddresses) + require.Equal(t, map[string]string{consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("test/model")}, service.Spec.Selector) + require.Equal(t, dynamo.HashModelName("test/model"), service.Labels[consts.KubeLabelDynamoBaseModelHash]) + require.Equal(t, "enabled", service.Annotations["example.com/model-discovery"]) + require.True(t, metav1.IsControlledBy(service, source)) + + t.Log("A pending ordinary checkpoint retains its startup and scaling gates alongside the pending child") + prefill := source.GetComponentByName("prefill") + prefill.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ + Enabled: true, CheckpointRef: ptr.To(friendlyCheckpointName), StartupPolicy: v1beta1.CheckpointStartupPolicyWaitForCheckpoint, + }} + prefill.ScalingAdapter = &v1beta1.ScalingAdapter{} + require.NoError(t, kube.Update(t.Context(), source)) + compatibilityHash, err := program.sharedResources.checkpoints.snapshotCompatibilityHashForComponent( + source, + "prefill", + prefill, + ) + require.NoError(t, err) + require.NotEmpty(t, compatibilityHash) + snapshot := dgdTestPodSnapshot(friendlyCheckpointName, compatibilityHash, false) + snapshot.Namespace = source.Namespace + require.NoError(t, kube.Create(t.Context(), snapshot)) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.DGDStatePending, result.Status.State) + require.Equal(t, string(reasonWaitingForCheckpoint), meta.FindStatusCondition(result.Status.Conditions, "Ready").Reason) + require.Equal(t, v1beta1.ComponentCheckpointStatus{CheckpointName: friendlyCheckpointName}, result.Status.Checkpoints["prefill"]) + require.NoError(t, kube.Get(t.Context(), key, pcs)) + require.Zero(t, pcs.Spec.Template.Cliques[0].Spec.Replicas) + adapters := &v1alpha1.DynamoGraphDeploymentScalingAdapterList{} + require.NoError(t, kube.List(t.Context(), adapters)) + require.Empty(t, adapters.Items) + + t.Log("An actionable child failure remains visible while the ordinary checkpoint is still pending") + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ + Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, + Reason: "LPXReconciliationFailed", Message: "publish LPX workload: storage volume mount is missing", + }) + require.NoError(t, kube.Status().Update(t.Context(), child)) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) + failed := meta.FindStatusCondition(result.Status.Conditions, "Ready") + require.Equal(t, "LPXReconciliationFailed", failed.Reason) + require.Contains(t, failed.Message, "storage volume mount is missing") + require.Equal(t, v1beta1.ComponentCheckpointStatus{CheckpointName: friendlyCheckpointName}, result.Status.Checkpoints["prefill"]) + + t.Log("Removing the last ordinary component deletes its PCS without disturbing the child") + source.Spec.Components = source.Spec.Components[:1] + for range 2 { + _, err = program.workloads.Reconcile(t.Context(), source, nil, nil) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), key, pcs))) + } + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.Equal(t, child, stored) +} + +func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { + t.Log("Seed an ordinary DGD with obsolete LPX-owned download and placement status") + checkedAt := metav1.Unix(1, 0) + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "download-status", Namespace: "default", Generation: 1}, + Status: v1beta1.DynamoGraphDeploymentStatus{ + State: v1beta1.DGDStateSuccessful, ObservedGeneration: 1, + LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}, LastCheckedAt: &checkedAt}, + Placement: &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: 1}}, + }, + }, + } + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). + WithStatusSubresource(source).WithObjects(source).Build() + r := &DynamoGraphDeploymentReconciler{ + Client: kube, Recorder: events.NewFakeRecorder(10), + Config: &configv1alpha1.OperatorConfiguration{ + Namespace: configv1alpha1.NamespaceConfiguration{Restricted: source.Namespace}, + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + }, + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, + } + + t.Log("Reconcile ordinary state without scheduling an LPX download refresh") + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)}) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + updated := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), updated)) + require.Nil(t, updated.Status.LPX) +} diff --git a/deploy/operator/internal/controller/dgd_scaling_adapters_reconciler.go b/deploy/operator/internal/controller/dgd_scaling_adapters_reconciler.go index 7f94db6320af..45445bd9a729 100644 --- a/deploy/operator/internal/controller/dgd_scaling_adapters_reconciler.go +++ b/deploy/operator/internal/controller/dgd_scaling_adapters_reconciler.go @@ -19,6 +19,7 @@ package controller import ( "context" + "crypto/sha256" "fmt" "strings" @@ -28,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -68,7 +70,7 @@ func (r *dgdScalingAdaptersReconciler) Reconcile( }, } - // Remove the adapter when scaling is no longer enabled for the component. + // Remove the adapter when component-level scaling is no longer enabled. if component.ScalingAdapter == nil { if err := r.Delete(ctx, adapter); err != nil { if apierrors.IsNotFound(err) { @@ -151,7 +153,7 @@ func (r *dgdScalingAdaptersReconciler) Reconcile( } } - // Delete adapters whose components have been removed from the DGD. + // Delete adapters whose component target was removed. adapterList := &nvidiacomv1alpha1.DynamoGraphDeploymentScalingAdapterList{} if err := r.List( ctx, @@ -166,11 +168,17 @@ func (r *dgdScalingAdaptersReconciler) Reconcile( for i := range adapterList.Items { adapter := &adapterList.Items[i] componentName := adapter.Spec.DGDRef.ServiceName + + // Retain adapters whose component still exists. if dgd.GetComponentByName(componentName) != nil { continue } - logger.Info("Deleting orphaned DynamoGraphDeploymentScalingAdapter", "adapter", adapter.Name, "component", componentName) + logger.Info( + "Deleting orphaned DynamoGraphDeploymentScalingAdapter", + "adapter", adapter.Name, + "component", componentName, + ) if err := r.Delete(ctx, adapter); err != nil { if apierrors.IsNotFound(err) { continue @@ -196,5 +204,19 @@ func (r *dgdScalingAdaptersReconciler) Reconcile( } func generateAdapterName(dgdName, componentName string) string { - return fmt.Sprintf("%s-%s", dgdName, strings.ToLower(componentName)) + name := fmt.Sprintf("%s-%s", dgdName, strings.ToLower(componentName)) + if len(k8svalidation.IsDNS1123Subdomain(name)) == 0 { + return name + } + + // Preserve uniqueness after constraining an invalid combination to a DNS label. + hash := sha256.Sum256([]byte(name)) + hashSuffix := fmt.Sprintf("-%x", hash[:8]) + prefix := strings.ReplaceAll(name, ".", "-") + maxPrefixLength := k8svalidation.DNS1123LabelMaxLength - len(hashSuffix) + if len(prefix) > maxPrefixLength { + prefix = prefix[:maxPrefixLength] + } + prefix = strings.TrimRight(prefix, "-") + return prefix + hashSuffix } diff --git a/deploy/operator/internal/controller/dgd_ssh_keys_reconciler.go b/deploy/operator/internal/controller/dgd_ssh_keys_reconciler.go index b34d899849fe..1d84551401fb 100644 --- a/deploy/operator/internal/controller/dgd_ssh_keys_reconciler.go +++ b/deploy/operator/internal/controller/dgd_ssh_keys_reconciler.go @@ -40,7 +40,7 @@ func (r *dgdSSHKeysReconciler) Reconcile( ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment, ) error { - if r.manager == nil || !dgd.HasAnyMultinodeComponent() { + if r.manager == nil || (!dgd.HasAnyMultinodeComponent() && !dgd.HasLPXComponent()) { return nil } if err := r.manager.EnsureAndReplicate(ctx, dgd.Namespace); err != nil { diff --git a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go index 8e19ce0b778d..aa5508903f2c 100644 --- a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go @@ -262,12 +262,23 @@ func TestGroveRenderDeploymentWorkerHashSuffix(t *testing.T) { dgd := createTestDGD("test-dgd", map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": {ComponentType: consts.ComponentTypeWorker}, }) + lpxSource := newLPXHandoffSource(t, "node-local-v2-hybrid") + dgd.Spec.Components = append(dgd.Spec.Components, lpxSource.Spec.Components[0], nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, + PodTemplate: &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"test": "original"}}}, + }) + before := dgd.DeepCopy() t.Log("Render the Grove deployment") rendered, err := groveRenderDeployment(dgd, nil, tt.workerHashSuffix) require.NoError(t, err) worker := rendered.GetComponentByName("worker") require.NotNil(t, worker) + require.Len(t, rendered.Spec.Components, 2) + require.Nil(t, rendered.GetComponentByName("lpx")) + require.Equal(t, "worker", rendered.Spec.Components[0].ComponentName) + require.Equal(t, "frontend", rendered.Spec.Components[1].ComponentName) + rendered.Spec.Components[1].PodTemplate.Labels["test"] = "rendered" t.Log("Verify the rendered suffix and source DGD immutability") if tt.workerHashSuffix { @@ -279,6 +290,7 @@ func TestGroveRenderDeploymentWorkerHashSuffix(t *testing.T) { assert.Nil(t, worker.PodTemplate) } assert.Nil(t, dgd.GetComponentByName("worker").PodTemplate) + assert.Equal(t, before, dgd) }) } } diff --git a/deploy/operator/internal/controller/dgd_workload_program.go b/deploy/operator/internal/controller/dgd_workload_program.go index 0387b923301e..953e106ad91f 100644 --- a/deploy/operator/internal/controller/dgd_workload_program.go +++ b/deploy/operator/internal/controller/dgd_workload_program.go @@ -89,8 +89,7 @@ func (r *DynamoGraphDeploymentReconciler) selectWorkloadProgram( func newWorkloadProgramResult( dgd *nvidiacomv1beta1.DynamoGraphDeployment, ) workloadProgramResult { - status := dgd.DeepCopy().Status - return workloadProgramResult{Status: status} + return workloadProgramResult{Status: *dgd.Status.DeepCopy()} } func (r *workloadProgramResult) Eventf( @@ -142,7 +141,7 @@ func (r *workloadProgramResult) applyReconcileResult( ) { r.Status.State = result.State r.Status.Components = result.ComponentStatus - if rollingUpdateInProgress(r.Status.RollingUpdate) { + if result.State != nvidiacomv1beta1.DGDStateFailed && rollingUpdateInProgress(r.Status.RollingUpdate) { r.Status.State = nvidiacomv1beta1.DGDStatePending } meta.SetStatusCondition(&r.Status.Conditions, readyCondition(generation, r.Status, result)) @@ -154,7 +153,7 @@ func readyCondition( status nvidiacomv1beta1.DynamoGraphDeploymentStatus, workloads ReconcileResult, ) metav1.Condition { - if rollingUpdateInProgress(status.RollingUpdate) { + if workloads.State != nvidiacomv1beta1.DGDStateFailed && rollingUpdateInProgress(status.RollingUpdate) { return metav1.Condition{ Type: "Ready", Status: metav1.ConditionFalse, diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go index 5dc7c48953b1..69db5a2965c7 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go @@ -341,10 +341,12 @@ func (r *DynamoComponentDeploymentReconciler) reconcileDeploymentResources(ctx c "deploymentAvailableReplicas", deployment.Status.AvailableReplicas, "deploymentReadyReplicas", deployment.Status.ReadyReplicas) + ready := IsDeploymentReady(deployment) serviceReplicaStatus := &nvidiacomv1beta1.ComponentReplicaStatus{ ComponentKind: nvidiacomv1beta1.ComponentKindDeployment, ComponentNames: []string{deployment.Name}, RuntimeNamespace: dynamo.GetDCDRuntimeNamespace(dynamoComponentDeployment), + Ready: ready, Replicas: deployment.Status.Replicas, UpdatedReplicas: deployment.Status.UpdatedReplicas, ReadyReplicas: &deployment.Status.ReadyReplicas, @@ -362,7 +364,7 @@ func (r *DynamoComponentDeploymentReconciler) reconcileDeploymentResources(ctx c } gpuShapeStatus := &gpuShape - if IsDeploymentReady(deployment) { + if ready { return ComponentReconcileResult{ modified: deploymentModified, status: metav1.ConditionTrue, @@ -559,6 +561,7 @@ func getLeaderWorkerSetReplicasStatus(leaderWorkerSet *leaderworkersetv1.LeaderW return nvidiacomv1beta1.ComponentReplicaStatus{ ComponentKind: nvidiacomv1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{leaderWorkerSet.Name}, + Ready: IsLeaderWorkerSetReady(leaderWorkerSet), Replicas: leaderWorkerSet.Status.Replicas, UpdatedReplicas: leaderWorkerSet.Status.UpdatedReplicas, ReadyReplicas: &leaderWorkerSet.Status.ReadyReplicas, @@ -1002,7 +1005,7 @@ func (r *DynamoComponentDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) } m := ctrl.NewControllerManagedBy(mgr). - For(&nvidiacomv1beta1.DynamoComponentDeployment{}, builder.WithPredicates(generationOrDeletionChangedPredicate())). + For(&nvidiacomv1beta1.DynamoComponentDeployment{}, builder.WithPredicates(commonController.GenerationOrDeletionChangedPredicate())). Named(commonconsts.ResourceTypeDynamoComponentDeployment). Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.Funcs{ // ignore creation cause we don't want to be called again after we create the deployment diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go index 31e7dec4744d..a2c8190d64ee 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go @@ -1314,7 +1314,8 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing. Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -1455,7 +1456,8 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing. Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -2211,6 +2213,7 @@ func Test_reconcileLeaderWorkerSetResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{"test-component-0"}, + Ready: true, ReadyReplicas: ptr.To(int32(1)), UpdatedReplicas: 1, Replicas: 1, @@ -2289,6 +2292,7 @@ func Test_reconcileLeaderWorkerSetResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{"test-component-0"}, + Ready: true, ReadyReplicas: ptr.To(int32(3)), UpdatedReplicas: 3, Replicas: 3, @@ -2585,6 +2589,7 @@ func Test_reconcileDeploymentResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindDeployment, ComponentNames: []string{"test-component"}, + Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 6ad1ea0068e7..a8b15675eca6 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -184,7 +184,13 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, err } - // Dispatch exclusively through the persisted provider. + // LPX workloads have their own Grove owner; component-provider rendering cannot realize them. + if dynamoDeployment.HasLPXComponent() && provider != workloadProviderGrove { + programResult := newWorkloadProgramResult(dynamoDeployment) + programResult.Fail(dynamoDeployment.Generation, "LPXRejected", fmt.Errorf("LPX requires the Grove workload provider")) + return ctrl.Result{}, r.persistWorkloadProgramResult(ctx, dynamoDeployment, programResult) + } + program, err := r.selectWorkloadProgram(provider) if err != nil { return ctrl.Result{}, err @@ -236,6 +242,11 @@ func (r *DynamoGraphDeploymentReconciler) persistWorkloadProgramResult( } func (r *DynamoGraphDeploymentReconciler) FinalizeResource(ctx context.Context, dynamoDeployment *nvidiacomv1beta1.DynamoGraphDeployment) error { + // Wait for the LPX child's cleanup before deleting graph-owned checkpoints. + if err := (&dgdLPXHandoff{Client: r.Client}).Finalize(ctx, dynamoDeployment); err != nil { + return err + } + syncer := newDGDResourceSyncer(r.Client, r.Recorder) return newDGDCheckpointsReconciler( syncer, @@ -270,9 +281,10 @@ func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) err ctrlBuilder := ctrl.NewControllerManagedBy(mgr). For(&nvidiacomv1beta1.DynamoGraphDeployment{}, builder.WithPredicates( - generationOrDeletionChangedPredicate(), + dgdPrimaryPredicate(), )). Named(consts.ResourceTypeDynamoGraphDeployment). + Owns(&nvidiacomv1alpha1.LPXGraphDeployment{}). Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(mapDGDWorkerPodToRequests), diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index ca5af50d7f25..426cdf89212d 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -20,6 +20,7 @@ package controller import ( "context" "fmt" + "strings" "testing" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -30,6 +31,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" @@ -46,6 +48,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" @@ -170,11 +173,14 @@ func TestDynamoGraphDeploymentReconcileFinalizesDeletingStoredCheckpointIncompat WithObjects(dgd). WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}). Build() + recorder := events.NewFakeRecorder(10) + config := &configv1alpha1.OperatorConfiguration{} + runtimeConfig := &controller_common.RuntimeConfig{} reconciler := &DynamoGraphDeploymentReconciler{ Client: kubeClient, - Recorder: events.NewFakeRecorder(10), - Config: &configv1alpha1.OperatorConfiguration{}, - RuntimeConfig: &controller_common.RuntimeConfig{}, + Recorder: recorder, + Config: config, + RuntimeConfig: runtimeConfig, } result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ @@ -201,17 +207,20 @@ func TestDynamoGraphDeploymentReconcileFinalizesWithoutSnapshotTypes(t *testing. }} controller_common.AddFinalizer(dgd) testScheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(testScheme)) require.NoError(t, v1beta1.AddToScheme(testScheme)) kubeClient := fake.NewClientBuilder(). WithScheme(testScheme). WithObjects(dgd). WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}). Build() + recorder := events.NewFakeRecorder(10) + runtimeConfig := &controller_common.RuntimeConfig{} reconciler := &DynamoGraphDeploymentReconciler{ Client: kubeClient, - Recorder: events.NewFakeRecorder(10), + Recorder: recorder, Config: &configv1alpha1.OperatorConfiguration{}, - RuntimeConfig: &controller_common.RuntimeConfig{}, + RuntimeConfig: runtimeConfig, } t.Log("Finalize while treating unregistered Snapshot resources as unavailable") @@ -602,6 +611,26 @@ func TestDGDScalingAdaptersReconciler_Reconcile(t *testing.T) { } } +func TestGenerateAdapterName(t *testing.T) { + t.Run("preserves valid existing name format", func(t *testing.T) { + assert.Equal(t, "my-dgd-myservice", generateAdapterName("my-dgd", "MyService")) + + dgdName := strings.Repeat("a", 60) + assert.Equal(t, dgdName+"-runtime", generateAdapterName(dgdName, "runtime")) + }) + + t.Run("hashes overlong names deterministically", func(t *testing.T) { + dgdName := strings.Repeat("a", 250) + require.Empty(t, k8svalidation.IsDNS1123Subdomain(dgdName)) + got := generateAdapterName(dgdName, "runtime") + + assert.Empty(t, k8svalidation.IsDNS1123Subdomain(got)) + assert.LessOrEqual(t, len(got), k8svalidation.DNS1123SubdomainMaxLength) + assert.Equal(t, got, generateAdapterName(dgdName, "runtime")) + assert.NotEqual(t, got, generateAdapterName(dgdName, "another-runtime")) + }) +} + func TestDGDScalingAdaptersReconciler_EmitsDeleteEventOnlyAfterSuccessfulDelete(t *testing.T) { notFound := apierrors.NewNotFound( schema.GroupResource{ @@ -610,6 +639,7 @@ func TestDGDScalingAdaptersReconciler_EmitsDeleteEventOnlyAfterSuccessfulDelete( }, "test-dgd-removed", ) + tests := []struct { name string deleteErr error @@ -701,6 +731,27 @@ func TestDynamoGraphDeploymentReconciler_mapAutoSnapshotJobToDGDRequestsAllowsRe func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { ctx := context.Background() + newPCSGPodClique := func(pcsg, component string, replica int32) *grovev1alpha1.PodClique { + return &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d-%s", pcsg, replica, component), + Namespace: "default", + Labels: map[string]string{ + grovecommon.LabelPodCliqueScalingGroup: pcsg, + grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(replica), + commonconsts.KubeLabelDynamoComponent: component, + }, + }, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + ScheduledReplicas: 1, + ObservedGeneration: ptr.To(int64(1)), + }, + } + } tests := []struct { name string @@ -772,6 +823,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, + Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), @@ -839,6 +891,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, + Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -851,7 +904,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { Replicas: 2, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), - RuntimeNamespace: "default-test-dgd", ScheduledReplicas: ptr.To(int32(2)), }, }, @@ -889,11 +941,12 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { Replicas: 1, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 1, - UpdatedReplicas: 1, - AvailableReplicas: 1, - ScheduledReplicas: 1, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ScheduledReplicas: 1, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, &grovev1alpha1.PodCliqueScalingGroup{ @@ -905,13 +958,16 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { Replicas: 1, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 1, - UpdatedReplicas: 1, - AvailableReplicas: 1, - ScheduledReplicas: 1, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ScheduledReplicas: 1, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, + newPCSGPodClique("test-dgd-0-decode", "decode", 0), + newPCSGPodClique("test-dgd-0-prefill", "prefill", 0), }, wantReconcileResult: ReconcileResult{ State: v1beta1.DGDStateSuccessful, @@ -921,6 +977,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "decode": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-decode"}, + Ready: true, Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: ptr.To(int32(1)), @@ -930,6 +987,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, + Ready: true, Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: ptr.To(int32(1)), @@ -984,13 +1042,16 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 1, // Only 1 available, but 2 desired - ScheduledReplicas: 2, // both scheduled; availability (not scheduling) is the shortfall - ObservedGeneration: ptr.To(int64(1)), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 1, // Only 1 available, but 2 desired + ScheduledReplicas: 2, // both scheduled; availability (not scheduling) is the shortfall + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, + newPCSGPodClique("test-dgd-0-aggregated", "aggregated", 0), + newPCSGPodClique("test-dgd-0-aggregated", "aggregated", 1), }, wantReconcileResult: ReconcileResult{ State: v1beta1.DGDStatePending, @@ -1000,6 +1061,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, + Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -1073,24 +1135,57 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { g.Expect(err).NotTo(gomega.HaveOccurred()) t.Log("Expect workers to withhold their runtime namespace until a PCS revision is accepted") - want := tt.wantReconcileResult - want.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus, len(tt.wantReconcileResult.ComponentStatus)) + for i := range dgd.Spec.Components { + component := &dgd.Spec.Components[i] + if !dynamo.IsWorkerComponent(string(component.ComponentType)) { + continue + } + componentStatus, exists := result.ComponentStatus[component.ComponentName] + g.Expect(exists).To(gomega.BeTrue()) + g.Expect(componentStatus.RuntimeNamespace).To(gomega.BeEmpty()) + } + + pcs := &grovev1alpha1.PodCliqueSet{} + g.Expect(fakeKubeClient.Get(ctx, client.ObjectKey{Name: "test-dgd", Namespace: "default"}, pcs)).To(gomega.Succeed()) + pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) + pcs.Status.CurrentGenerationHash = ptr.To("current") + g.Expect(fakeKubeClient.Update(ctx, pcs)).To(gomega.Succeed()) + + result, err = reconciler.newGroveProgram().workloads.Reconcile( + ctx, + dgd, + nil, + nil, + ) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + t.Log("Expect accepted, completed workers to publish the rendered hash-suffixed namespace") + wantFinal := tt.wantReconcileResult + wantFinal.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus, len(tt.wantReconcileResult.ComponentStatus)) for componentName, componentStatus := range tt.wantReconcileResult.ComponentStatus { componentStatus.GPUsPerEngine = ptr.To(int64(0)) componentStatus.GPUsPerReplica = ptr.To(int64(0)) - want.ComponentStatus[componentName] = componentStatus + wantFinal.ComponentStatus[componentName] = componentStatus } + workerHash, hashErr := dynamo.ComputeDGDWorkersSpecHash(dgd) + g.Expect(hashErr).NotTo(gomega.HaveOccurred()) for i := range dgd.Spec.Components { component := &dgd.Spec.Components[i] if !dynamo.IsWorkerComponent(string(component.ComponentType)) { continue } - componentStatus := want.ComponentStatus[component.ComponentName] - componentStatus.RuntimeNamespace = "" - want.ComponentStatus[component.ComponentName] = componentStatus + componentStatus := wantFinal.ComponentStatus[component.ComponentName] + if componentStatus.RuntimeNamespace == "" { + continue + } + componentStatus.RuntimeNamespace = dynamo.ComponentRuntimeNamespace( + dgd.GetDynamoNamespaceForComponent(component), + string(component.ComponentType), + workerHash, + ) + wantFinal.ComponentStatus[component.ComponentName] = componentStatus } - - g.Expect(result).To(gomega.Equal(want)) + g.Expect(result).To(gomega.Equal(wantFinal)) }) } } @@ -1361,6 +1456,7 @@ func TestPrepareGroveTopologyConstraintUpgrade(t *testing.T) { func TestPreserveGrovePodCliqueSetReplicas(t *testing.T) { g := gomega.NewGomegaWithT(t) + t.Log("Build desired and live replica counts for ordinary and grouped cliques") desired := &grovev1alpha1.PodCliqueSet{ Spec: grovev1alpha1.PodCliqueSetSpec{ Template: grovev1alpha1.PodCliqueSetTemplateSpec{ @@ -1368,11 +1464,18 @@ func TestPreserveGrovePodCliqueSetReplicas(t *testing.T) { {Name: "frontend", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, {Name: "prefill", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, {Name: "new-worker", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 5}}, + {Name: "leader", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, + {Name: "worker", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, + {Name: "grouped", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, + {Name: "router", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, + {Name: "decoder", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}}, }, PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{ {Name: "decode-group", CliqueNames: []string{"decode"}, Replicas: ptr.To(int32(1))}, {Name: "prefill-group", CliqueNames: []string{"prefill"}, Replicas: ptr.To(int32(1))}, {Name: "new-group", Replicas: ptr.To(int32(7))}, + {Name: "compound", CliqueNames: []string{"leader", "worker", "decoder"}, Replicas: ptr.To(int32(1))}, + {Name: "grouped", CliqueNames: []string{"grouped"}, Replicas: ptr.To(int32(1))}, }, }, }, @@ -1383,17 +1486,26 @@ func TestPreserveGrovePodCliqueSetReplicas(t *testing.T) { Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{ {Name: "frontend", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 2}}, {Name: "prefill", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 4}}, + {Name: "leader", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 9}}, + {Name: "worker", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 9}}, + {Name: "grouped", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 4}}, + {Name: "router", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 3}}, + {Name: "decoder", Spec: grovev1alpha1.PodCliqueSpec{Replicas: 9}}, }, PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{ {Name: "decode-group", CliqueNames: []string{"decode"}}, {Name: "prefill-group", CliqueNames: []string{"prefill"}, Replicas: ptr.To(int32(6))}, + {Name: "compound", CliqueNames: []string{"leader", "worker", "decoder"}, Replicas: ptr.To(int32(9))}, + {Name: "grouped", CliqueNames: []string{"grouped"}, Replicas: ptr.To(int32(4))}, }, }, }, } - preserveGrovePodCliqueSetReplicas(desired, existing) + t.Log("Preserve live horizontal scale while keeping grouped role cardinalities") + preserveGrovePodCliqueSetReplicas(desired, existing, nil) + t.Log("Verify existing scale, grouped cardinalities and new-resource defaults") replicasByClique := map[string]int32{} for _, clique := range desired.Spec.Template.Cliques { replicasByClique[clique.Name] = clique.Spec.Replicas @@ -1402,11 +1514,18 @@ func TestPreserveGrovePodCliqueSetReplicas(t *testing.T) { "frontend": 2, "prefill": 1, "new-worker": 5, + "leader": 1, + "worker": 1, + "grouped": 1, + "router": 3, + "decoder": 1, })) g.Expect(desired.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas).To(gomega.BeNil()) g.Expect(desired.Spec.Template.PodCliqueScalingGroupConfigs[1].Replicas).NotTo(gomega.BeNil()) g.Expect(*desired.Spec.Template.PodCliqueScalingGroupConfigs[1].Replicas).To(gomega.Equal(int32(6))) g.Expect(*desired.Spec.Template.PodCliqueScalingGroupConfigs[2].Replicas).To(gomega.Equal(int32(7))) + g.Expect(*desired.Spec.Template.PodCliqueScalingGroupConfigs[3].Replicas).To(gomega.Equal(int32(9))) + g.Expect(*desired.Spec.Template.PodCliqueScalingGroupConfigs[4].Replicas).To(gomega.Equal(int32(4))) } func TestPreserveGrovePodCliqueSetReplicasSkipsCheckpointGatedComponents(t *testing.T) { @@ -2039,6 +2158,63 @@ func TestDGDRestartReconciler_ComputeStatus(t *testing.T) { Phase: v1alpha1.RestartPhaseCompleted, }, }, + { + name: "Grove pathway - ready child remains in progress until PCS observes restart generation", + dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ + Restart: &v1alpha1.Restart{ + ID: newID, + Strategy: &v1alpha1.RestartStrategy{ + Type: v1alpha1.RestartStrategyTypeParallel, + }, + }, + Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ + "frontend": { + Replicas: ptr.To(int32(1)), + }, + }, + }, + dgdStatus: v1alpha1.DynamoGraphDeploymentStatus{ + Restart: &v1alpha1.RestartStatus{ + ObservedID: newID, + Phase: v1alpha1.RestartPhaseRestarting, + InProgress: []string{"frontend"}, + }, + }, + existingResources: []client.Object{ + &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dgd", + Namespace: "default", + Generation: 2, + }, + Status: grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(int64(1)), + }, + }, + &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dgd-0-frontend", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueSpec{ + Replicas: 1, + }, + Status: grovev1alpha1.PodCliqueStatus{ + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + }, + groveEnabled: true, + wantRestartStatus: &v1alpha1.RestartStatus{ + ObservedID: newID, + Phase: v1alpha1.RestartPhaseRestarting, + InProgress: []string{"frontend"}, + }, + }, { name: "Grove pathway - sequential restart in progress", dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ @@ -2087,6 +2263,51 @@ func TestDGDRestartReconciler_ComputeStatus(t *testing.T) { InProgress: []string{"frontend"}, }, }, + { + name: "Grove pathway - removed in-progress component resets sequential restart", + dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ + Restart: &v1alpha1.Restart{ + ID: newID, + Strategy: &v1alpha1.RestartStrategy{ + Type: v1alpha1.RestartStrategyTypeSequential, + Order: []string{"frontend", "decode"}, + }, + }, + Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ + "frontend": { + Replicas: ptr.To(int32(1)), + }, + "decode": { + Replicas: ptr.To(int32(1)), + }, + }, + }, + dgdStatus: v1alpha1.DynamoGraphDeploymentStatus{ + Restart: &v1alpha1.RestartStatus{ + ObservedID: newID, + Phase: v1alpha1.RestartPhaseRestarting, + InProgress: []string{"removed"}, + }, + }, + existingResources: []client.Object{ + &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dgd", + Namespace: "default", + Generation: 1, + }, + Status: grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(int64(1)), + }, + }, + }, + groveEnabled: true, + wantRestartStatus: &v1alpha1.RestartStatus{ + ObservedID: newID, + Phase: v1alpha1.RestartPhaseRestarting, + InProgress: []string{"frontend"}, + }, + }, { name: "parallel restart - new restart request during ongoing restart resets to all services (DCD pathway)", dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ @@ -3514,6 +3735,7 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { ScheduleGatedReplicas: 0, ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To("previous-revision"), + CurrentPodTemplateHash: ptr.To("previous-template"), }, } } @@ -3572,6 +3794,13 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { }, want: true, }, + { + name: "LPX-only Pod template hash change is filtered", + mutate: func(pc *grovev1alpha1.PodClique) { + pc.Status.CurrentPodTemplateHash = ptr.To("target-template") + }, + want: false, + }, { name: "update completion change is significant", mutate: func(pc *grovev1alpha1.PodClique) { @@ -3604,7 +3833,7 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { oldPC := base() newPC := base() tt.mutate(newPC) - assert.Equal(t, tt.want, podCliqueStatusChangeIsSignificant(oldPC, newPC)) + assert.Equal(t, tt.want, controller_common.PodCliqueStatusChangeIsSignificant(oldPC, newPC)) }) } @@ -3617,7 +3846,7 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { }} newPodClique := oldPodClique.DeepCopy() newPodClique.Status.Conditions[0].Message = "two nodes unavailable" - assert.False(t, podCliqueStatusChangeIsSignificant(oldPodClique, newPodClique)) + assert.False(t, controller_common.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique)) } func TestPCSGStatusChangeIsSignificant(t *testing.T) { @@ -3714,7 +3943,7 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { oldPCSG := base() newPCSG := base() tt.mutate(newPCSG) - assert.Equal(t, tt.want, pcsgStatusChangeIsSignificant(oldPCSG, newPCSG)) + assert.Equal(t, tt.want, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldPCSG, newPCSG)) }) } @@ -3727,7 +3956,7 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { }} newScalingGroup := oldScalingGroup.DeepCopy() newScalingGroup.Status.Conditions[0].Message = "two replicas unavailable" - assert.False(t, pcsgStatusChangeIsSignificant(oldScalingGroup, newScalingGroup)) + assert.False(t, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup)) } func TestGroveChildEventPredicates(t *testing.T) { @@ -3736,10 +3965,4 @@ func TestGroveChildEventPredicates(t *testing.T) { assert.False(t, podCliquePredicates.Create(event.CreateEvent{Object: podClique})) assert.False(t, podCliquePredicates.Delete(event.DeleteEvent{Object: podClique})) assert.False(t, podCliquePredicates.Generic(event.GenericEvent{Object: podClique})) - - scalingGroup := &grovev1alpha1.PodCliqueScalingGroup{} - scalingGroupPredicates := pcsgEventPredicates() - assert.False(t, scalingGroupPredicates.Create(event.CreateEvent{Object: scalingGroup})) - assert.False(t, scalingGroupPredicates.Delete(event.DeleteEvent{Object: scalingGroup})) - assert.False(t, scalingGroupPredicates.Generic(event.GenericEvent{Object: scalingGroup})) } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index d9f0d01e2767..a087bf613926 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1325,8 +1325,10 @@ func (r *DynamoGraphDeploymentRequestReconciler) validateGPUHardwareInfo(ctx con // // If the error does not match any known category, "unknown" is returned. func GetGPUDiscoveryFailureReason(err error) string { + const unknownReason = "unknown" + if err == nil { - return "unknown" + return unknownReason } errMsg := strings.ToLower(err.Error()) @@ -1357,7 +1359,7 @@ func GetGPUDiscoveryFailureReason(err error) string { case strings.Contains(errMsg, "failed to create helm path"): return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" } - return "unknown" + return unknownReason } // createProfilingJob creates a Kubernetes Job for profiling using SyncResource. @@ -2271,6 +2273,8 @@ func updateDeploymentInfo(dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, d totalReplicas += component.Replicas if component.AvailableReplicas != nil { totalAvailable += *component.AvailableReplicas + } else if component.ReadyReplicas != nil { + totalAvailable += *component.ReadyReplicas } } diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go new file mode 100644 index 000000000000..6d29614d7d9d --- /dev/null +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -0,0 +1,460 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" +) + +const ( + lpxAttemptPreparedReason string = "LPXAttemptPrepared" + lpxSchedulingAttemptActiveReason string = "LPXSchedulingAttemptActive" + lpxSchedulingDispositionObservedReason string = "LPXSchedulingDispositionObserved" + lpxSchedulingDeadlineExceededReason string = "LPXSchedulingDeadlineExceeded" + lpxAttemptAuthorityLostReason string = "LPXAttemptAuthorityLost" + lpxAttemptRecordingFinalizer = "scheduling.lpu.nvidia.com/dynamo-attempt-recording" +) + +type lpxDeadlineTransition struct { + result reconcileOutcome + attempt *nvidiacomv1beta1.LPXAttemptStatus + requeueAfter time.Duration +} + +func (*lpxDeadlineTransition) lpxClassification() {} + +func newLPXDeadlineTransition(attempt *nvidiacomv1beta1.LPXAttemptStatus, state nvidiacomv1beta1.DGDState, reason string, message string, requeueAfter time.Duration) *lpxDeadlineTransition { + return &lpxDeadlineTransition{ + result: reconcileOutcome{State: state, Reason: reason, Message: message}, + attempt: attempt, + requeueAfter: requeueAfter, + } +} + +// completeLPXDeadline preserves successful deadline wakes and converts active-deadline errors into bounded retries. +func completeLPXDeadline( + ctx context.Context, + deadlineAt time.Time, + result ctrl.Result, + reconcileErr error, +) (ctrl.Result, error) { + // Preserve ordinary results when no scheduling deadline is active. + if deadlineAt.IsZero() { + return result, reconcileErr + } + + // Record failures and bound completion by the remaining scheduling deadline. + if reconcileErr != nil { + log.FromContext(ctx).Error(reconcileErr, "retrying reconciliation under the LPX scheduling deadline") + } + delay := time.Until(deadlineAt) + if reconcileErr != nil { + if delay <= 0 || delay > lpxRetirementRequeueAfter { + delay = lpxRetirementRequeueAfter + } + return ctrl.Result{RequeueAfter: delay}, nil + } + if delay <= 0 { + delay = time.Nanosecond + } + if result.RequeueAfter <= 0 || delay < result.RequeueAfter { + result.RequeueAfter = delay + } + return result, nil +} + +func (r *graphReconciler) reconcileLPXAttemptDeadline( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, +) (lpxClassification, time.Time, []lpxv1alpha1.LPUPipelineRequest, error) { + // Failed reconciliations can retain a current durable attempt before the + // complete result is observed. Invalidate only the attempt's own generation, + // including when the new source no longer selects a scheduling deadline. + attempt := currentLPXAttemptStatus(deployment) + if attempt != nil && attempt.ObservedGeneration != deployment.Generation { + deployment.Status.Placement = nil + attempt = nil + } + deadlineSeconds := lpxAttemptDeadlineSeconds(source) + if !source.HasLPXComponent() || deadlineSeconds == nil { + return nil, time.Time{}, nil, nil + } + if attempt == nil { + return nil, time.Time{}, nil, nil + } + deadlineAt := time.Time{} + if attempt.DeadlineAt != nil { + deadlineAt = attempt.DeadlineAt.Time + } + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil { + return nil, deadlineAt, nil, err + } + observed := lpxRequestsByName(requests) + if updateLPXAttemptFromLive(deployment, deadlineSeconds, attempt, observed) { + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, + "Persisted the exact LPX request identity and immutable scheduling deadline", time.Nanosecond, + ), attempt.DeadlineAt.Time, nil, nil + } + if attempt.ExceededAt != nil { + state, err := r.continueTerminalLPXAttempt( + ctx, deployment, attempt, observed, lpxSchedulingDeadlineExceededReason, + "LPX scheduling deadline exceeded; no retry is authorized", + ) + return state, time.Time{}, nil, err + } + active, disposition, missing := exactLPXAttemptState(attempt, observed) + if attempt.DisarmedAt != nil && !missing && !active { + return nil, time.Time{}, requests, nil + } + now := time.Now() + if attempt.DeadlineAt != nil && (attempt.DisarmedAt == nil || !missing) && !now.Before(attempt.DeadlineAt.Time) { + decision := metav1.NewTime(now) + attempt.ExceededAt = &decision + attempt.DisarmedAt = nil + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStateFailed, lpxSchedulingDeadlineExceededReason, + "LPX scheduling deadline exceeded; persisting expiry before exact cleanup", time.Nanosecond, + ), attempt.DeadlineAt.Time, nil, nil + } + if missing { + state, err := r.continueTerminalLPXAttempt( + ctx, deployment, attempt, observed, lpxAttemptAuthorityLostReason, + "An exact LPX request disappeared; retiring the aggregate without replacement", + ) + return state, time.Time{}, nil, err + } + if attempt.DeadlineAt == nil { + return nil, time.Time{}, requests, nil + } + if disposition { + observedAt := metav1.NewTime(now) + attempt.DisarmedAt = &observedAt + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingDispositionObservedReason, + "Recorded the exact aggregate scheduling disposition before deadline expiry", time.Nanosecond, + ), attempt.DeadlineAt.Time, nil, nil + } + if attempt.DisarmedAt != nil { + attempt.DisarmedAt = nil + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, + "LPX scheduling resumed under the original deadline", time.Nanosecond, + ), attempt.DeadlineAt.Time, nil, nil + } + return nil, deadlineAt, requests, nil +} + +func (r *graphReconciler) reconcileLPXAttemptPreparation( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + deadlineSeconds *int64, + want *nvidiacomv1beta1.LPXAttemptStatus, + currents map[string]*lpxv1alpha1.LPUPipelineRequest, +) (*lpxDeadlineTransition, error) { + if want == nil { + return nil, nil + } + current := currentLPXAttemptStatus(deployment) + if current == nil || current.ObservedGeneration != deployment.Generation { + updateLPXAttemptFromLive(deployment, deadlineSeconds, want, currents) + reason, message := lpxAttemptPreparedReason, "Prepared the complete LPX request set before publication" + if want.DeadlineAt != nil { + reason, message = lpxSchedulingAttemptActiveReason, "Recovered the exact first publication and its deadline" + } + return newLPXDeadlineTransition( + want, nvidiacomv1beta1.DGDStatePending, reason, message, time.Nanosecond, + ), nil + } + if !sameLPXAttemptProjection(current, want) { + return nil, fmt.Errorf("current LPX attempt status does not match the desired request projection") + } + if current.PodCliqueSetUID != want.PodCliqueSetUID { + return nil, fmt.Errorf("current LPX attempt status does not match the exact Grove identity") + } + if _, _, missing := exactLPXAttemptState(current, currents); missing { + return newLPXDeadlineTransition( + current, nvidiacomv1beta1.DGDStateFailed, lpxAttemptAuthorityLostReason, + "An exact LPX request disappeared; publication remains fenced", time.Nanosecond, + ), nil + } + if !updateLPXAttemptFromLive(deployment, deadlineSeconds, current, currents) { + return nil, nil + } + return newLPXDeadlineTransition( + current, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, + "Persisted the exact LPX request identity and immutable scheduling deadline", time.Nanosecond, + ), nil +} + +// revalidateLPXAttemptPublication fences live authority after preparation has +// established the current nonnil attempt and an unpublished desired request. +func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment) error { + attempt := currentLPXAttemptStatus(deployment) + if attempt.ExceededAt != nil || attempt.DisarmedAt != nil { + return fmt.Errorf("LPX publication is forbidden after a durable attempt decision") + } + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil { + return err + } + if _, _, missing := exactLPXAttemptState(attempt, lpxRequestsByName(requests)); missing { + return fmt.Errorf("recorded LPX request authority changed before sibling publication") + } + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs); err != nil { + return err + } + if pcs.UID != attempt.PodCliqueSetUID || !pcs.DeletionTimestamp.IsZero() { + return fmt.Errorf("PodCliqueSet authority changed before LPX publication") + } + if attempt.DeadlineAt != nil && !time.Now().Before(attempt.DeadlineAt.Time) { + return fmt.Errorf("LPX scheduling deadline elapsed before sibling publication") + } + return nil +} + +func lpxAttemptDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) *int64 { + if source.Spec.Scheduling == nil { + return nil + } + return source.Spec.Scheduling.AttemptDeadlineSeconds +} + +// desiredLPXAttemptStatus projects desired for its reconcile-local LPXGraphDeployment generation; desired must be nonnil. +func desiredLPXAttemptStatus(desired *lpxMaterializing, observedGeneration int64, pcsUID types.UID) *nvidiacomv1beta1.LPXAttemptStatus { + requests := make([]nvidiacomv1beta1.LPXAttemptRequestStatus, len(desired.requests)) + for i := range desired.requests { + requests[i] = nvidiacomv1beta1.LPXAttemptRequestStatus{ + Name: desired.requests[i].requestName, AttemptDigest: desired.requests[i].attemptDigest, + } + } + sort.Slice(requests, func(i, j int) bool { return requests[i].Name < requests[j].Name }) + return &nvidiacomv1beta1.LPXAttemptStatus{ + ObservedGeneration: observedGeneration, PodCliqueSetUID: pcsUID, Requests: requests, + } +} + +func currentLPXAttemptStatus(deployment *nvidiacomv1alpha1.LPXGraphDeployment) *nvidiacomv1beta1.LPXAttemptStatus { + if deployment == nil || deployment.Status.Placement == nil { + return nil + } + return deployment.Status.Placement.LPXAttempt +} + +func sameLPXAttemptProjection(a, b *nvidiacomv1beta1.LPXAttemptStatus) bool { + if a == nil || b == nil || a.ObservedGeneration != b.ObservedGeneration || len(a.Requests) != len(b.Requests) { + return false + } + for i := range a.Requests { + if a.Requests[i].Name != b.Requests[i].Name || + a.Requests[i].AttemptDigest != b.Requests[i].AttemptDigest { + return false + } + } + return true +} + +// updateLPXAttemptFromLive incorporates server-persisted requests into the non-nil, +// reconcile-owned attempt and reports whether a status transition is needed. +func updateLPXAttemptFromLive( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + deadlineSeconds *int64, + attempt *nvidiacomv1beta1.LPXAttemptStatus, + observed map[string]*lpxv1alpha1.LPUPipelineRequest, +) bool { + changed := false + var firstPublishedAt time.Time + for i := range attempt.Requests { + row := &attempt.Requests[i] + request := observed[row.Name] + if request == nil { + continue + } + if row.UID == "" { + if request.Annotations[lpxAttemptDigestAnnotation] != row.AttemptDigest || + request.Annotations[lpxDeploymentGenerationAnnotation] != strconv.FormatInt(deployment.Generation, 10) { + continue + } + changed = true + row.UID = request.UID + } else if request.UID != row.UID { + continue + } + if attempt.PodCliqueSetUID == "" { + changed = true + attempt.PodCliqueSetUID = types.UID(request.Annotations[lpxPCSUIDAnnotation]) + } + if attempt.DeadlineAt == nil && + (firstPublishedAt.IsZero() || request.CreationTimestamp.Time.Before(firstPublishedAt)) { + firstPublishedAt = request.CreationTimestamp.Time + } + } + if attempt.DeadlineAt == nil && !firstPublishedAt.IsZero() { + deadline := metav1.NewTime(firstPublishedAt.Add( + time.Duration(*deadlineSeconds) * time.Second, + )) + changed = true + attempt.DeadlineAt = &deadline + } + return changed +} + +func lpxRequestsByName(requests []lpxv1alpha1.LPUPipelineRequest) map[string]*lpxv1alpha1.LPUPipelineRequest { + byName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(requests)) + for i := range requests { + byName[requests[i].Name] = &requests[i] + } + return byName +} + +func (r *graphReconciler) releaseLPXAttemptRecordingFinalizer(ctx context.Context, request *lpxv1alpha1.LPUPipelineRequest) error { + if request.DeletionTimestamp.IsZero() || len(request.Finalizers) != 1 || request.Finalizers[0] != lpxAttemptRecordingFinalizer { + return nil + } + controllerutil.RemoveFinalizer(request, lpxAttemptRecordingFinalizer) + if err := r.Update(ctx, request); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("release LPX attempt recording finalizer on %q: %w", request.Name, err) + } + return nil +} + +// exactLPXAttemptState reports aggregate request state; attempt must be nonnil. +func exactLPXAttemptState( + attempt *nvidiacomv1beta1.LPXAttemptStatus, + observed map[string]*lpxv1alpha1.LPUPipelineRequest, +) (active bool, disposition bool, missing bool) { + disposition = true + for i := range attempt.Requests { + // Distinguish unpublished rows from recorded request authority loss. + row := &attempt.Requests[i] + request := observed[row.Name] + if row.UID == "" { + disposition = false + continue + } + if request == nil || request.UID != row.UID || !request.DeletionTimestamp.IsZero() { + disposition = false + missing = true + continue + } + + // Reuse the canonical publication state after exact row ownership is proven. + switch state := classifyPublishedLPX(request).(type) { + case *lpxBound: + continue + case *lpxSchedulerObserved: + switch state.Phase { + case lpxv1alpha1.RequestPhasePending, lpxv1alpha1.RequestPhasePlanned, + lpxv1alpha1.RequestPhaseReserving, lpxv1alpha1.RequestPhaseBinding: + active = true + case lpxv1alpha1.RequestPhaseNoFit, lpxv1alpha1.RequestPhaseUnsupported: + continue + } + } + disposition = false + } + return active, disposition, missing +} + +func (r *graphReconciler) continueTerminalLPXAttempt( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + attempt *nvidiacomv1beta1.LPXAttemptStatus, + observed map[string]*lpxv1alpha1.LPUPipelineRequest, + reason string, + message string, +) (lpxClassification, error) { + // Retire recorded identities in their canonical request order. + pending := false + for _, row := range attempt.Requests { + request := observed[row.Name] + if row.UID == "" || request == nil || request.UID != row.UID { + continue + } + pending = true + if !request.DeletionTimestamp.IsZero() { + if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { + return nil, err + } + continue + } + uid, resourceVersion := request.UID, request.ResourceVersion + if err := r.Delete(ctx, request, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ + UID: &uid, ResourceVersion: &resourceVersion, + }}); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("delete terminal LPX request %q: %w", request.Name, err) + } + } + if pending { + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, attempt.PodCliqueSetUID); err != nil { + return nil, err + } + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, lpxRetirementRequeueAfter, + ), nil + } + pending, err := r.retireLPXAttemptPodCliqueSet(ctx, deployment, attempt.PodCliqueSetUID) + if err != nil { + return nil, err + } + requeueAfter := time.Duration(0) + if pending { + requeueAfter = lpxRetirementRequeueAfter + } + return newLPXDeadlineTransition( + attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, requeueAfter, + ), nil +} + +func (r *graphReconciler) retireLPXAttemptPodCliqueSet( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + uid types.UID, +) (bool, error) { + pcs := &grovev1alpha1.PodCliqueSet{} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]} + if err := r.apiReader.Get(ctx, key, pcs); err != nil { + return false, client.IgnoreNotFound(err) + } + if uid == "" || pcs.UID != uid { + return false, nil + } + if !metav1.IsControlledBy(pcs, deployment) { + return false, fmt.Errorf("refusing to retire PodCliqueSet %q without the exact LPXGraphDeployment owner", pcs.Name) + } + if pcs.Spec.Replicas != 0 { + pcs.Spec.Replicas = 0 + return true, r.Update(ctx, pcs) + } + if !pcs.DeletionTimestamp.IsZero() { + return true, nil + } + pcsUID, resourceVersion := pcs.UID, pcs.ResourceVersion + err := r.Delete(ctx, pcs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ + UID: &pcsUID, ResourceVersion: &resourceVersion, + }}) + return true, client.IgnoreNotFound(err) +} diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go new file mode 100644 index 000000000000..696a2cd778c8 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -0,0 +1,303 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "testing" + "time" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" +) + +func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + reconciler := newLPXTestReconciler(t, registry, dgd, source) + desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) + require.Nil(t, rejected) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.Equal(t, lpxAttemptPreparedReason, lpxResult(classification).Reason) + prepared := applyLPXAttemptTransition(t, dgd, classification) + require.NotEmpty(t, prepared.PodCliqueSetUID) + require.Empty(t, ownedLPXRequests(t, ctx, reconciler, dgd)) + + _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + requests := ownedLPXRequests(t, ctx, reconciler, dgd) + require.Len(t, requests, 1) + require.Contains(t, requests[0].Finalizers, lpxAttemptRecordingFinalizer) + dgd.Status.Placement = nil + _, classification, err = reconciler.reconcileSelectedLPXSafetyPreflight(ctx, dgd, source) + require.NoError(t, err) + require.NotNil(t, classification, "an existing complete LPR set must recover through preflight") + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: prepared} + + require.NoError(t, reconciler.Delete(ctx, &requests[0])) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) + require.NoError(t, err) + armed := applyLPXAttemptTransition(t, dgd, classification) + require.Equal(t, requests[0].UID, armed.Requests[0].UID) + require.True(t, armed.DeadlineAt.Time.Equal(requests[0].CreationTimestamp.Add(30*time.Second))) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) + require.NoError(t, err) + require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) +} + +func TestLPXPreflightReusesActiveDeadlineRequestObservation(t *testing.T) { + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + dgd := newLPXTestDeployment(t, source) + source.Spec.Scheduling = deadlineTestScheduling() + deadline := metav1.NewTime(time.Now().Add(time.Minute)) + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: &nvidiacomv1beta1.LPXAttemptStatus{ + ObservedGeneration: dgd.Generation, + DeadlineAt: &deadline, + Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ + Name: "unpublished", AttemptDigest: "sha256:attempt", + }}, + }} + reconciler := newLPXTestReconciler(t, nil, dgd, source) + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + requestLists := 0 + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + List: func( + ctx context.Context, + delegated client.WithWatch, + list client.ObjectList, + opts ...client.ListOption, + ) error { + if _, ok := list.(*lpxv1alpha1.LPUPipelineRequestList); ok { + requestLists++ + } + return delegated.List(ctx, list, opts...) + }, + }) + + t.Log("Reuse the active deadline request observation during safety preflight") + classification, wake, requests, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + require.Nil(t, classification) + selected, classification, err := reconciler.reconcileLPXSafetyPreflight(t.Context(), dgd, source, requests) + require.NoError(t, err) + require.Nil(t, selected) + require.Nil(t, classification) + require.True(t, wake.Equal(deadline.Time)) + require.Equal(t, 1, requestLists) +} + +func TestLPXAttemptPartialPublicationLossFencesEverySibling(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXSpecDecodeTestDGD(t) + source.Spec.Scheduling = deadlineTestScheduling() + reconciler := newLPXTestReconciler(t, registry, dgd, source) + desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) + require.Nil(t, rejected) + require.Len(t, desired.requests, 3) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + + for range 3 { + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + applyLPXAttemptTransition(t, dgd, classification) + } + requests := ownedLPXRequests(t, ctx, reconciler, dgd) + require.Len(t, requests, 2, "preparation plus one persisted transition per successful create") + + dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Second))) + require.ErrorContains(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd), "deadline") + dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Minute))) + require.NoError(t, reconciler.Delete(ctx, &requests[0])) + require.Error(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd)) + classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) + require.NoError(t, err) + require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) + require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) + applyLPXAttemptTransition(t, dgd, classification) + + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) + require.NoError(t, err) + require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) + _, _ = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.LessOrEqual(t, len(ownedLPXRequests(t, ctx, reconciler, dgd)), 1) +} + +func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { + t.Log("Seed an exact Bound request whose deadline has not yet been recovered") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + dgd := newLPXTestDeployment(t, source) + source.Spec.Scheduling = deadlineTestScheduling() + request := deadlineTestRequest(dgd, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) + request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} + pcs := deadlineTestPCS(dgd, "pcs-uid") + attempt := deadlineTestAttempt(dgd, request, time.Now().Add(time.Minute)) + attempt.DeadlineAt, attempt.PodCliqueSetUID = nil, "" + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} + reconciler := newLPXTestReconciler(t, nil, dgd, source, request, pcs) + + t.Log("Recover the native deadline and record the Bound scheduling disposition") + classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + recovered := applyLPXAttemptTransition(t, dgd, classification) + require.True(t, recovered.DeadlineAt.Time.Equal(request.CreationTimestamp.Add(30*time.Second))) + require.Equal(t, types.UID(request.Annotations[lpxPCSUIDAnnotation]), recovered.PodCliqueSetUID) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + disarmed := applyLPXAttemptTransition(t, dgd, classification) + require.NotNil(t, disarmed.DisarmedAt) + + t.Log("A durably recorded disposition survives the elapsed deadline") + disarmed.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + require.Nil(t, classification) + + t.Log("Resumed scheduling expires under the original elapsed deadline") + request.Status.Phase = lpxv1alpha1.RequestPhasePending + reconciler = newLPXTestReconciler(t, nil, dgd, source, request, pcs) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + result := lpxResult(classification) + require.Equal(t, lpxSchedulingDeadlineExceededReason, result.Reason) + transition := classification.(*lpxDeadlineTransition) + require.NotNil(t, transition.attempt.ExceededAt) + require.Nil(t, transition.attempt.DisarmedAt) + + t.Log("Persist expiry before cleaning up the exact request and PCS") + applyLPXAttemptTransition(t, dgd, classification) + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) + + t.Log("Start exact request deletion before holding the shared PCS at zero") + _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + deleting := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name) + require.False(t, deleting.DeletionTimestamp.IsZero()) + storedPCS := &grovev1alpha1.PodCliqueSet{} + pcsKey := client.ObjectKeyFromObject(pcs) + require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) + require.Zero(t, storedPCS.Spec.Replicas) + + t.Log("Release the recording finalizer only after scheduler cleanup finishes") + deleting.Finalizers = []string{lpxAttemptRecordingFinalizer} + require.NoError(t, reconciler.Update(t.Context(), deleting)) + _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(reconciler.Get(t.Context(), pcsKey, storedPCS))) + require.Zero(t, classification.(*lpxDeadlineTransition).requeueAfter) + + t.Log("A completed old attempt cannot touch same-name replacement identities") + replacement := request.DeepCopy() + replacement.UID = "replacement-uid" + replacement.ResourceVersion = "2" + replacement.Annotations[lpxPCSUIDAnnotation] = "replacement-pcs-uid" + pcs = deadlineTestPCS(dgd, "replacement-pcs-uid") + reconciler = newLPXTestReconciler(t, nil, dgd, source, replacement, pcs) + + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + require.Equal(t, lpxSchedulingDeadlineExceededReason, lpxResult(classification).Reason) + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, replacement.Name).DeletionTimestamp.IsZero()) + require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, int32(1), storedPCS.Spec.Replicas) + + t.Log("A late Bound observation without a durable disposition still expires") + late := deadlineTestAttempt(dgd, request, time.Now().Add(-time.Minute)) + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: late} + request.Status.Phase = lpxv1alpha1.RequestPhaseBound + reconciler = newLPXTestReconciler(t, nil, dgd, source, request) + classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + require.Equal(t, lpxSchedulingDeadlineExceededReason, lpxResult(classification).Reason) +} + +func applyLPXAttemptTransition(t *testing.T, dgd *nvidiacomv1alpha1.LPXGraphDeployment, classification lpxClassification) *nvidiacomv1beta1.LPXAttemptStatus { + t.Helper() + transition := classification.(*lpxDeadlineTransition) + require.NotNil(t, transition.attempt) + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: transition.attempt.DeepCopy()} + return dgd.Status.Placement.LPXAttempt +} + +func deadlineTestAttempt(dgd *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest, deadline time.Time) *nvidiacomv1beta1.LPXAttemptStatus { + return &nvidiacomv1beta1.LPXAttemptStatus{ + ObservedGeneration: dgd.Generation, + PodCliqueSetUID: types.UID(request.Annotations[lpxPCSUIDAnnotation]), + DeadlineAt: ptr.To(metav1.NewTime(deadline)), + Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ + Name: request.Name, AttemptDigest: request.Annotations[lpxAttemptDigestAnnotation], UID: request.UID, + }}, + } +} + +func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { + generation := int64(1) + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: dgd.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", + Generation: generation, CreationTimestamp: metav1.NewTime(created), + Annotations: map[string]string{ + lpxAttemptDigestAnnotation: "sha256:attempt", lpxDeploymentUIDAnnotation: string(dgd.UID), + lpxDeploymentGenerationAnnotation: "1", lpxPCSUIDAnnotation: "pcs-uid", + }, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + }, + Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{Phase: phase, ObservedGeneration: &generation}, + } + if phase == lpxv1alpha1.RequestPhaseBound { + request.Status.LastPlanRevision = 1 + request.Status.Committed = &lpxv1alpha1.Committed{ + Execution: lpxv1alpha1.CommittedExecution{AcceptedGeneration: &generation, NodeLocal: &lpxv1alpha1.NodeLocalExecution{}}, + Plan: lpxv1alpha1.CommittedPlan{ + PlannedFromGeneration: generation, Revision: 1, PlanDigest: "sha256:plan", + Placement: lpxv1alpha1.PlanPlacement{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, NodeLocal: &lpxv1alpha1.NodeLocalPlacement{}}, + }, + } + } + return request +} + +func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { + return &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation], Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + }, + Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, + } +} + +func deadlineTestScheduling() *nvidiacomv1beta1.SchedulingSpec { + return &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} +} + +func ownedLPXRequests(t *testing.T, ctx context.Context, reconciler *graphReconciler, dgd *nvidiacomv1alpha1.LPXGraphDeployment) []lpxv1alpha1.LPUPipelineRequest { + t.Helper() + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + return requests +} diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go new file mode 100644 index 000000000000..0ecd19d6f199 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "testing" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestLPXRoleDependenciesIncludeDraftAndConductorFallback(t *testing.T) { + t.Log("Give both components consumed Agent claims while the target conductor uses its Agent fallback") + source := newLPXSpecDecodeTestSource() + for _, component := range lpx.Components(source) { + pod := &component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec + pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(component.ComponentName + "-gpu")}} + pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} + } + require.Nil(t, lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate) + before := source.DeepCopy() + require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu"}, lpxDRAClaimReferences(true)(source)) + require.Equal(t, before, source) + + t.Log("An independent conductor template contributes its own consumed dependency") + conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader) + conductor.PodTemplate = lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.DeepCopy() + conductor.PodTemplate.Spec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("conductor-gpu") + require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu", "conductor-gpu"}, lpxDRAClaimReferences(true)(source)) +} + +func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { + t.Log("Materialize two draft instances and one target in their single shared scaling group") + deployment, source, registry := newLPXSpecDecodeTestDGD(t) + prepare, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), deployment, source) + objects := lpxMaterializedObjects(t, prepare, deployment, source, selected) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + } + } + pcs := findLPXTestPodCliqueSet(t, objects) + + t.Log("Observe each draft condition against an independent copy of the ready baseline") + for _, scenario := range []string{"ready", "partial", "missing", "foreign", "old revision", "unobserved"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Seed isolated Grove observations and fetch the first draft") + kubeClient := fake.NewClientBuilder().WithScheme(prepare.Scheme()).Build() + createLPXTestObjects(t, t.Context(), kubeClient, objects...) + firstDraft := findLPXTestClique(t, objects, selected.plan.Agents[0].CliqueName).DeepCopy() + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(firstDraft), firstDraft)) + require.Equal(t, "draft", firstDraft.Labels[lpx.StageLabel]) + + t.Log("Make only the first draft incomplete while retaining a fully ready second draft and target") + switch scenario { + case "partial": + firstDraft.Status.ReadyReplicas-- + case "foreign": + firstDraft.OwnerReferences = []metav1.OwnerReference{{UID: "foreign", Controller: ptr.To(true)}} + case "old revision": + firstDraft.Annotations[dynamo.LPXInputRevisionAnnotation] = "old" + case "unobserved": + firstDraft.Status.ObservedGeneration = ptr.To(firstDraft.Generation - 1) + } + if scenario == "missing" { + require.NoError(t, kubeClient.Delete(t.Context(), firstDraft)) + } else { + require.NoError(t, kubeClient.Update(t.Context(), firstDraft)) + } + + t.Log("Project logical draft counts and make both authored components await the complete pair") + readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, source, deployment, pcs) + require.NoError(t, err) + require.Len(t, readiness.ComponentStatuses, 2) + draft, target := readiness.ComponentStatuses["draft"], readiness.ComponentStatuses["lpx"] + require.Equal(t, v1beta1.ComponentKindPodClique, draft.ComponentKind) + require.Len(t, draft.ComponentNames, 2) + require.Equal(t, int32(1), target.Replicas) + if scenario == "ready" { + require.True(t, readiness.Ready) + require.Equal(t, int32(2), draft.Replicas) + require.Equal(t, int32(2), draft.UpdatedReplicas) + require.Equal(t, ptr.To(int32(2)), draft.ReadyReplicas) + require.Equal(t, ptr.To(int32(2)), draft.ScheduledReplicas) + require.Equal(t, ptr.To(int32(1)), target.AvailableReplicas) + } else { + require.False(t, readiness.Ready) + require.Equal(t, ptr.To(int32(1)), draft.ReadyReplicas) + require.Equal(t, ptr.To(int32(0)), target.AvailableReplicas) + } + require.Equal(t, readiness.Ready, draft.Ready) + require.Equal(t, readiness.Ready, target.Ready) + }) + } +} diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go new file mode 100644 index 000000000000..8629820d9ee4 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -0,0 +1,424 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "fmt" + "maps" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const lpxGraphDeploymentFinalizer = "nvidia.com/lpx-graph-deployment" + +// graphReconciler owns the complete compiled engine lifecycle. +// The source DGD supplies intent, never an alternate status or owner identity. +type graphReconciler struct { + client.Client + recorder events.EventRecorder + apiReader client.Reader + runtimeConfig *commoncontroller.RuntimeConfig + modelRegistry lpxModelRegistry + Config *configv1alpha1.OperatorConfiguration + DockerSecretRetriever dynamo.SecretsRetriever +} + +func (r *graphReconciler) GetRecorder() events.EventRecorder { + return r.recorder +} + +// Setup registers the LPX controller and its dependencies. +func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runtimeConfig *commoncontroller.RuntimeConfig, secrets dynamo.SecretsRetriever) error { + // Disabled integration still serves child status and explicit deletion. + r := &graphReconciler{ + Client: mgr.GetClient(), + recorder: mgr.GetEventRecorder("lpxgraphdeployment"), + apiReader: mgr.GetAPIReader(), + runtimeConfig: runtimeConfig, + Config: config, + DockerSecretRetriever: secrets, + } + + // Resolve availability before activating any runtime dependencies. + enabled := r.unavailableReason() == "" + if enabled { + var err error + r.modelRegistry, err = newLPXModelRegistry(config) + if err != nil { + return err + } + } + + if err := r.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create LPXGraphDeployment controller: %w", err) + } + if enabled { + if err := setupLPUEviction(mgr); err != nil { + return fmt.Errorf("unable to create LPU Eviction controller: %w", err) + } + } + return nil +} + +// unavailableReason owns the LPX activation policy; an empty reason means enabled. +// Status and explicit deletion remain available regardless of this policy. +func (r *graphReconciler) unavailableReason() string { + if !r.runtimeConfig.Gate.Enabled(features.Grove) { + return "Grove is disabled" + } + if !r.runtimeConfig.Gate.Enabled(features.LPX) { + return "LPX integration is disabled" + } + return "" +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update +// +kubebuilder:rbac:groups=scheduler.grove.io,resources=podgangs,verbs=get;list;watch +// +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;update;delete + +func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) { + deployment := &v1alpha1.LPXGraphDeployment{} + if err := r.Get(ctx, req.NamespacedName, deployment); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !deployment.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.finalize(ctx, deployment) + } + + unavailableReason := r.unavailableReason() + previous := deployment.Status.DeepCopy() + if deployment.Status.ObservedGeneration != deployment.Generation { + deployment.Status.Components = nil + deployment.Status.ModelDownload = nil + } + state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} + var deadlineAt time.Time + defer func() { + if err != nil { + state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXReconciliationFailed", truncateLPXMessage(err.Error()) + } else if unavailableReason == "" { + deployment.Status.ObservedGeneration = deployment.Generation + } + ready, failed := metav1.ConditionFalse, metav1.ConditionFalse + if state.State == v1beta1.DGDStateSuccessful { + ready = metav1.ConditionTrue + } + if state.State == v1beta1.DGDStateFailed { + failed = metav1.ConditionTrue + } + // Every authored component observes the complete shared runtime gate. + for name, component := range deployment.Status.Components { + component.Ready = ready == metav1.ConditionTrue + deployment.Status.Components[name] = component + } + for condition, value := range map[string]metav1.ConditionStatus{"Ready": ready, "Failed": failed} { + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{Type: condition, Status: value, ObservedGeneration: deployment.Generation, Reason: state.Reason, Message: state.Message}) + } + if !apiequality.Semantic.DeepEqual(previous, &deployment.Status) { + if statusErr := r.Status().Update(ctx, deployment); statusErr != nil { + err = errors.Join(err, fmt.Errorf("persist LPX lifecycle status: %w", statusErr)) + } + } + if ready == metav1.ConditionTrue && deployment.Status.ModelDownload != nil && deployment.Status.ModelDownload.LastCheckedAt != nil && result.RequeueAfter == 0 { + result.RequeueAfter = max(time.Second, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) + } + result, err = completeLPXDeadline(ctx, deadlineAt, result, err) + }() + + // Like Grove, disabling integration reports unavailability without touching workloads. + if unavailableReason != "" { + state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXUnavailable", unavailableReason + return ctrl.Result{}, nil + } + + source := &v1beta1.DynamoGraphDeployment{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Name}, source); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + state = lpxResult(&lpxRejected{reason: "The exact source DGD no longer exists"}) + return r.retireInvalidWorkload(ctx, deployment, state.Message) + } + if sourceErr := dynamo.ValidateLPXSource(deployment, source); sourceErr != nil { + state = lpxResult(&lpxRejected{reason: sourceErr.Error()}) + return r.retireInvalidWorkload(ctx, deployment, sourceErr.Error()) + } + if !controllerutil.ContainsFinalizer(deployment, lpxGraphDeploymentFinalizer) { + controllerutil.AddFinalizer(deployment, lpxGraphDeploymentFinalizer) + return ctrl.Result{RequeueAfter: time.Nanosecond}, r.Update(ctx, deployment) + } + + classification, wake, requests, err := r.reconcileLPXAttemptDeadline(ctx, deployment, source) + deadlineAt = wake + var selected *lpxMaterializing + if classification == nil && err == nil { + selected, classification, err = r.reconcileLPXSafetyPreflight(ctx, deployment, source, requests) + } + if err != nil { + return ctrl.Result{}, err + } + if classification != nil { + state, result, err = r.finishLPXPreflight(ctx, deployment, classification) + return result, err + } + ready, err := r.reconcileModelDownloads(ctx, deployment, source, selected == nil) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + state.Reason, state.Message = modelDownloadPendingReason, modelDownloadPendingMessage + return ctrl.Result{RequeueAfter: modelDownloadRequeueAfter}, nil + } + if selected == nil { + selected, classification, err = r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source) + if err != nil { + return ctrl.Result{}, err + } + if classification != nil { + state, result, err = r.finishLPXPreflight(ctx, deployment, classification) + return result, err + } + } + state, result, err = r.reconcileWorkload(ctx, deployment, source, selected) + return result, err +} + +// finishLPXPreflight projects a non-nil classification and retires rejected workloads. +func (r *graphReconciler) finishLPXPreflight(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, classification lpxClassification) (reconcileOutcome, ctrl.Result, error) { + state := lpxResult(classification) + if _, rejected := classification.(*lpxRejected); rejected { + result, err := r.retireInvalidWorkload(ctx, deployment, state.Message) + return state, result, err + } + return state, projectLPXLifecycleStatus(deployment, classification), nil +} + +// reconcileWorkload publishes and observes the selected engine after source, +// deadline, placement, and model-download gates have passed. deployment, source +// and selected are non-nil and supplied by that preflight. +func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment, selected *lpxMaterializing) (reconcileOutcome, ctrl.Result, error) { + state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} + + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs); err != nil { + if !apierrors.IsNotFound(err) { + return state, ctrl.Result{}, err + } + pcs = nil + } + desired, resources, err := renderPodCliqueSet(ctx, source, r.Config, r.runtimeConfig, r.Client, r.DockerSecretRetriever, selected.workload, selected.plan, deployment) + if err != nil { + retiring, fenceErr := r.retireInvalidLPXWorkload(ctx, deployment, fmt.Sprintf("The current LPX workload can no longer be rendered safely: %v", err)) + if fenceErr != nil { + return state, ctrl.Result{}, errors.Join(err, fenceErr) + } + if retiring != nil { + state = lpxResult(retiring) + return state, projectLPXLifecycleStatus(deployment, retiring), nil + } + return state, ctrl.Result{}, err + } + if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { + return state, ctrl.Result{}, err + } + for _, resource := range resources { + if err := r.syncLPXResource(ctx, deployment, resource); err != nil { + return state, ctrl.Result{}, err + } + } + synced, changed, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired, selected) + if err != nil { + var retiring *lpxRetiring + if errors.As(err, &retiring) { + state = lpxResult(retiring) + return state, projectLPXLifecycleStatus(deployment, retiring), nil + } + if errors.Is(err, errLPXGrovePodCliqueSetRecreating) { + return state, ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter}, nil + } + return state, ctrl.Result{}, err + } + if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { + return state, ctrl.Result{}, err + } + readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, source, deployment, synced) + if err != nil { + return state, ctrl.Result{}, err + } + deployment.Status.Components = readiness.ComponentStatuses + shape, err := dynamo.ResolveLPXGPUShape(ctx, r.Client, synced) + if err != nil { + return state, ctrl.Result{}, err + } + // The serving component owns shared GPU costs exactly once. + name := selected.workload.LPXComponentName() + component := deployment.Status.Components[name] + component.GPUsPerEngine, component.GPUsPerReplica = ptr.To(shape.GPUsPerEngine), ptr.To(shape.GPUsPerReplica) + deployment.Status.Components[name] = component + state.Reason, state.Message = readiness.Classification, readiness.Message + if readiness.Ready && !changed { + state.State = v1beta1.DGDStateSuccessful + } + classification, err := r.reconcileSelectedLPX(ctx, deployment, selected) + if err != nil { + return state, ctrl.Result{}, err + } + state = overlayLPXResult(state, classification) + return state, projectLPXLifecycleStatus(deployment, classification), nil +} + +// projectLPXLifecycleStatus transfers reconcile-owned results to the same child before returning. +func projectLPXLifecycleStatus(deployment *v1alpha1.LPXGraphDeployment, classification lpxClassification) ctrl.Result { + if transition, ok := classification.(*lpxDeadlineTransition); ok { + if deployment.Status.Placement == nil { + deployment.Status.Placement = &v1beta1.PlacementStatus{} + } + deployment.Status.Placement.LPXAttempt = transition.attempt + return ctrl.Result{RequeueAfter: transition.requeueAfter} + } + if _, retiring := classification.(*lpxRetiring); retiring { + return ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter} + } + return ctrl.Result{} +} + +func (r *graphReconciler) retireInvalidWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, reason string) (ctrl.Result, error) { + retiring, err := r.retireInvalidLPXWorkload(ctx, deployment, reason) + if err != nil { + return ctrl.Result{}, err + } + if retiring != nil { + return ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter}, nil + } + return ctrl.Result{}, nil +} + +func (r *graphReconciler) finalize(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { + if !controllerutil.ContainsFinalizer(deployment, lpxGraphDeploymentFinalizer) { + return nil + } + if err := r.finalizeLPXRequests(ctx, deployment); err != nil { + return err + } + controllerutil.RemoveFinalizer(deployment, lpxGraphDeploymentFinalizer) + return r.Update(ctx, deployment) +} + +// Publication is an identity-sensitive write boundary: an uncached read fences +// a source/child replacement or input edit, not an attempt to repair informer lag. +func (r *graphReconciler) validateLPXPublicationSource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return err + } + source := &v1beta1.DynamoGraphDeployment{} + if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Name}, source); err != nil { + return err + } + return dynamo.ValidateLPXSource(deployment, source) +} + +// validateLPXDeploymentAuthority fences writes against a replaced or newer child, +// independently of whether its source remains valid for publication. +func (r *graphReconciler) validateLPXDeploymentAuthority(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { + current := &v1alpha1.LPXGraphDeployment{} + if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(deployment), current); err != nil { + return err + } + if current.UID != deployment.UID || current.Generation != deployment.Generation || !apiequality.Semantic.DeepEqual(current.Spec, deployment.Spec) || !apiequality.Semantic.DeepEqual(current.OwnerReferences, deployment.OwnerReferences) || !current.DeletionTimestamp.IsZero() { + return fmt.Errorf("LPXGraphDeployment authority changed before LPX resource mutation") + } + for _, key := range []string{dynamo.LPXPCSNameAnnotation, dynamo.LPXRestartAnnotation, lpx.DGDGenerationAnnotation} { + if current.Annotations[key] != deployment.Annotations[key] { + return fmt.Errorf("LPXGraphDeployment publication metadata changed before LPX resource mutation") + } + } + return nil +} + +func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, desired client.Object) error { + // Observe the resource once and reject foreign ownership before preparing a write. + live := desired.DeepCopyObject().(client.Object) + if err := r.Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + live = nil + } else if !metav1.IsControlledBy(live, deployment) { + return fmt.Errorf("refusing to adopt LPX resource %s/%s without its exact owner", live.GetNamespace(), live.GetName()) + } + + // Prepare LPX-owned metadata while preserving shared synchronization's bookkeeping. + metadataChanged := false + if live != nil { + labels, annotations := desired.GetLabels(), maps.Clone(desired.GetAnnotations()) + if annotations == nil { + annotations = make(map[string]string) + } + for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { + annotations[key] = live.GetAnnotations()[key] + } + metadataChanged = !maps.Equal(labels, live.GetLabels()) || !maps.Equal(annotations, live.GetAnnotations()) + if metadataChanged { + live.SetLabels(maps.Clone(labels)) + live.SetAnnotations(annotations) + } + } + + // Content updates carry prepared metadata; only metadata-only drift needs another path. + modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, live, desired) + if err != nil || modified || !metadataChanged { + return err + } + return r.Update(ctx, synced) +} + +func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { + component := lpx.ServingComponent(source) + if !commoncontroller.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, source.Annotations) { + service := &corev1.Service{} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.NormalizeKubeResourceName(dynamo.GetDCDResourceName(source, component.ComponentName, ""))} + if err := r.Get(ctx, key, service); err != nil { + return client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(service, deployment) { + return fmt.Errorf("refusing to delete a foreign LPX endpoint %q", service.Name) + } + return client.IgnoreNotFound(r.Delete(ctx, service, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &service.UID, ResourceVersion: &service.ResourceVersion}})) + } + service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ + ServiceName: dynamo.GetDCDResourceName(source, component.ComponentName, ""), Namespace: source.Namespace, + ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, + DynamoNamespace: source.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, + Labels: dynamo.GetDGDComponentResourceLabels(source, component.ComponentName, component), + Annotations: dynamo.GetDGDComponentResourceAnnotations(source, component.ComponentName, component), + }) + if err != nil { + return err + } + service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue + return r.syncLPXResource(ctx, deployment, service) +} diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go new file mode 100644 index 000000000000..29358cb5726c --- /dev/null +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -0,0 +1,632 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "testing" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { + t.Log("Publish a speculative engine using its frozen child identity") + child, source, registry := newLPXSpecDecodeTestDGD(t) + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:old"}}}}, + }, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "prefill:old"}}}}, + }) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) + createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) + _, err := r.reconcileSelectedLPX(t.Context(), child, selected) + require.NoError(t, err) + beforeRequests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), beforeRequests)) + require.NotEmpty(t, beforeRequests.Items) + beforePCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, selected) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + beforeChild := child.DeepCopy() + + t.Log("Reorder engines, update the frontend and enable an ordinary checkpoint without changing LPX") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + components := source.Spec.Components + components[0], components[1] = components[1], components[0] + source.GetComponentByName("frontend").PodTemplate.Spec.Containers[0].Image = "frontend:next" + source.GetComponentByName("prefill").Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ + Enabled: true, CheckpointRef: ptr.To("prefill-checkpoint"), + }} + source.Generation++ + require.NoError(t, r.Update(t.Context(), source)) + beforeSource := source.DeepCopy() + require.Empty(t, lpx.ValidateSelectedIntent(source)) + require.NoError(t, dynamo.ValidateLPXSource(child, source)) + afterSelected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) + require.Nil(t, rejected) + afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, afterSelected) + require.Equal(t, beforePCS, afterPCS) + require.Equal(t, child.Annotations[lpx.DGDGenerationAnnotation], afterPCS.Annotations[lpx.DGDGenerationAnnotation]) + _, err = r.reconcileSelectedLPX(t.Context(), child, afterSelected) + require.NoError(t, err) + afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), afterRequests)) + require.Equal(t, beforeRequests, afterRequests) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, beforeChild, child) + require.Equal(t, beforeSource, source) +} + +func TestLPXCurrentAttemptSurvivesUnobservedReconciliation(t *testing.T) { + t.Log("A failed reconciliation may have a current durable attempt and an older result generation") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + child := newLPXTestDeployment(t, source) + source.Spec.Scheduling = deadlineTestScheduling() + child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ + ObservedGeneration: child.Generation, + DeadlineAt: ptr.To(metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(time.Minute))), + Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "current-request", UID: "current-uid"}}, + }} + r := newLPXTestReconciler(t, nil, child, source) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + before := child.Status.Placement.DeepCopy() + readError := errors.New("temporary request observation failure") + r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, requests := list.(*lpxv1alpha1.LPUPipelineRequestList); requests { + return readError + } + return delegated.List(ctx, list, opts...) + }, + }) + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err, "active deadlines bound transient errors with a scheduled retry") + require.Positive(t, result.RequeueAfter) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.NotEqual(t, child.Generation, child.Status.ObservedGeneration) + require.Equal(t, before, child.Status.Placement) + require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Failed").Message, readError.Error()) +} + +func TestLPXGPUCapacityReportsTheCompleteEngine(t *testing.T) { + for _, mode := range []string{"lpu-only", "hybrid", "speculative"} { + t.Run(mode, func(t *testing.T) { + t.Log("Reconcile the actual LPX workload and publish its per-engine GPU shape") + var child *v1alpha1.LPXGraphDeployment + var source *v1beta1.DynamoGraphDeployment + var registry *lpx.ModelRegistry + var want int64 + switch mode { + case "lpu-only": + child, source, registry = newLPXTestDGD(t, lpx.PipelineSingle) + case "hybrid": + child, source, registry = newLPXTestDGD(t, lpx.PipelineLPX) + source.Spec.Components[0].Replicas = ptr.To(int32(3)) + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + pod := &lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec + pod.ResourceClaims = nil + pod.Containers[0].Resources.Limits = corev1.ResourceList{consts.KubeResourceGPUNvidia: resource.MustParse("4")} + want = 8 + case "speculative": + child, source, registry = newLPXSpecDecodeTestDGD(t) + } + r := newLPXTestReconciler(t, registry, child, source) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Contains(t, child.Status.Components, "lpx") + require.Equal(t, ptr.To(want), child.Status.Components["lpx"].GPUsPerEngine) + require.Equal(t, ptr.To(want), child.Status.Components["lpx"].GPUsPerReplica) + + }) + } +} + +func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { + for _, scenario := range []struct { + name, reason, message string + preserve bool + }{ + {name: "intent", reason: "LPXRejected", message: "providerOverride"}, + {name: "name budget", reason: "LPXRejected", message: "spec.components[0].name: Invalid value"}, + {name: "selected workload", reason: "LPXRejected", message: "fixed engine replicas"}, + {name: "render", reason: "LPXReconciliationFailed", message: "model storage volume mount"}, + {name: "invalid source", reason: "LPXRejected", message: "source"}, + {name: "transient snapshot", reason: "LPXReconciliationFailed", message: "temporary snapshot timeout", preserve: true}, + {name: "inconsistent snapshot", reason: "LPXReconciliationFailed", message: "immutable LPX build snapshot"}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Stage a real PCS and discovery endpoint before Grove can supply publication identities") + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + r := newLPXTestReconciler(t, registry, child, source) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), request) + require.NoError(t, err) + pcs := &grovev1alpha1.PodCliqueSet{} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: child.Annotations[dynamo.LPXPCSNameAnnotation]} + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + pcs.UID = "staged-pcs" + require.NoError(t, r.Update(t.Context(), pcs)) + endpoint := &corev1.Service{} + endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.GetDCDResourceName(source, "lpx", "")} + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + endpoint.UID = "staged-endpoint" + require.NoError(t, r.Update(t.Context(), endpoint)) + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, requests.Items) + + t.Log("Healthy repeated reconciliation must preserve unpublished workload identities") + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + beforePCS, beforeEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + require.Equal(t, beforePCS, pcs) + require.Equal(t, beforeEndpoint, endpoint) + + t.Log("Introduce a current terminal input failure, or a transient snapshot outage") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + component := lpx.ServingComponent(source) + switch scenario.name { + case "intent": + source.Spec.ProviderOverride = &v1beta1.ProviderOverride{Target: "PodCliqueSet"} + case "name budget": + component.ComponentName = "serving-engines" + child.Annotations[dynamo.LPXPCSNameAnnotation] = dynamo.PCSNameForLPX(source) + case "selected workload": + component.Replicas = ptr.To(int32(2)) + case "render": + component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].VolumeMounts = nil + case "invalid source": + child.OwnerReferences[0].UID = "replaced-source" + case "transient snapshot", "inconsistent snapshot": + cause := errors.New("temporary snapshot timeout") + if !scenario.preserve { + cause = lpx.ErrBuildSnapshotInconsistent + } + r.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: cause} + } + source.Generation++ + require.NoError(t, r.Update(t.Context(), source)) + child.Generation++ + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + require.NoError(t, r.Update(t.Context(), child)) + + t.Log("Converge cleanup and retain the original actionable failure on repeated reconciliation") + for range 3 { + _, err = r.Reconcile(t.Context(), request) + if scenario.reason == "LPXReconciliationFailed" && err != nil { + require.ErrorContains(t, err, scenario.message) + } else { + require.NoError(t, err) + } + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").Message, scenario.message) + } + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + failure := meta.FindStatusCondition(child.Status.Conditions, "Failed") + require.NotNil(t, failure) + require.Equal(t, metav1.ConditionTrue, failure.Status) + require.Equal(t, scenario.reason, failure.Reason) + require.Contains(t, failure.Message, scenario.message) + if scenario.preserve { + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + require.Equal(t, beforePCS, pcs) + require.Equal(t, beforeEndpoint, endpoint) + } else { + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, pcs)), "terminal failure retained the staged PCS") + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), endpointKey, endpoint)), "terminal failure retained the obsolete endpoint") + } + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, requests.Items) + }) + } +} + +func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T) { + const replacedEndpoint = "replaced endpoint" + for _, scenario := range []string{"cleanup", "new child generation", replacedEndpoint, "updated PCS", "pending finalizer"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Seed obsolete owned staging beside ordinary and foreign objects at current target names") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + child := newLPXTestDeployment(t, source) + owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: "obsolete-pcs", Namespace: child.Namespace, UID: "old-pcs", OwnerReferences: owner, + }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}} + endpoint := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: "obsolete-endpoint", Namespace: child.Namespace, UID: "old-endpoint", OwnerReferences: owner, + }} + ordinary := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: "ordinary-model", Namespace: child.Namespace, UID: "ordinary-service", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, + }} + foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() + foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" + foreignPCS.Name = child.Annotations[dynamo.LPXPCSNameAnnotation] + foreignEndpoint.Name = dynamo.GetDCDResourceName(source, "lpx", "") + foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences + if scenario == "pending finalizer" { + pcs.Finalizers = []string{"example.com/staged-cleanup"} + } + r := newLPXTestReconciler(t, nil, child, source, pcs, endpoint, ordinary, foreignPCS, foreignEndpoint) + for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint} { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) + } + + t.Log("Race observed child or resource authority immediately before terminal deletion") + deletes := 0 + wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := delegated.List(ctx, list, opts...); err != nil { + return err + } + if _, cliques := list.(*grovev1alpha1.PodCliqueSetList); cliques && scenario == "new child generation" { + current := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(child), current)) + current.Generation++ + return delegated.Update(ctx, current) + } + return nil + }, + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + deletes++ + preconditions := (&client.DeleteOptions{}).ApplyOptions(opts).Preconditions + require.NotNil(t, preconditions) + require.Equal(t, object.GetUID(), *preconditions.UID) + require.Equal(t, object.GetResourceVersion(), *preconditions.ResourceVersion) + if scenario == replacedEndpoint && object.GetName() == endpoint.Name || scenario == "updated PCS" && object.GetName() == pcs.Name { + current := object.DeepCopyObject().(client.Object) + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(object), current)) + if scenario == replacedEndpoint { + current.SetUID("replacement-endpoint") + } + current.SetLabels(map[string]string{"example.com/newer": "true"}) + require.NoError(t, delegated.Update(ctx, current)) + } + return delegated.Delete(ctx, object, opts...) + }, + }) + r.Client, r.apiReader = wrapped, wrapped + retiring, err := r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + switch scenario { + case "new child generation": + require.ErrorContains(t, err, "authority changed") + require.Zero(t, deletes) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(endpoint), &corev1.Service{})) + case replacedEndpoint, "updated PCS": + require.True(t, apierrors.IsConflict(err), "object resource-version precondition must reject the stale deletion: %v", err) + object := client.Object(pcs) + if scenario == replacedEndpoint { + object = endpoint + } + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) + require.Equal(t, "true", object.GetLabels()["example.com/newer"]) + default: + require.NoError(t, err) + require.NotNil(t, retiring) + if scenario == "pending finalizer" { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.False(t, pcs.DeletionTimestamp.IsZero()) + retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + require.NoError(t, err) + require.NotNil(t, retiring) + pcs.Finalizers = nil + require.NoError(t, r.Update(t.Context(), pcs)) + } + retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + require.NoError(t, err) + require.Nil(t, retiring) + require.Equal(t, 2, deletes, "cleanup must not repeat deletes after they are accepted") + } + + t.Log("Ordinary DGD resources and foreign same-name objects remain byte-for-byte unchanged") + for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint} { + before := object.DeepCopyObject().(client.Object) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) + require.Equal(t, before, object) + } + }) + } +} + +func TestLPXEndpointLifecycle(t *testing.T) { + t.Log("Configure an LPX serving component using Kubernetes discovery") + source := newLPXTestSource(lpx.PipelineSingle, "test-build") + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + source.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} + source.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} + source.Spec.Labels = map[string]string{"example.com/policy": "enabled"} + child := newLPXTestDeployment(t, source) + r := newLPXTestReconciler(t, nil, child, source) + + t.Log("Keep the ordinary model Service under the source DGD's ownership") + modelService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.GenerateServiceName("test/model"), Namespace: source.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, + Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: source.Name}, + }, + Spec: corev1.ServiceSpec{ClusterIP: corev1.ClusterIPNone, + Selector: map[string]string{consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("test/model")}}, + } + require.NoError(t, r.Create(t.Context(), modelService)) + modelKey := client.ObjectKeyFromObject(modelService) + require.NoError(t, r.Get(t.Context(), modelKey, modelService)) + beforeModelService := modelService.DeepCopy() + + t.Log("Publish the LPX-owned endpoint with its serving-role selector") + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + service := &corev1.Service{} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.GetDCDResourceName(source, "lpx", "")} + require.NoError(t, r.Get(t.Context(), key, service)) + require.True(t, metav1.IsControlledBy(service, child)) + require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) + + t.Log("Converge propagated endpoint metadata without losing identity, content or sync bookkeeping") + for _, value := range []string{"changed", ""} { + if value == "" { + source.Spec.Labels, source.Spec.Annotations = nil, nil + } else { + source.Spec.Labels["example.com/policy"] = value + source.Spec.Annotations["example.com/model-discovery"] = value + } + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + updated := &corev1.Service{} + require.NoError(t, r.Get(t.Context(), key, updated)) + require.Equal(t, value, updated.Labels["example.com/policy"]) + require.Equal(t, value, updated.Annotations["example.com/model-discovery"]) + require.Equal(t, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled], updated.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) + if value == "" { + require.NotContains(t, updated.Labels, "example.com/policy") + require.NotContains(t, updated.Annotations, "example.com/model-discovery") + } + require.Equal(t, service.Spec, updated.Spec) + require.Equal(t, service.OwnerReferences, updated.OwnerReferences) + for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { + require.NotEmpty(t, updated.Annotations[key]) + require.Equal(t, service.Annotations[key], updated.Annotations[key]) + } + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + unchanged := &corev1.Service{} + require.NoError(t, r.Get(t.Context(), key, unchanged)) + require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) + } + + t.Log("Remove the LPX-owned discovery endpoint when switching back to non-Kubernetes discovery") + delete(source.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) + require.NoError(t, r.Get(t.Context(), modelKey, modelService)) + require.Equal(t, beforeModelService, modelService, "LPX cleanup must leave the DGD-owned model Service unchanged") +} + +func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) { + t.Log("Publish a runtime ConfigMap under the exact LPX child") + source := newLPXTestSource(lpx.PipelineSingle, "test-build") + child := newLPXTestDeployment(t, source) + r := newLPXTestReconciler(t, nil, child, source) + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "lpx-runtime", Namespace: child.Namespace, + Labels: map[string]string{"example.com/policy": "enabled"}, Annotations: map[string]string{"example.com/policy": "enabled"}}, + Data: map[string]string{"config": "initial runtime"}, + } + require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) + recorded := r.recorder.(*events.FakeRecorder).Events + require.Len(t, recorded, 1) + require.Equal(t, "Normal CreateConfigMap Created ConfigMap "+child.Namespace, <-recorded) + key := client.ObjectKeyFromObject(desired) + before := &corev1.ConfigMap{} + require.NoError(t, r.Get(t.Context(), key, before)) + updates := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { + updates++ + return delegated.Update(ctx, object, opts...) + }, + }) + + t.Log("Change runtime content and remove all authored metadata while retaining refreshed bookkeeping") + desired.Data["config"] = "updated runtime" + desired.Labels, desired.Annotations = nil, nil + desired.OwnerReferences = nil + require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) + require.Len(t, recorded, 1) + require.Equal(t, "Normal UpdateConfigMap Updated ConfigMap "+child.Namespace, <-recorded) + require.Equal(t, 1, updates, "content and metadata must converge in one resource update") + updated := &corev1.ConfigMap{} + require.NoError(t, r.Get(t.Context(), key, updated)) + require.Equal(t, desired.Data, updated.Data) + require.Empty(t, updated.Labels) + require.Len(t, updated.Annotations, 2) + hash, err := commoncontroller.GetSpecHash(desired) + require.NoError(t, err) + require.Equal(t, hash, updated.Annotations[commoncontroller.NvidiaAnnotationHashKey]) + require.NotEqual(t, before.Annotations[commoncontroller.NvidiaAnnotationHashKey], hash) + require.NotEmpty(t, updated.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) + require.Equal(t, before.OwnerReferences, updated.OwnerReferences) + require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) + require.Empty(t, recorded) + unchanged := &corev1.ConfigMap{} + require.NoError(t, r.Get(t.Context(), key, unchanged)) + require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) + + t.Log("A foreign resource at the same key is never adopted or rewritten") + updated.OwnerReferences[0].UID = "foreign-child" + require.NoError(t, r.Update(t.Context(), updated)) + desired.Data["config"] = "must-not-publish" + require.ErrorContains(t, r.syncLPXResource(t.Context(), child, desired), "refusing to adopt LPX resource") + require.NoError(t, r.Get(t.Context(), key, unchanged)) + require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) + require.Equal(t, updated.Data, unchanged.Data) +} + +func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { + for _, scenario := range []struct { + name, componentName string + sharedDraft bool + messages []string + }{ + {name: "invalid intent", messages: []string{ + "spec.providerOverride: Forbidden:", + "spec.components[0].providerOverride: Forbidden:", + `LPX worker component requires a "main" runtime container`, + "does not support checkpointing", + "spec.topologyConstraint: Forbidden:", + "spec.components[0].topologyConstraint: Forbidden:", + }}, + {name: "disabled Grove", messages: []string{"Grove is disabled"}}, + {name: "component provider", messages: []string{"requires the Grove workload provider"}}, + {name: "too long serving name", componentName: "serving-engines", messages: []string{ + "spec.components[0].name: Invalid value", "combined Grove resource name length 46 exceeds the 45-character limit", + }}, + {name: "reordered shared draft with too long serving name", componentName: "serving-engines", sharedDraft: true, messages: []string{ + "spec.components[2].name: Invalid value", "combined Grove resource name length 46 exceeds the 45-character limit", + }}, + {name: "maximum legal serving name", componentName: "serving-engine"}, + {name: "reordered shared draft with maximum legal serving name", componentName: "serving-engine", sharedDraft: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Reconcile authored LPX intent against cold remote builds") + source := newLPXTestSource(lpx.PipelineSingle, modelDownloadTestBuildID) + if scenario.sharedDraft { + source = newLPXSpecDecodeTestSource() + source.Spec.Components[0].LPX.BuildID = modelDownloadTestBuildID + source.Spec.Components[1].LPX.BuildID = modelDownloadTestSecondBuildID + source.Spec.Components[1].ComponentName = "shared-draft-name-is-not-materialized" + source.Spec.Components = []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend}, + source.Spec.Components[1], source.Spec.Components[0], + } + } + component := lpx.ServingComponent(source) + if scenario.componentName != "" { + component.ComponentName = scenario.componentName + } + switch scenario.name { + case "invalid intent": + source.Spec.ProviderOverride = &v1beta1.ProviderOverride{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueSet", + } + source.Spec.ProviderOverride.Value.Raw = []byte(`{"spec":{"template":{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}}}`) + component.ProviderOverride = &v1beta1.ProviderOverride{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueTemplateSpec", + } + component.ProviderOverride.Value.Raw = []byte(`{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}`) + component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate = &corev1.PodTemplateSpec{} + component.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{Enabled: true}} + source.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} + component.TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} + case "component provider": + delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) + source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent + } + registry, err := lpx.NewModelRegistry("gs://bucket/registry", nil) + require.NoError(t, err) + observedRegistry := &fakeModelDownloadRegistry{ModelRegistry: registry} + child := newLPXTestDeployment(t, source) + r := newLPXTestReconciler(t, observedRegistry, child, source) + if source.Spec.ProviderOverride != nil || component.ProviderOverride != nil { + t.Log("Match the stored provider override after its raw JSON is normalized") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + revision, err := dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + child.Spec.InputRevision = revision + require.NoError(t, r.Update(t.Context(), child)) + } + if scenario.name == "disabled Grove" { + r.runtimeConfig.Gate.Grove = false + } + + t.Log("Reject unsupported intent before downloads and let valid names reach the download gate") + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + failed := meta.FindStatusCondition(stored.Status.Conditions, "Failed") + require.NotNil(t, failed) + if len(scenario.messages) == 0 { + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, modelDownloadPendingReason, failed.Reason) + wantCalls := []string{modelDownloadTestBuildID} + if scenario.sharedDraft { + wantCalls = append(wantCalls, modelDownloadTestSecondBuildID) + } + require.Equal(t, wantCalls, observedRegistry.calls) + } else { + require.Equal(t, metav1.ConditionTrue, failed.Status) + if scenario.name == "disabled Grove" { + require.Equal(t, "LPXUnavailable", failed.Reason) + } else { + require.Equal(t, "LPXRejected", failed.Reason) + } + for _, message := range scenario.messages { + require.Contains(t, failed.Message, message) + } + require.Empty(t, observedRegistry.calls) + } + require.Zero(t, observedRegistry.acquireBuildSnapshotCalls) + pcs := &grovev1alpha1.PodCliqueSetList{} + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), pcs)) + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, pcs.Items) + require.Empty(t, requests.Items) + }) + } +} + +func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { + t.Log("Prepare an immutable render from the current source and child") + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) + pcs := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, selected) + require.NoError(t, r.validateLPXPublicationSource(t.Context(), child)) + + t.Log("A source edit racing publication prevents the old render from creating a PCS") + liveSource := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) + liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" + require.NoError(t, r.Update(t.Context(), liveSource)) + _, _, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs, selected) + require.ErrorContains(t, err, "input revision") + allPCS := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, r.List(t.Context(), allPCS)) + require.Empty(t, allPCS.Items) + + t.Log("Child publication metadata cannot change behind an unchanged spec generation") + liveChild := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), liveChild)) + liveChild.Annotations[lpx.DGDGenerationAnnotation] = "999" + require.NoError(t, r.Update(t.Context(), liveChild)) + require.ErrorContains(t, r.validateLPXPublicationSource(t.Context(), child), "publication metadata changed") +} diff --git a/deploy/operator/internal/controller/lpx/dependencies.go b/deploy/operator/internal/controller/lpx/dependencies.go new file mode 100644 index 000000000000..83ce91f020ea --- /dev/null +++ b/deploy/operator/internal/controller/lpx/dependencies.go @@ -0,0 +1,154 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dra" + resourcev1 "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + lpxTopologyBindingRefIndex = "lpx.topologyBinding" + lpxResourceClaimRefIndex = "lpx.resourceClaim" + lpxResourceClaimTemplateRefIndex = "lpx.resourceClaimTemplate" + lpxDeviceClassRefIndex = "lpx.deviceClass" +) + +// lpxTopologyBindingReferences indexes non-nil DGD objects by their LPX topology policy. +func lpxTopologyBindingReferences(obj client.Object) []string { + source := obj.(*nvidiacomv1beta1.DynamoGraphDeployment) + if !source.HasLPXComponent() || source.Spec.Experimental == nil || source.Spec.Experimental.KvTransferPolicy == nil { + return nil + } + name := source.Spec.Experimental.KvTransferPolicy.ClusterTopologyName + if name == "" { + return nil + } + return []string{name} +} + +// lpxDRAClaimReferences indexes non-nil DGD objects by consumed GPU-role claims. +// Extraction uses only source intent, including references not yet created. +func lpxDRAClaimReferences(template bool) client.IndexerFunc { + return func(obj client.Object) []string { + // Index every authored role, including Agent templates used by conductor fallback. + source := obj.(*nvidiacomv1beta1.DynamoGraphDeployment) + references := sets.New[string]() + for index := range source.Spec.Components { + component := &source.Spec.Components[index] + if !component.IsLPX() { + continue + } + for _, role := range component.Roles { + if role.PodTemplate == nil { + continue + } + pod := &role.PodTemplate.Spec + consumed := sets.New[string]() + for _, container := range dra.AllContainers(pod) { + for _, claim := range container.Resources.Claims { + consumed.Insert(claim.Name) + } + } + for _, claim := range pod.ResourceClaims { + if !consumed.Has(claim.Name) { + continue + } + name := claim.ResourceClaimName + if template { + name = claim.ResourceClaimTemplateName + } + if name != nil && *name != "" { + references.Insert(*name) + } + } + } + } + return references.UnsortedList() + } +} + +// lpxDeviceClassReferences indexes claims and templates, never fetched DGD state. +// A claim-only edit therefore updates this index without requiring a DGD edit. +func lpxDeviceClassReferences(obj client.Object) []string { + var spec *resourcev1.ResourceClaimSpec + switch resource := obj.(type) { + case *resourcev1.ResourceClaim: + spec = &resource.Spec + case *resourcev1.ResourceClaimTemplate: + spec = &resource.Spec.Spec + default: + return nil + } + + // Every alternative can become the selected device request. + classes := sets.New[string]() + for _, request := range spec.Devices.Requests { + if request.Exactly != nil && request.Exactly.DeviceClassName != "" { + classes.Insert(request.Exactly.DeviceClassName) + } + for _, alternative := range request.FirstAvailable { + if alternative.DeviceClassName != "" { + classes.Insert(alternative.DeviceClassName) + } + } + } + return classes.UnsortedList() +} + +func (r *graphReconciler) indexedLPXDependencyRequests(ctx context.Context, dependency client.Object, index string) []ctrl.Request { + // Cached field queries load only the LPX sources referencing this dependency. + sources := &nvidiacomv1beta1.DynamoGraphDeploymentList{} + if err := r.List(ctx, sources, client.InNamespace(dependency.GetNamespace()), client.MatchingFields{index: dependency.GetName()}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX sources for dependency", "index", index, "dependency", client.ObjectKeyFromObject(dependency)) + return nil + } + + // Cluster-scoped events still respect each source's configured namespace filter. + requests := make([]ctrl.Request, 0, len(sources.Items)) + for i := range sources.Items { + source := &sources.Items[i] + if commoncontroller.NamespaceAllowed(r.Config, r.runtimeConfig, source, source.Namespace) { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)}) + } + } + return requests +} + +func (r *graphReconciler) mapLPXDeviceClassToRequests(ctx context.Context, class *resourcev1.DeviceClass) []ctrl.Request { + // DeviceClass names live on claims/templates, so resolve both indexed hops. + claims := &resourcev1.ResourceClaimList{} + if err := r.List(ctx, claims, client.MatchingFields{lpxDeviceClassRefIndex: class.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX ResourceClaim dependencies", "deviceClass", class.Name) + return nil + } + templates := &resourcev1.ResourceClaimTemplateList{} + if err := r.List(ctx, templates, client.MatchingFields{lpxDeviceClassRefIndex: class.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX ResourceClaimTemplate dependencies", "deviceClass", class.Name) + return nil + } + + // Multiple claims and engines can refer to the same class in one graph. + consumers := sets.New[ctrl.Request]() + for i := range claims.Items { + for _, request := range r.indexedLPXDependencyRequests(ctx, &claims.Items[i], lpxResourceClaimRefIndex) { + consumers.Insert(request) + } + } + for i := range templates.Items { + for _, request := range r.indexedLPXDependencyRequests(ctx, &templates.Items[i], lpxResourceClaimTemplateRefIndex) { + consumers.Insert(request) + } + } + return consumers.UnsortedList() +} diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go new file mode 100644 index 000000000000..c35334f1099a --- /dev/null +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -0,0 +1,264 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "fmt" + "slices" + "strconv" + "strings" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" +) + +const ( + lpuEvictionEventReason = "LPUEviction" + podDisruptionReasonTaintManagerDeletion = "DeletionByTaintManager" + podDisruptionReasonEvictionAPI = "EvictionByEvictionAPI" +) + +type lpuEvictionReconciler struct { + client.Client + Recorder events.EventRecorder +} + +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get +func (r *lpuEvictionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var pod corev1.Pod + if err := r.Get(ctx, req.NamespacedName, &pod); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if !r.isTrigger(&pod) { + return ctrl.Result{}, nil + } + + pods, err := r.podsForTrigger(ctx, &pod) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.deletePods(ctx, &pod, pods) +} + +// setupLPUEviction registers eviction of disrupted node-local LPX Agent cohorts. +func setupLPUEviction(mgr ctrl.Manager) error { + r := &lpuEvictionReconciler{Client: mgr.GetClient(), Recorder: mgr.GetEventRecorder("lpu-eviction")} + return ctrl.NewControllerManagedBy(mgr). + Named("lpu-eviction"). + Watches(&corev1.Pod{}, &handler.EnqueueRequestForObject{}, + builder.WithPredicates(r.evictionPredicate()), + ). + Complete(r) +} + +func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { + mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[dynamolpx.WorkloadModeAnnotation]) + if mode == "" { + return nil, nil + } + pods, err := r.allModelPods(ctx, trigger) + if err != nil { + return nil, err + } + switch mode { + case lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly: + case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: + // Resolve the trigger's runtime partition from Dynamo's generated ConfigMap. + partitionByPodIndex, err := r.runtimePartitionByPodIndex(ctx, trigger) + if err != nil { + return nil, err + } + triggerPartition, ok := partitionByPodIndex[trigger.Labels[grovecommon.LabelPodCliquePodIndex]] + if !ok { + return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime partition for Grove pod index %q", trigger.Namespace, trigger.Name, trigger.Labels[grovecommon.LabelPodCliquePodIndex]) + } + + // Keep only Agent Pods in the trigger's Dynamo-authored runtime partition. + selected := pods[:0] + triggerConfigHash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] + for _, pod := range pods { + // Ignore Pods from a concurrent rollout with a different runtime table. + if pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != triggerConfigHash { + continue + } + partition, ok := partitionByPodIndex[pod.Labels[grovecommon.LabelPodCliquePodIndex]] + if !ok { + return nil, fmt.Errorf("LPU-GPU eviction candidate pod %s/%s has no runtime partition for Grove pod index %q", pod.Namespace, pod.Name, pod.Labels[grovecommon.LabelPodCliquePodIndex]) + } + if partition == triggerPartition { + selected = append(selected, pod) + } + } + pods = selected + default: + return nil, nil + } + if len(pods) == 0 { + return nil, fmt.Errorf("matched LPU eviction mode returned no pods for %s/%s", trigger.Namespace, trigger.Name) + } + return pods, nil +} + +func (r *lpuEvictionReconciler) runtimePartitionByPodIndex(ctx context.Context, trigger *corev1.Pod) (map[string]int, error) { + // Require the immutable runtime-table identity stamped on the trigger Pod. + triggerConfigHash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] + if triggerConfigHash == "" { + return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap hash", trigger.Namespace, trigger.Name) + } + + // Load the same runtime partition table mounted by every Agent Pod. + var config corev1.ConfigMap + configName := trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] + "-lpu" + if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: configName}, &config); err != nil { + return nil, fmt.Errorf("get LPU runtime ConfigMap %s/%s: %w", trigger.Namespace, configName, err) + } + + // Fail closed when a rollout has already replaced the trigger's runtime table. + configHash := dynamolpx.LPUConfigMapHash(&config) + if configHash != triggerConfigHash { + return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s does not match trigger pod %s/%s", config.Namespace, config.Name, trigger.Namespace, trigger.Name) + } + + // Require one node count and offset for every runtime partition row. + counts := strings.Fields(config.Data["nodes_per_partition"]) + offsets := strings.Fields(config.Data["partition_node_offsets"]) + if len(counts) == 0 || len(counts) != len(offsets) { + return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s has inconsistent partition node counts and offsets", config.Namespace, config.Name) + } + + // Expand each non-overlapping row into the Grove pod indexes it owns. + partitionByPodIndex := make(map[string]int) + for partition := range counts { + count, countErr := strconv.Atoi(counts[partition]) + offset, offsetErr := strconv.Atoi(offsets[partition]) + if countErr != nil || count < 1 || offsetErr != nil || offset < 0 { + return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s has invalid partition row %d", config.Namespace, config.Name, partition) + } + for rank := 0; rank < count; rank++ { + podIndex := strconv.Itoa(offset + rank) + if _, duplicate := partitionByPodIndex[podIndex]; duplicate { + return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s maps Grove pod index %s more than once", config.Namespace, config.Name, podIndex) + } + partitionByPodIndex[podIndex] = partition + } + } + return partitionByPodIndex, nil +} + +func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { + labels := trigger.GetLabels() + dgdName := labels[commonconsts.KubeLabelDynamoGraphDeploymentName] + component := labels[commonconsts.KubeLabelDynamoComponent] + pcsg := labels[grovecommon.LabelPodCliqueScalingGroup] + pcsgReplica := labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] + if dgdName == "" || component == "" || pcsg == "" || pcsgReplica == "" { + return nil, fmt.Errorf("LPU eviction trigger pod %s/%s missing required labels", trigger.Namespace, trigger.Name) + } + + var pods corev1.PodList + if err := r.List(ctx, &pods, client.InNamespace(trigger.Namespace), client.MatchingLabels{ + commonconsts.KubeLabelDynamoGraphDeploymentName: dgdName, + commonconsts.KubeLabelDynamoComponent: component, + grovecommon.LabelPodCliqueScalingGroup: pcsg, + grovecommon.LabelPodCliqueScalingGroupReplicaIndex: pcsgReplica, + }); err != nil { + return nil, fmt.Errorf("failed to list LPU pod candidates: %w", err) + } + + model := trigger.Annotations[lpxv1alpha1.PodModelAnnotation] + + return slices.DeleteFunc(pods.Items, func(pod corev1.Pod) bool { + return !r.isLPUAgentPod(&pod) || + model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model + }), nil +} + +func (r *lpuEvictionReconciler) deletePods(ctx context.Context, trigger *corev1.Pod, pods []corev1.Pod) error { + deleted := 0 + for i := range pods { + pod := &pods[i] + if pod.DeletionTimestamp != nil { + continue + } + uid := pod.UID + if err := r.Delete(ctx, pod, client.GracePeriodSeconds(0), client.Preconditions{UID: &uid}); err != nil { + if errors.IsNotFound(err) { + continue + } + return fmt.Errorf("failed to delete LPU pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + deleted++ + } + + // Repeated observations and concurrent deletions must not report another eviction. + if deleted == 0 { + return nil + } + + log.FromContext(ctx).Info("deleted LPU pod set", + "trigger", trigger.Name, + "deletedPods", deleted, + ) + r.Recorder.Eventf(trigger, nil, corev1.EventTypeWarning, lpuEvictionEventReason, "Delete", + "Pod %s is being disrupted; deleted %d LPU pods", + trigger.Name, deleted, + ) + return nil +} + +func (r *lpuEvictionReconciler) isLPUAgentPod(pod *corev1.Pod) bool { + componentType := pod.Labels[commonconsts.KubeLabelDynamoComponentType] + return componentType == commonconsts.ComponentTypeLPX && + pod.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent +} + +func (r *lpuEvictionReconciler) isTrigger(pod *corev1.Pod) bool { + if pod.Spec.SchedulerName != dynamolpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { + return false + } + return slices.ContainsFunc(pod.Status.Conditions, func(condition corev1.PodCondition) bool { + return condition.Type == corev1.DisruptionTarget && + condition.Status == corev1.ConditionTrue && + (condition.Reason == podDisruptionReasonTaintManagerDeletion || condition.Reason == podDisruptionReasonEvictionAPI) + }) +} + +func (r *lpuEvictionReconciler) evictionPredicate() predicate.Predicate { + shouldEnqueueDeletingPod := func(obj client.Object) bool { + pod, ok := obj.(*corev1.Pod) + return ok && r.isTrigger(pod) + } + + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return shouldEnqueueDeletingPod(e.Object) + }, + DeleteFunc: func(event.DeleteEvent) bool { return false }, + GenericFunc: func(event.GenericEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + return shouldEnqueueDeletingPod(e.ObjectNew) + }, + } +} diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go new file mode 100644 index 000000000000..13a58bf6831f --- /dev/null +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -0,0 +1,368 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "sort" + "testing" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" +) + +const ( + lpuEvictionTestNamespace = "test-ns" + lpuEvictionTestDGD = "test-dgd" + lpuEvictionTestComponent = "lpx-worker" + lpuEvictionTestPCSG = "lpx-worker" + lpuEvictionTestModel = "test-model" +) + +func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + cb := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&corev1.Pod{}). + WithObjects(newLPUEvictionConfigMap()) + for _, o := range objs { + cb = cb.WithObjects(o) + } + c := cb.Build() + + return &lpuEvictionReconciler{Client: c, Recorder: events.NewFakeRecorder(16)}, c +} + +func newLPUEvictionConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: lpuEvictionTestDGD + "-lpu", Namespace: lpuEvictionTestNamespace}, + Data: map[string]string{ + "nodes_per_partition": "2\n1", + "partition_node_offsets": "0\n2", + }, + } +} + +func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting bool, conditionReason string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: lpuEvictionTestNamespace, + UID: types.UID(name + "-uid"), + Labels: map[string]string{ + commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, + commonconsts.KubeLabelDynamoComponent: lpuEvictionTestComponent, + commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, + grovecommon.LabelPodCliqueScalingGroup: lpuEvictionTestPCSG, + grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, + }, + Annotations: map[string]string{ + commonconsts.AnnotationExtraResourcesHash: dynamolpx.LPUConfigMapHash(newLPUEvictionConfigMap()), + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + }, + }, + Spec: corev1.PodSpec{SchedulerName: dynamolpx.SchedulerName}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + if podIndex != "" { + pod.Labels[grovecommon.LabelPodCliquePodIndex] = podIndex + } + pod.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + if conditionReason != "" { + pod.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.DisruptionTarget, + Status: corev1.ConditionTrue, + Reason: conditionReason, + }} + } + if deleting { + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Finalizers = []string{"test-finalizer"} + } + return pod +} + +func newLPUOnlyConductor() *corev1.Pod { + pod := newLPUEvictionPod("conductor-0", "0", "", false, "") + pod.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor + return pod +} + +func reconcileLPUEviction(t *testing.T, r *lpuEvictionReconciler, name string) ctrl.Result { + t.Helper() + + result, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: name, Namespace: lpuEvictionTestNamespace}, + }) + require.NoError(t, err) + return result +} + +func remainingLPUEvictionPods(t *testing.T, c client.Client) []string { + t.Helper() + + var remaining corev1.PodList + require.NoError(t, c.List(context.Background(), &remaining, client.InNamespace(lpuEvictionTestNamespace))) + + names := make([]string, 0, len(remaining.Items)) + for _, pod := range remaining.Items { + if pod.DeletionTimestamp != nil { + continue + } + names = append(names, pod.Name) + } + sort.Strings(names) + return names +} + +func lpuEvictionPodNames(pods []corev1.Pod) []string { + names := make([]string, 0, len(pods)) + for _, pod := range pods { + names = append(names, pod.Name) + } + sort.Strings(names) + return names +} + +func TestLPUEviction_PodsForTrigger(t *testing.T) { + t.Run("lpu-gpu rejects a runtime table newer than the trigger", func(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = "stale" + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, _ := newLPUEvictionReconciler(trigger, sibling) + + pods, err := r.podsForTrigger(context.Background(), trigger) + + require.ErrorContains(t, err, "does not match trigger pod") + assert.Nil(t, pods) + }) + + t.Run("lpu-gpu excludes Pods from a concurrent rollout", func(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + samePartition := newLPUEvictionPod("agent-1", "0", "1", false, "") + samePartition.Annotations[commonconsts.AnnotationExtraResourcesHash] = "other" + r, _ := newLPUEvictionReconciler(trigger, samePartition) + + pods, err := r.podsForTrigger(context.Background(), trigger) + + require.NoError(t, err) + assert.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) + }) + + t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, _ := newLPUEvictionReconciler(trigger, sibling) + + pods, err := r.podsForTrigger(context.Background(), trigger) + + require.Error(t, err) + assert.Nil(t, pods) + }) + + t.Run("lpu-gpu pod without model filters only by partition", func(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) + sibling := newLPUEvictionPod("agent-1", "0", "0", false, "") + r, _ := newLPUEvictionReconciler(trigger, sibling) + + pods, err := r.podsForTrigger(context.Background(), trigger) + + require.NoError(t, err) + require.NotNil(t, pods) + assert.Equal(t, []string{"agent-0", "agent-1"}, lpuEvictionPodNames(pods)) + }) +} + +func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { + tests := []struct { + name string + reason string + deletedConcurrently bool + }{ + {name: "taint manager", reason: podDisruptionReasonTaintManagerDeletion}, + {name: "eviction api", reason: podDisruptionReasonEvictionAPI}, + {name: "deleted concurrently", reason: podDisruptionReasonTaintManagerDeletion, deletedConcurrently: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Observe a disrupted Agent alongside its conductor and another replica") + trigger := newLPUEvictionPod("agent-0", "0", "0", true, tt.reason) + conductor := newLPUOnlyConductor() + agent := newLPUEvictionPod("agent-1", "0", "1", false, "") + otherReplica := newLPUEvictionPod("agent-other-replica", "1", "0", false, "") + + r, c := newLPUEvictionReconciler( + trigger, + conductor, + agent, + otherReplica, + ) + if tt.deletedConcurrently { + r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + // Another actor removes the sibling before this reconciler's delete reaches the server. + require.NoError(t, delegated.Delete(ctx, object, opts...)) + return delegated.Delete(ctx, object, opts...) + }, + }) + } + + t.Log("Delete only the same replica's Agent and report only our successful deletion") + result := reconcileLPUEviction(t, r, "agent-0") + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"agent-other-replica", "conductor-0"}, remainingLPUEvictionPods(t, c)) + recorded := r.Recorder.(*events.FakeRecorder).Events + if tt.deletedConcurrently { + require.Empty(t, recorded) + } else { + require.Len(t, recorded, 1) + require.Contains(t, <-recorded, "deleted 1 LPU pods") + } + + t.Log("Reconcile the retained deleting trigger without another mutation or warning") + assert.Equal(t, ctrl.Result{}, reconcileLPUEviction(t, r, "agent-0")) + require.Empty(t, recorded) + }) + } +} + +func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { + trigger := newLPUEvictionPod("stage-0-partition-3", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + samePartition := newLPUEvictionPod("stage-1-partition-3", "0", "1", false, "") + otherPartition := newLPUEvictionPod("stage-0-partition-4", "0", "2", false, "") + otherModel := newLPUEvictionPod("stage-0-draft-partition-3", "0", "0", false, "") + otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" + noPartition := newLPUOnlyConductor() + otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + require.NotContains(t, trigger.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + + r, c := newLPUEvictionReconciler( + trigger, + samePartition, + otherPartition, + otherModel, + noPartition, + otherReplica, + ) + + result := reconcileLPUEviction(t, r, "stage-0-partition-3") + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"conductor-0", "replica-1-partition-3", "stage-0-draft-partition-3", "stage-0-partition-4"}, remainingLPUEvictionPods(t, c)) +} + +func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", false, podDisruptionReasonTaintManagerDeletion) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + + r, c := newLPUEvictionReconciler(trigger, sibling) + + result := reconcileLPUEviction(t, r, "agent-0") + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"agent-0", "agent-1"}, remainingLPUEvictionPods(t, c)) + + var updated corev1.Pod + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(trigger), &updated)) + assert.Empty(t, updated.Finalizers) +} + +func TestLPUEviction_PodPredicateEnqueuesOnlyDeletingPods(t *testing.T) { + predicate := (&lpuEvictionReconciler{}).evictionPredicate() + ordinary := newLPUEvictionPod("agent-0", "0", "0", false, "") + deleting := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + + assert.False(t, predicate.Create(event.CreateEvent{Object: ordinary})) + assert.False(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: ordinary.DeepCopy()})) + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: deleting})) +} + +func TestLPUEviction_Noops(t *testing.T) { + tests := []struct { + name string + triggerName string + conditionReason string + phase corev1.PodPhase + statusReason string + missingMode bool + nonLPU bool + conductor bool + }{ + {name: "other disruption reason", conditionReason: "OtherDisruptionReason"}, + {name: "failed evicted without disruption condition", phase: corev1.PodFailed, statusReason: "Evicted"}, + {name: "missing workload mode", conditionReason: podDisruptionReasonTaintManagerDeletion, missingMode: true}, + {name: "non LPX pod", triggerName: "worker-0", conditionReason: podDisruptionReasonTaintManagerDeletion, nonLPU: true}, + {name: "conductor", triggerName: "conductor-0", conditionReason: podDisruptionReasonTaintManagerDeletion, conductor: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Observe a deleting Pod outside the Agent cascade contract") + triggerName := tt.triggerName + if triggerName == "" { + triggerName = "agent-0" + } + trigger := newLPUEvictionPod(triggerName, "0", "0", true, tt.conditionReason) + trigger.Status.Phase = tt.phase + trigger.Status.Reason = tt.statusReason + if tt.nonLPU { + trigger.Labels[commonconsts.KubeLabelDynamoComponentType] = commonconsts.ComponentTypeWorker + } + if tt.conductor { + trigger.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor + } + if tt.missingMode { + delete(trigger.Annotations, dynamolpx.WorkloadModeAnnotation) + } + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, c := newLPUEvictionReconciler(trigger, sibling) + + t.Log("Reconcile without deleting the surviving Agent") + result := reconcileLPUEviction(t, r, trigger.Name) + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"agent-1"}, remainingLPUEvictionPods(t, c)) + }) + } +} + +func TestLPUEviction_MissingGroveLabelsReturnsError(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + delete(trigger.Labels, grovecommon.LabelPodCliqueScalingGroup) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + + r, c := newLPUEvictionReconciler(trigger, sibling) + + result, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: trigger.Name, Namespace: lpuEvictionTestNamespace}, + }) + require.Error(t, err) + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"agent-1"}, remainingLPUEvictionPods(t, c)) +} diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go new file mode 100644 index 000000000000..9732740c0965 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lpx + +import ( + "context" + "fmt" + "strconv" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// reconcileGrovePodCliqueSetForLPX converges the already-rendered PodCliqueSet +// behind the LPX retirement fence. Only existingPodCliqueSet may be nil, when +// the controller observed no PCS; all decisions and writes use that observation. +func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + existingPodCliqueSet *grovev1alpha1.PodCliqueSet, + desired *grovev1alpha1.PodCliqueSet, + selectedLPX *lpxMaterializing, +) (*grovev1alpha1.PodCliqueSet, bool, error) { + // A missing observation permits only creation after the publication fence. + if existingPodCliqueSet == nil { + if err := r.fenceLPXPublicationBeforeGroveSpecWrite(ctx, deployment); err != nil { + return nil, false, err + } + modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired) + return synced, modified, err + } + + // Retire and replace a changed attempt or spec; never update its spec in place. + change, err := commoncontroller.GetSpecChangeResult(existingPodCliqueSet, desired) + if err != nil { + return nil, false, fmt.Errorf("compare cached LPX PodCliqueSet with its successor: %w", err) + } + if change.SpecNeedsUpdate || !selectedLPX.hasCurrentLPXAttemptAnnotations(existingPodCliqueSet, deployment) { + return nil, false, r.recreateStaleLPXPodCliqueSet(ctx, deployment, desired, selectedLPX) + } + + // A current PCS keeps its spec and identity; only owned metadata may change. + if err := commoncontroller.CheckControllerOwnership(existingPodCliqueSet, deployment, r.Scheme()); err != nil { + return nil, false, err + } + prepared := existingPodCliqueSet + if change.NeedsUpdate { + prepared = existingPodCliqueSet.DeepCopy() + metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, *change.NewHash) + metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, commoncontroller.NvidiaAnnotationGenerationKey, strconv.FormatInt(change.NewGeneration, 10)) + } + for _, key := range [...]string{ + dynamolpx.WorkloadDigestAnnotation, + lpxDeploymentUIDAnnotation, + lpxDeploymentGenerationAnnotation, + dynamo.LPXInputRevisionAnnotation, + dynamolpx.DGDUIDAnnotation, + dynamolpx.DGDGenerationAnnotation, + } { + value, found := desired.Annotations[key] + currentValue, currentFound := prepared.Annotations[key] + if found == currentFound && (!found || value == currentValue) { + continue + } + if prepared == existingPodCliqueSet { + prepared = existingPodCliqueSet.DeepCopy() + } + if found { + metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, key, value) + } else { + delete(prepared.Annotations, key) + } + } + if prepared == existingPodCliqueSet { + return existingPodCliqueSet, false, nil + } + + // The observed resource version prevents overwriting any concurrent PCS edit. + if err := r.Update(ctx, prepared); err != nil { + r.GetRecorder().Eventf(existingPodCliqueSet, nil, corev1.EventTypeWarning, "UpdatePodCliqueSet", "Update", "Failed to update metadata for PodCliqueSet %s: %s", desired.Namespace, err) + return nil, false, err + } + r.GetRecorder().Eventf(prepared, nil, corev1.EventTypeNormal, "UpdatePodCliqueSet", "Update", "Updated metadata for PodCliqueSet %s", desired.Namespace) + return prepared, true, nil +} diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go new file mode 100644 index 000000000000..e70617757457 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -0,0 +1,1598 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash" + "slices" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" +) + +const ( + lpxAttemptDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-attempt-digest" + lpxDeploymentGenerationAnnotation = dynamo.LPXDeploymentGenerationAnnotation + lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation + lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" + lpxPCSUIDAnnotation = "scheduling.lpu.nvidia.com/podcliqueset-uid" + lpxPodGangUIDAnnotation = "scheduling.lpu.nvidia.com/podgang-uid" + lpxRetirementRequeueAfter = 5 * time.Second + lpxLifecycleListPageSize = 64 + maxDGDConditionMessageSize = 32768 +) + +const ( + lpxRetiringReason string = "LPXRetiring" + lpxSchedulerStatusInvalidReason string = "LPXSchedulerStatusInvalid" +) + +var errLPXGrovePodCliqueSetRecreating = errors.New("LPX successor is waiting while the stale Grove PodCliqueSet") + +// lpxClassification is deliberately private and closed: reconciliation can +// report only one legal protocol state, never a freely assembled collection +// of correlated booleans. +type lpxClassification interface { + lpxClassification() +} + +type lpxRejected struct { + reason string +} + +type lpxMaterializing struct { + workloadDigest string + requests []lpxModelMaterializing + workload *lpx.SelectedWorkload + plan *lpx.MaterializationPlan + deadlineSeconds *int64 +} + +type reconcileOutcome struct { + State nvidiacomv1beta1.DGDState + Reason string + Message string +} + +// lpxModelMaterializing is the request-local state for one model projection. +// The enclosing lpxMaterializing owns the shared Grove identity. +type lpxModelMaterializing struct { + requestName string + attemptDigest string + modelProjection *lpx.ModelProjection + replicaIndex int32 +} + +type lpxGroveSnapshot struct { + podCliqueSet *grovev1alpha1.PodCliqueSet + scalingGroup *grovev1alpha1.PodCliqueScalingGroup + generationHash string + incomplete string + podGangs []groveschedulerv1alpha1.PodGang +} + +type lpxGroveIdentity struct { + pcsUID types.UID + podGangName string + podGangUID types.UID + cyborgClique *lpxv1alpha1.CyborgPodCliqueReference + complete bool +} + +type lpxClosed struct { + incomplete string +} + +type lpxOpen struct{} + +type lpxBound struct{} + +type lpxSchedulerObserved lpxv1alpha1.LPUPipelineRequestStatus + +type lpxRetiring struct { + retirementReason string +} + +// Error carries retirement through Grove's error-shaped write boundary. +func (*lpxRetiring) Error() string { return "LPX publication retired before Grove spec write" } + +func (*lpxRejected) lpxClassification() {} +func (*lpxClosed) lpxClassification() {} +func (*lpxOpen) lpxClassification() {} +func (*lpxBound) lpxClassification() {} +func (*lpxSchedulerObserved) lpxClassification() {} +func (*lpxRetiring) lpxClassification() {} + +func lpxResult(classification lpxClassification) reconcileOutcome { + result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} + switch state := classification.(type) { + case *lpxRejected: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = "LPXRejected" + result.Message = state.reason + case *lpxClosed: + result.Reason = "LPXPublicationPending" + result.Message = state.incomplete + case *lpxOpen: + result.Reason = "LPXPublished" + result.Message = "LPX request is published" + case *lpxBound: + result.Reason = "AwaitingLPXRuntimeActivation" + result.Message = "LPX scheduling is bound; runtime activation and serving readiness remain external" + case *lpxSchedulerObserved: + result = lpxSchedulerResult(state) + result.Message = lpxMessageWithDiagnostics(result.Message, state.Diagnostics) + case *lpxRetiring: + result.Reason = lpxRetiringReason + result.Message = state.retirementReason + case *lpxDeadlineTransition: + result = state.result + default: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = "LPXInternalClassificationError" + result.Message = "Unknown LPX reconciliation classification" + } + return result +} + +// lpxSchedulerResult classifies the receipt without formatting diagnostics before selection. +func lpxSchedulerResult(state *lpxSchedulerObserved) reconcileOutcome { + result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} + switch state.Phase { + case lpxv1alpha1.RequestPhasePending: + result.Reason = "LPXSchedulerPending" + result.Message = "LPX scheduler is waiting to plan the current request" + case lpxv1alpha1.RequestPhaseNoFit: + result.Reason = "LPXNoFit" + result.Message = "LPX scheduler found no placement for the current request" + case lpxv1alpha1.RequestPhaseUnsupported: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = "LPXUnsupported" + result.Message = "LPX scheduler cannot support the current request" + case lpxv1alpha1.RequestPhasePlanned: + result.Reason = "LPXPlanned" + result.Message = "LPX scheduler committed a placement plan for the current request" + case lpxv1alpha1.RequestPhaseReserving: + result.Reason = "LPXReserving" + result.Message = "LPX scheduler is reserving the planned LPU allocation" + case lpxv1alpha1.RequestPhaseBinding: + result.Reason = "LPXBinding" + result.Message = "LPX scheduler is binding the planned workload Pods" + case lpxv1alpha1.RequestPhaseDegraded: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = "LPXDegraded" + result.Message = "LPX scheduler detected that the committed plan is no longer valid" + case lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: + result = lpxSchedulerReleaseResult(state) + case lpxv1alpha1.RequestPhaseBound: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = lpxSchedulerStatusInvalidReason + result.Message = "LPX scheduler reported Bound without the exact current-generation plan proof required by Dynamo" + default: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = lpxSchedulerStatusInvalidReason + result.Message = fmt.Sprintf("LPX scheduler reported unknown phase %q", state.Phase) + } + return result +} + +func lpxSchedulerReleaseResult(state *lpxSchedulerObserved) reconcileOutcome { + result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} + phase := "releasing" + result.Reason = "LPXReleasing" + result.Message = "LPX scheduler is releasing the current plan" + if state.Phase == lpxv1alpha1.RequestPhaseReleased { + phase = "released" + result.Reason = "LPXReleased" + result.Message = "LPX scheduler released the current plan and is finalizing its journal" + } + if state.Committed == nil || state.Committed.Execution.Release == nil { + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = lpxSchedulerStatusInvalidReason + result.Message = fmt.Sprintf("LPX scheduler reported %s without the required release journal", phase) + return result + } + releaseReason := state.Committed.Execution.Release.Reason + switch releaseReason { + case lpxv1alpha1.ReleaseReasonDependencyChanged, lpxv1alpha1.ReleaseReasonBindingFailed: + result.State = nvidiacomv1beta1.DGDStateFailed + case lpxv1alpha1.ReleaseReasonRequestChanged, lpxv1alpha1.ReleaseReasonRequestDeleted: + default: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = lpxSchedulerStatusInvalidReason + result.Message = fmt.Sprintf("LPX scheduler reported %s with unknown release reason %q", phase, releaseReason) + return result + } + result.Message = fmt.Sprintf("%s after %s", result.Message, releaseReason) + return result +} + +func lpxMessageWithDiagnostics(summary string, diagnostics []lpxv1alpha1.StatusDiagnostic) string { + if len(diagnostics) == 0 { + return summary + } + var message strings.Builder + message.WriteString(summary) + message.WriteString(". LPX diagnostics: ") + for index, diagnostic := range diagnostics { + if index > 0 { + message.WriteString("; ") + } + message.WriteString(diagnostic.Code) + message.WriteString(" [") + message.WriteString(diagnostic.Subject) + message.WriteString("]: ") + message.WriteString(diagnostic.Detail) + + // Later diagnostics cannot change the retained prefix. + if message.Len() > maxDGDConditionMessageSize { + break + } + } + return truncateLPXMessage(message.String()) +} + +func truncateLPXMessage(message string) string { + if len(message) <= maxDGDConditionMessageSize { + return message + } + const suffix = "…" + last := maxDGDConditionMessageSize - len(suffix) + for last > 0 && !utf8.ValidString(message[:last]) { + last-- + } + return message[:last] + suffix +} + +func overlayLPXResult( + base reconcileOutcome, + classification lpxClassification, +) reconcileOutcome { + if _, ok := classification.(*lpxBound); ok { + return base + } + return lpxResult(classification) +} + +// reconcileLPXKnownIntentFence deletes a publication before any new graph can +// be rendered when selection, DGD generation, model, or DGD incarnation changes. +func (r *graphReconciler) reconcileLPXKnownIntentFence( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + requests []lpxv1alpha1.LPUPipelineRequest, +) (lpxClassification, []lpxv1alpha1.LPUPipelineRequest, error) { + var err error + if requests == nil { + requests, err = r.listOwnedLPXRequests(ctx, deployment) + } + if err != nil { + return nil, nil, err + } + selected := source.HasLPXComponent() + + modelNames, modelErr := lpx.SelectedModelNames(source) + for index := range requests { + request := &requests[index] + keep := selected && modelErr == nil && + slices.Contains(modelNames, request.Annotations[lpxModelAnnotation]) && + request.Annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && + request.Annotations[lpxDeploymentGenerationAnnotation] == strconv.FormatInt(deployment.Generation, 10) && + request.Annotations[dynamo.LPXInputRevisionAnnotation] == deployment.Spec.InputRevision && + request.Annotations[lpx.ExecutionBackendAnnotation] == "" + if keep { + continue + } + reason := "LPX scheduler was deselected" + if selected { + reason = "The DGD incarnation, generation, model, or selected input changed" + } + retiring, retireErr := r.retireLPXRequest(ctx, deployment, request, reason) + return retiring, nil, retireErr + } + return nil, requests, nil +} + +func (r *graphReconciler) prepareLPXMaterializing( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, +) (*lpxMaterializing, *lpxRejected, error) { + workload, err := lpx.ResolveSelectedWorkload(ctx, source, r.modelRegistry) + if err != nil { + if errors.Is(err, lpx.ErrBuildSnapshotAcquisition) { + return nil, nil, err + } + return nil, &lpxRejected{reason: err.Error()}, nil + } + projections := workload.ModelProjections() + plan, err := workload.PlanNodeLocalMaterialization(deployment.Annotations[dynamo.LPXPCSNameAnnotation]) + if err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } + workloadDigest := workload.Digest().String() + requests := make([]lpxModelMaterializing, 0, len(projections)*int(plan.Replicas)) + for replicaIndex := int32(0); replicaIndex < plan.Replicas; replicaIndex++ { + for _, projection := range projections { + requestDigest := digestAttemptKey( + deployment.Namespace, + deployment.Name, + deployment.UID, + deployment.Generation, + projection.Model(), + projection.Digest().String(), + replicaIndex, + ) + requests = append(requests, lpxModelMaterializing{ + requestName: lpxRequestName(deployment.Name, requestDigest), + attemptDigest: requestDigest, + modelProjection: projection, + replicaIndex: replicaIndex, + }) + } + } + return &lpxMaterializing{ + workloadDigest: workloadDigest, + requests: requests, + workload: workload, + plan: plan, + deadlineSeconds: lpxAttemptDeadlineSeconds(source), + }, nil, nil +} + +// retireInvalidLPXWorkload also retires staging that never acquired an LPR. +// Healthy publication and spec-write fences intentionally do not use this path. +func (r *graphReconciler) retireInvalidLPXWorkload( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + reason string, +) (*lpxRetiring, error) { + // Preserve request-first retirement before removing unpublished staging. + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil { + return nil, err + } + if len(requests) > 0 { + return r.retireLPXRequest(ctx, deployment, &requests[0], reason) + } + + // Source edits can rename both resources; discover only this exact child's old objects. + owned := make([]client.Object, 0) + for _, list := range []client.ObjectList{&corev1.ServiceList{}, &grovev1alpha1.PodCliqueSetList{}} { + if err := visitLifecycleObjectPages(ctx, r.apiReader, list, func(object k8sruntime.Object) error { + resource := object.(client.Object) + if metav1.IsControlledBy(resource, deployment) { + owned = append(owned, resource) + } + return nil + }, client.InNamespace(deployment.Namespace)); err != nil { + return nil, err + } + } + if len(owned) == 0 { + return nil, nil + } + + // A newer child seen during listing must not lose its staging to this stale failure. + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return nil, err + } + for _, resource := range owned { + if !resource.GetDeletionTimestamp().IsZero() { + continue + } + uid, resourceVersion := resource.GetUID(), resource.GetResourceVersion() + if err := r.Delete(ctx, resource, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ + UID: &uid, ResourceVersion: &resourceVersion, + }}); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + } + return &lpxRetiring{retirementReason: reason}, nil +} + +func digestAttemptKey( + namespace string, + name string, + uid types.UID, + generation int64, + model string, + projectionDigest string, + replicaIndex int32, +) string { + h := sha256.New() + writeLPXHashField(h, "namespace", namespace) + writeLPXHashField(h, "name", name) + writeLPXHashField(h, "uid", string(uid)) + writeLPXHashField(h, "generation", strconv.FormatInt(generation, 10)) + writeLPXHashField(h, "model", model) + // Keep the versioned field tag stable so a naming-only refactor does not + // rotate deterministic LPR names. + writeLPXHashField(h, "compiler-snapshot", projectionDigest) + if replicaIndex > 0 { + writeLPXHashField(h, "group-replica", strconv.FormatInt(int64(replicaIndex), 10)) + } + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +func writeLPXHashField(h hash.Hash, tag, value string) { + var size [8]byte + for _, field := range []string{tag, value} { + binary.BigEndian.PutUint64(size[:], uint64(len(field))) + _, _ = h.Write(size[:]) + _, _ = h.Write([]byte(field)) + } +} + +// hasCurrentLPXAttemptAnnotations requires nonnil desired and its DGD; a nil object interface does not match. +func (desired *lpxMaterializing) hasCurrentLPXAttemptAnnotations( + object metav1.Object, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) bool { + if object == nil || !object.GetDeletionTimestamp().IsZero() { + return false + } + annotations := object.GetAnnotations() + executionBackend := annotations[lpx.ExecutionBackendAnnotation] + return annotations[lpx.WorkloadDigestAnnotation] == desired.workloadDigest && + annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && + annotations[lpxDeploymentGenerationAnnotation] == strconv.FormatInt(deployment.Generation, 10) && + annotations[dynamo.LPXInputRevisionAnnotation] == deployment.Spec.InputRevision && + executionBackend == "" +} + +func lpxRequestName(dgdName, digest string) string { + prefix := strings.ReplaceAll(dgdName, ".", "-") + if len(prefix) > 23 { + prefix = strings.TrimRight(prefix[:23], "-") + } + hexDigest := strings.TrimPrefix(digest, "sha256:") + return fmt.Sprintf("lpx-%s-%s", prefix, hexDigest[:32]) +} + +// reconcileLPXAttemptFence requires a nonnil desired whose requests remain immutable during the call. +func (r *graphReconciler) reconcileLPXAttemptFence( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, +) (map[string]*lpxv1alpha1.LPUPipelineRequest, *lpxRetiring, error) { + // Retain each desired position for digest lookup and collision precedence. + desiredByName := make(map[string]int, len(desired.requests)) + for index := range desired.requests { + desiredByName[desired.requests[index].requestName] = index + } + currents := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) + foreignCollision := len(desired.requests) + var stale *lpxv1alpha1.LPUPipelineRequest + if err := visitLifecycleObjectPages( + ctx, + r.apiReader, + &lpxv1alpha1.LPUPipelineRequestList{}, + func(object k8sruntime.Object) error { + request := object.(*lpxv1alpha1.LPUPipelineRequest) + desiredIndex, desiredName := desiredByName[request.Name] + if !metav1.IsControlledBy(request, deployment) { + // Preserve desired-order error precedence without retaining collision state. + if desiredName { + foreignCollision = min(foreignCollision, desiredIndex) + } + return nil + } + if desiredName && request.Annotations[lpxAttemptDigestAnnotation] == desired.requests[desiredIndex].attemptDigest { + currents[request.Name] = request + return nil + } + if stale == nil || request.Name < stale.Name { + stale = request + } + return nil + }, + client.InNamespace(deployment.Namespace), + ); err != nil { + return nil, nil, err + } + if stale != nil { + retiring, retireErr := r.retireLPXRequest(ctx, deployment, stale, "The immutable build snapshot or complete attempt identity changed") + return nil, retiring, retireErr + } + // A deterministic-name collision must never be adopted or deleted. + if foreignCollision < len(desired.requests) { + return nil, nil, fmt.Errorf("LPX request name %q is occupied by a foreign owner", desired.requests[foreignCollision].requestName) + } + return currents, nil, nil +} + +func visitLifecycleObjectPages( + ctx context.Context, + reader client.Reader, + list client.ObjectList, + visit func(k8sruntime.Object) error, + opts ...client.ListOption, +) error { + continueToken := "" + pageOptions := make([]client.ListOption, 0, len(opts)+2) + for { + page := list.DeepCopyObject().(client.ObjectList) + pageOptions = pageOptions[:0] + pageOptions = append(pageOptions, opts...) + pageOptions = append(pageOptions, client.Limit(lpxLifecycleListPageSize)) + if continueToken != "" { + pageOptions = append(pageOptions, client.Continue(continueToken)) + } + if err := reader.List(ctx, page, pageOptions...); err != nil { + return err + } + if err := meta.EachListItem(page, visit); err != nil { + return err + } + nextToken := page.GetContinue() + if nextToken == "" { + break + } + if nextToken == continueToken { + return fmt.Errorf("lifecycle list %T repeated continuation token %q", list, nextToken) + } + continueToken = nextToken + } + return nil +} + +func (r *graphReconciler) listOwnedLPXRequests( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) ([]lpxv1alpha1.LPUPipelineRequest, error) { + requests, available, err := r.listOwnedLPXRequestsIfAvailable(ctx, deployment) + if err == nil && !available { + return nil, fmt.Errorf("LPX request API is unavailable after startup discovery") + } + return requests, err +} + +func (r *graphReconciler) listOwnedLPXRequestsIfAvailable( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) ([]lpxv1alpha1.LPUPipelineRequest, bool, error) { + list := &lpxv1alpha1.LPUPipelineRequestList{} + owned := make([]lpxv1alpha1.LPUPipelineRequest, 0) + visit := func(object k8sruntime.Object) error { + request := object.(*lpxv1alpha1.LPUPipelineRequest) + if metav1.IsControlledBy(request, deployment) { + owned = append(owned, *request) + } + return nil + } + err := visitLifecycleObjectPages( + ctx, + r.apiReader, + list, + visit, + client.InNamespace(deployment.Namespace), + ) + if err != nil { + if meta.IsNoMatchError(err) || k8sruntime.IsNotRegisteredError(err) { + return nil, false, nil + } + return nil, false, err + } + sort.Slice(owned, func(i, j int) bool { return owned[i].Name < owned[j].Name }) + return owned, true, nil +} + +// retireFirstOwnedLPXRequest uses the sorted authoritative first request as the retirement witness. +func (r *graphReconciler) retireFirstOwnedLPXRequest( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + reason string, +) (*lpxRetiring, error) { + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil || len(requests) == 0 { + return nil, err + } + return r.retireLPXRequest(ctx, deployment, &requests[0], reason) +} + +func (r *graphReconciler) retireLPXRequest( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + request *lpxv1alpha1.LPUPipelineRequest, + reason string, +) (*lpxRetiring, error) { + if request == nil || !metav1.IsControlledBy(request, deployment) { + return nil, fmt.Errorf("refusing to retire an LPX request without the exact LPXGraphDeployment controller owner") + } + if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { + return nil, err + } + + // Read the complete publication set directly from the API before changing shared Grove state. + type requestDeletePrecondition struct { + name string + uid types.UID + resourceVersion string + } + owned := make([]requestDeletePrecondition, 0) + if err := visitLifecycleObjectPages( + ctx, + r.apiReader, + &lpxv1alpha1.LPUPipelineRequestList{}, + func(object k8sruntime.Object) error { + live := object.(*lpxv1alpha1.LPUPipelineRequest) + if metav1.IsControlledBy(live, deployment) && live.DeletionTimestamp.IsZero() { + owned = append(owned, requestDeletePrecondition{ + name: live.Name, + uid: live.UID, + resourceVersion: live.ResourceVersion, + }) + } + return nil + }, + client.InNamespace(deployment.Namespace), + ); err != nil { + return nil, fmt.Errorf("list the exact LPXGraphDeployment-owned LPX requests for attempt-wide retirement: %w", err) + } + + // Delete every live publication with exact-object preconditions before scaling the shared PCS. + for _, current := range owned { + live := &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{ + Namespace: deployment.Namespace, + Name: current.name, + UID: current.uid, + ResourceVersion: current.resourceVersion, + }} + if err := r.Delete(ctx, live, &client.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: ¤t.uid, ResourceVersion: ¤t.resourceVersion}, + }); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("delete exact LPXGraphDeployment-owned LPX request %q: %w", current.name, err) + } + } + + // Hold the shared Grove attempt at zero only after every publication is retiring. + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, ""); err != nil { + return nil, fmt.Errorf("scale down the LPX PodCliqueSet after retiring all attempt requests: %w", err) + } + return &lpxRetiring{ + retirementReason: reason, + }, nil +} + +// scaleDownLPXPodCliqueSet holds the complete Grove attempt at zero while LPX +// releases its deleted requests. LPX requires every exact old workload Pod to be +// absent and treats a same-name replacement UID as a cleanup blocker. +// Grove supports mutable PodCliqueSet scale-to-zero and deletes every child +// replica without changing the immutable clique composition. +func (r *graphReconciler) scaleDownLPXPodCliqueSet( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsUID types.UID, +) error { + key := types.NamespacedName{ + Namespace: deployment.Namespace, + Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation], + } + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, key, pcs); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if pcsUID != "" && pcs.UID != pcsUID { + return nil + } + if !metav1.IsControlledBy(pcs, deployment) { + return fmt.Errorf("refusing to scale down PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + } + if !pcs.DeletionTimestamp.IsZero() || pcs.Spec.Replicas == 0 { + return nil + } + pcs.Spec.Replicas = 0 + return r.Update(ctx, pcs) +} + +func (r *graphReconciler) reconcileSelectedLPX( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, +) (lpxClassification, error) { + currents := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) + + // Index the desired publication names before listing current requests. + desiredNames := make(map[string]struct{}, len(desired.requests)) + for _, request := range desired.requests { + desiredNames[request.requestName] = struct{}{} + } + + // Collect only exact LPXGraphDeployment-owned publications for the desired attempt. + if err := visitLifecycleObjectPages( + ctx, + r.apiReader, + &lpxv1alpha1.LPUPipelineRequestList{}, + func(object k8sruntime.Object) error { + live := object.(*lpxv1alpha1.LPUPipelineRequest) + if _, desired := desiredNames[live.Name]; !desired { + return nil + } + if !metav1.IsControlledBy(live, deployment) { + return fmt.Errorf("refusing to adopt foreign LPX request %q", live.Name) + } + currents[live.Name] = live + return nil + }, + client.InNamespace(deployment.Namespace), + ); err != nil { + return nil, err + } + return r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents, true) +} + +//nolint:gocyclo // Current-request retirement must remain visibly ordered before Grove observation. +func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, + currents map[string]*lpxv1alpha1.LPUPipelineRequest, + allowPublication bool, +) (lpxClassification, error) { + if currents == nil { + currents = make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) + } + + // Retire invalid live requests before observing their Grove dependencies. + for _, request := range desired.requests { + live := currents[request.requestName] + if live == nil { + continue + } + if sizeErr := lpx.ValidateRequestSize(live); sizeErr != nil { + return r.retireLPXRequest( + ctx, + deployment, + live, + fmt.Sprintf("LPX request exceeds the scheduler size budget: %v", sizeErr), + ) + } + if !live.DeletionTimestamp.IsZero() { + return r.retireLPXRequest( + ctx, + deployment, + live, + "Waiting for the current LPX request finalizer to finish cleanup", + ) + } + } + groveIdentities, pending, classification, err := r.reconcileSelectedLPXGroveIdentityPrefix( + ctx, + deployment, + desired, + currents, + ) + if classification != nil || err != nil { + return classification, err + } + + // Validate staged identities without crossing the scheduler-publication boundary. + if !allowPublication { + for index := range desired.requests { + request := &desired.requests[index] + identity := groveIdentities[request.replicaIndex] + if identity != nil && identity.complete && currents[request.requestName] == nil { + return &lpxClosed{ + incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization", + }, nil + } + } + if pending != nil { + return pending, nil + } + } + + // Publish missing requests in the fully observed prefix through one immutable attempt projection. + var attemptProjection *nvidiacomv1beta1.LPXAttemptStatus + if baseGroveIdentity := groveIdentities[0]; baseGroveIdentity != nil && baseGroveIdentity.complete { + if desired.deadlineSeconds != nil { + attemptProjection = desiredLPXAttemptStatus(desired, deployment.Generation, baseGroveIdentity.pcsUID) + } + transition, err := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) + if err != nil { + return nil, err + } + if transition != nil { + return transition, nil + } + } + for index := range desired.requests { + request := &desired.requests[index] + groveIdentity := groveIdentities[request.replicaIndex] + if currents[request.requestName] != nil || groveIdentity == nil || !groveIdentity.complete { + continue + } + if attemptProjection != nil { + if err := r.revalidateLPXAttemptPublication(ctx, deployment); err != nil { + return nil, err + } + } + live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity, desired.deadlineSeconds != nil) + if createErr != nil { + return nil, createErr + } + currents[request.requestName] = live + transition, transitionErr := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) + if transitionErr != nil { + return nil, transitionErr + } + if transition != nil { + return transition, nil + } + } + + // Stream terminal, side-effect-free classifications in request order. + var firstNonBound lpxClassification + for _, request := range desired.requests { + if live := currents[request.requestName]; live != nil { + classification := classifyPublishedLPX(live) + if state, ok := classification.(*lpxSchedulerObserved); ok && lpxSchedulerResult(state).State == nvidiacomv1beta1.DGDStateFailed { + return classification, nil + } + if _, bound := classification.(*lpxBound); !bound && firstNonBound == nil { + firstNonBound = classification + } + } + } + + // Keep pending behind every published non-bound request. + if firstNonBound != nil { + return firstNonBound, nil + } + if pending != nil { + return pending, nil + } + return &lpxBound{}, nil +} + +func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, + currents map[string]*lpxv1alpha1.LPUPipelineRequest, +) ([]*lpxGroveIdentity, *lpxClosed, lpxClassification, error) { + snapshot, incomplete, err := r.observeLPXGroveSnapshot(ctx, deployment, desired, len(currents) == 0) + if err != nil { + return nil, nil, nil, err + } + if snapshot == nil { + if len(currents) == 0 { + return nil, nil, &lpxClosed{incomplete: incomplete}, nil + } + + // Use the first published request as the witness for attempt-wide retirement when the shared Grove snapshot disappears. + reason := fmt.Sprintf("The published Grove identity became unavailable: %s", incomplete) + firstName := "" + for name := range currents { + if firstName == "" || name < firstName { + firstName = name + } + } + retiring, err := r.retireLPXRequest(ctx, deployment, currents[firstName], reason) + return nil, nil, retiring, err + } + + groveIdentities := make([]*lpxGroveIdentity, desired.plan.Replicas) + replicaPlans := make([]*lpx.MaterializationPlan, desired.plan.Replicas) + allAgentGroups := make(map[string]struct{}, int(desired.plan.Replicas)*len(desired.plan.Agents)) + for replicaIndex := int32(0); replicaIndex < desired.plan.Replicas; replicaIndex++ { + plan := desired.plan.ForReplica(replicaIndex) + replicaPlans[replicaIndex] = plan + for _, agent := range plan.Agents { + allAgentGroups[agent.CliqueName] = struct{}{} + } + } + var pending *lpxClosed + for replicaIndex, plan := range replicaPlans { + identity, incomplete, err := r.observeLPXGroveIdentity(ctx, deployment, desired, snapshot, plan, allAgentGroups) + if err != nil { + return nil, nil, nil, err + } + if incomplete != "" { + if pending == nil { + pending = &lpxClosed{incomplete: incomplete} + } + } + if identity != nil { + identity.complete = pending == nil + groveIdentities[replicaIndex] = identity + } + } + + // Fence exact PCS, PodGang, and Cyborg replacement identities. + for index := range desired.requests { + request := &desired.requests[index] + live := currents[request.requestName] + if live == nil { + continue + } + identity := groveIdentities[request.replicaIndex] + if identity == nil { + reason := "The published replica no longer has its exact Grove identity" + if pending != nil && pending.incomplete != "" { + reason = fmt.Sprintf("%s: %s", reason, pending.incomplete) + } + retiring, err := r.retireLPXRequest(ctx, deployment, live, reason) + return nil, nil, retiring, err + } + if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { + retiring, retireErr := r.retireLPXRequest(ctx, deployment, live, err.Error()) + return nil, nil, retiring, retireErr + } + } + return groveIdentities, pending, nil, nil +} + +// classifyPublishedLPX classifies a non-nil published request from its current scheduler receipt. +func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassification { + // Expose only scheduler status observed for the current request generation. + status := current.Status + if current.Generation <= 0 || status == nil || status.ObservedGeneration == nil || *status.ObservedGeneration != current.Generation { + return &lpxOpen{} + } + + // Bound requires one complete committed receipt for this exact request and plan. + committed := status.Committed + if status.Phase == lpxv1alpha1.RequestPhaseBound && committed != nil && + committed.Execution.AcceptedGeneration != nil && *committed.Execution.AcceptedGeneration == current.Generation && + committed.Plan.PlannedFromGeneration == current.Generation && + committed.Plan.Revision > 0 && committed.Plan.Revision == status.LastPlanRevision && + committed.Plan.PlanDigest != "" && committed.Plan.Placement.ExecutionBackend == current.Spec.ExecutionBackend && + current.Spec.ExecutionBackend == lpxv1alpha1.ExecutionBackendNodeLocal && + committed.Plan.Placement.NodeLocal != nil && committed.Plan.Placement.RemoteDra == nil && + committed.Execution.NodeLocal != nil && committed.Execution.RemoteDra == nil { + return &lpxBound{} + } + return (*lpxSchedulerObserved)(status) +} + +func (r *graphReconciler) fenceLPXPublicationBeforeGroveSpecWrite( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) error { + if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { + return err + } + retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") + if err == nil && retiring != nil { + err = retiring + } + return err +} + +// recreateStaleLPXPodCliqueSet gives a successor LPX publication a fresh Grove +// identity whenever the selected attempt changed or its Grove spec needs a +// write. Grove permits rollout updates to mutable template fields, but rejects +// changes to clique composition and several scheduling fields. Treating the +// whole retired spec as replace-only avoids duplicating that evolving admission +// contract in Dynamo. +// +// The old request is the scheduler's publication boundary. Deletion is allowed +// only after every exactly-owned request is authoritatively gone and the +// exactly-owned PCS is held at zero, which is also the point at which LPX has +// finished releasing the old workload Pod cohort. +// desiredPCS and desired are non-nil; the caller observed a stale PCS. +func (r *graphReconciler) recreateStaleLPXPodCliqueSet( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desiredPCS *grovev1alpha1.PodCliqueSet, + desired *lpxMaterializing, +) error { + // Retire all exactly-owned requests before changing shared Grove state. + retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, "LPX publication was retired before recreating its Grove PodCliqueSet") + if err != nil { + return err + } + if retiring != nil { + return retiring + } + + key := types.NamespacedName{ + Namespace: deployment.Namespace, + Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation], + } + live := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, key, live); err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("%w is observed deleted", errLPXGrovePodCliqueSetRecreating) + } + return err + } + if !metav1.IsControlledBy(live, deployment) { + return fmt.Errorf("refusing to recreate PodCliqueSet %q without the exact LPXGraphDeployment controller owner", live.Name) + } + change, err := commoncontroller.GetSpecChangeResult(live, desiredPCS) + if err != nil { + return fmt.Errorf("compare authoritative LPX PodCliqueSet with its successor: %w", err) + } + if desired.hasCurrentLPXAttemptAnnotations(live, deployment) && !change.SpecNeedsUpdate { + // The authoritative object is current and the controller cache is not. + return fmt.Errorf("%w is observed by the controller cache", errLPXGrovePodCliqueSetRecreating) + } + if !live.DeletionTimestamp.IsZero() { + return fmt.Errorf("%w is deleted", errLPXGrovePodCliqueSetRecreating) + } + if live.Spec.Replicas != 0 { + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, ""); err != nil { + return err + } + return fmt.Errorf("%w is scaled to zero", errLPXGrovePodCliqueSetRecreating) + } + + uid := live.UID + resourceVersion := live.ResourceVersion + if err := r.Delete(ctx, live, &client.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}, + }); err != nil && !apierrors.IsNotFound(err) { + return err + } + return fmt.Errorf("%w is deleted", errLPXGrovePodCliqueSetRecreating) +} + +func (r *graphReconciler) createPublishedLPXRequest( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + materializing *lpxModelMaterializing, + grove *lpxGroveIdentity, + recordAttempt bool, +) (*lpxv1alpha1.LPUPipelineRequest, error) { + if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { + return nil, err + } + spec := materializing.modelProjection.RequestSpec(deployment.Namespace, grove.podGangName, grove.cyborgClique) + request := &lpxv1alpha1.LPUPipelineRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: lpxv1alpha1.GroupVersion.String(), + Kind: "LpuPipelineRequest", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: materializing.requestName, + Namespace: deployment.Namespace, + Labels: map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: deployment.Name, + }, + Annotations: lpxRequestAnnotations(deployment, materializing, grove), + }, + Spec: spec, + } + if recordAttempt { + controllerutil.AddFinalizer(request, lpxAttemptRecordingFinalizer) + } + if err := controllerutil.SetControllerReference(deployment, request, r.Client.Scheme()); err != nil { + return nil, err + } + if err := lpx.ValidateRequestSize(request); err != nil { + return nil, fmt.Errorf("constructed LPX request exceeds the scheduler budget: %w", err) + } + if err := r.Create(ctx, request); err != nil { + return nil, fmt.Errorf( + "create LPX request %q for model %q: %w", + request.Name, + materializing.modelProjection.Model(), + err, + ) + } + return request, nil +} + +func lpxRequestAnnotations( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + materializing *lpxModelMaterializing, + grove *lpxGroveIdentity, +) map[string]string { + // Callers validate the exact source controller owner before constructing requests. + sourceOwner := metav1.GetControllerOf(deployment) + return map[string]string{ + lpxAttemptDigestAnnotation: materializing.attemptDigest, + dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, + lpx.DGDUIDAnnotation: string(sourceOwner.UID), + lpx.DGDGenerationAnnotation: deployment.Annotations[lpx.DGDGenerationAnnotation], + lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), + lpxDeploymentUIDAnnotation: string(deployment.UID), + lpxModelAnnotation: materializing.modelProjection.Model(), + lpxPCSUIDAnnotation: string(grove.pcsUID), + lpxPodGangUIDAnnotation: string(grove.podGangUID), + lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), + } +} + +func validateCurrentLPXRequest( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxModelMaterializing, + grove *lpxGroveIdentity, + request *lpxv1alpha1.LPUPipelineRequest, +) error { + if request.UID == "" { + return fmt.Errorf("LPX request %q has no immutable UID", request.Name) + } + wantAnnotations := lpxRequestAnnotations(deployment, desired, grove) + for key, value := range wantAnnotations { + if request.Annotations[key] != value { + return fmt.Errorf("LPX request %q immutable annotation %q no longer matches the current attempt", request.Name, key) + } + } + wantSpec := desired.modelProjection.RequestSpec(deployment.Namespace, grove.podGangName, grove.cyborgClique) + if !apiequality.Semantic.DeepEqual(request.Spec, wantSpec) { + return fmt.Errorf("LPX request %q immutable intent no longer matches the current attempt", request.Name) + } + return nil +} + +func (r *graphReconciler) observeLPXGroveSnapshot( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, + noCurrentRequests bool, +) (*lpxGroveSnapshot, string, error) { + pcs, incomplete, err := r.observeLPXPodCliqueSet(ctx, deployment, desired, noCurrentRequests) + if err != nil || pcs == nil { + return nil, incomplete, err + } + generationHash, wait := observedLPXPCSGenerationHash(pcs) + if wait != "" { + if incomplete == "" { + incomplete = wait + } + } + podGangs, err := r.listLPXPublicationPodGangs(ctx, deployment, pcs) + if err != nil { + return nil, "", err + } + + // Every resolved runtime has one serving clique in this scaling group. + scalingGroupName := desired.plan.LPXScalingGroup + scalingGroup := &grovev1alpha1.PodCliqueScalingGroup{} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: scalingGroupName} + if err := r.apiReader.Get(ctx, key, scalingGroup); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Sprintf("Waiting for current LPU PodCliqueScalingGroup %q", scalingGroupName), nil + } + return nil, "", err + } + if scalingGroup.UID == "" || !scalingGroup.DeletionTimestamp.IsZero() || + !metav1.IsControlledBy(scalingGroup, pcs) { + return nil, fmt.Sprintf( + "LPU PodCliqueScalingGroup %q lacks the current PodCliqueSet owner identity", + scalingGroupName, + ), nil + } + return &lpxGroveSnapshot{ + podCliqueSet: pcs, + scalingGroup: scalingGroup, + generationHash: generationHash, + incomplete: incomplete, + podGangs: podGangs, + }, "", nil +} + +func (r *graphReconciler) observeLPXGroveIdentity( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, + snapshot *lpxGroveSnapshot, + plan *lpx.MaterializationPlan, + allAgentGroups map[string]struct{}, +) (*lpxGroveIdentity, string, error) { + lpxPodGang, ordinaryPodGang, wait := selectLPXPublicationPodGangs( + deployment.Namespace, + snapshot.podGangs, + plan, + allAgentGroups, + ) + if wait != "" { + return nil, wait, nil + } + + identity := &lpxGroveIdentity{ + pcsUID: snapshot.podCliqueSet.UID, + podGangName: lpxPodGang.Name, + podGangUID: lpxPodGang.UID, + } + incomplete := snapshot.incomplete + if !desired.hasCurrentLPXAttemptAnnotations(lpxPodGang, deployment) || + !desired.hasCurrentLPXAttemptAnnotations(ordinaryPodGang, deployment) { + if incomplete == "" { + incomplete = "Waiting for Grove to publish current-generation PodGangs" + } + } + + // A conductor needs complete identity; Cyborg can retain its reference while hashes converge. + cyborg, wait, err := r.observeLPXServingClique( + ctx, + deployment, + snapshot, + ordinaryPodGang.Name, + desired, + plan, + ) + if err != nil || (wait != "" && cyborg == nil) { + return nil, wait, err + } + identity.cyborgClique = cyborg + if wait != "" { + if incomplete == "" { + incomplete = wait + } + } + return identity, incomplete, nil +} + +func (r *graphReconciler) listLPXPublicationPodGangs( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, +) ([]groveschedulerv1alpha1.PodGang, error) { + podGangs := make([]groveschedulerv1alpha1.PodGang, 0, 2) + if err := visitLifecycleObjectPages( + ctx, + r.apiReader, + &groveschedulerv1alpha1.PodGangList{}, + func(object k8sruntime.Object) error { + podGang := object.(*groveschedulerv1alpha1.PodGang) + schedulerName := podGang.Labels[grovecommon.LabelSchedulerName] + if (schedulerName != lpx.SchedulerName && schedulerName != corev1.DefaultSchedulerName) || + podGang.UID == "" || !podGang.DeletionTimestamp.IsZero() || + !metav1.IsControlledBy(podGang, pcs) { + return nil + } + podGangs = append(podGangs, *podGang) + return nil + }, + client.InNamespace(deployment.Namespace), + client.MatchingLabels{ + grovecommon.LabelPartOfKey: pcs.Name, + consts.KubeLabelDynamoGraphDeploymentName: deployment.Name, + }, + ); err != nil { + return nil, err + } + return podGangs, nil +} + +func (r *graphReconciler) observeLPXPodCliqueSet( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, + noCurrentRequests bool, +) (*grovev1alpha1.PodCliqueSet, string, error) { + pcs := &grovev1alpha1.PodCliqueSet{} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]} + if err := r.apiReader.Get(ctx, key, pcs); err != nil { + if apierrors.IsNotFound(err) { + return nil, "Waiting for the selected Grove PodCliqueSet", nil + } + return nil, "", err + } + if noCurrentRequests && !metav1.IsControlledBy(pcs, deployment) { + return nil, "", fmt.Errorf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + } + if noCurrentRequests && pcs.Spec.Replicas == 0 { + return nil, "The retired Grove attempt is held at zero; successor publication waits for post-retirement Grove synchronization", nil + } + if pcs.UID == "" || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) { + return nil, "The selected Grove PodCliqueSet lacks the exact LPXGraphDeployment owner identity", nil + } + + incomplete := "" + if !desired.hasCurrentLPXAttemptAnnotations(pcs, deployment) { + incomplete = "The selected Grove PodCliqueSet is stale" + } + if pcs.Spec.Replicas != 1 { + if incomplete == "" { + incomplete = fmt.Sprintf("The selected Grove PodCliqueSet has %d replicas, want 1", pcs.Spec.Replicas) + } + } + return pcs, incomplete, nil +} + +func observedLPXPCSGenerationHash(pcs *grovev1alpha1.PodCliqueSet) (string, string) { + if pcs.Status.ObservedGeneration == nil || *pcs.Status.ObservedGeneration < pcs.Generation { + return "", fmt.Sprintf( + "Waiting for Grove to observe PodCliqueSet %q generation %d", + pcs.Name, + pcs.Generation, + ) + } + if pcs.Status.CurrentGenerationHash == nil || *pcs.Status.CurrentGenerationHash == "" { + return "", fmt.Sprintf( + "Waiting for Grove to publish PodCliqueSet %q generation identity", + pcs.Name, + ) + } + return *pcs.Status.CurrentGenerationHash, "" +} + +func hasCurrentLPXScalingGroupOwner( + clique *grovev1alpha1.PodClique, + scalingGroup *grovev1alpha1.PodCliqueScalingGroup, +) bool { + owner := metav1.GetControllerOf(clique) + return owner != nil && owner.APIVersion == grovev1alpha1.SchemeGroupVersion.String() && + owner.Kind == "PodCliqueScalingGroup" && owner.Name == scalingGroup.Name && + owner.UID == scalingGroup.UID +} + +func (r *graphReconciler) observeLPXServingClique( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + snapshot *lpxGroveSnapshot, + ordinaryPodGangName string, + desired *lpxMaterializing, + plan *lpx.MaterializationPlan, +) (*lpxv1alpha1.CyborgPodCliqueReference, string, error) { + // Every supported plan has exactly one ordinary-scheduler serving clique. + isCyborg := plan.CyborgClique != "" + name, role := plan.ConductorClique, lpxv1alpha1.PodRoleConductor + roleName, waitingRoleName := "Conductor", "conductor" + if isCyborg { + name, role = plan.CyborgClique, lpxv1alpha1.PodRoleCyborgWorker + roleName, waitingRoleName = "Cyborg", "Cyborg" + } + + // Read the single serving witness and validate its shared scheduler identity. + clique := &grovev1alpha1.PodClique{} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: name} + if err := r.apiReader.Get(ctx, key, clique); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Sprintf("Waiting for %s PodClique %q", waitingRoleName, name), nil + } + return nil, "", err + } + if clique.UID == "" || + clique.Labels[grovecommon.LabelPartOfKey] != snapshot.podCliqueSet.Name || + clique.Labels[grovecommon.LabelPodCliqueScalingGroup] != plan.LPXScalingGroup || + clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] != strconv.FormatInt(int64(plan.ReplicaIndex), 10) || + clique.Labels[grovecommon.LabelPodGang] != ordinaryPodGangName || + clique.Spec.PodSpec.SchedulerName != corev1.DefaultSchedulerName || + clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != role || + !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || + (isCyborg && !hasLPXCyborgDeviceIntent(clique.Spec.PodSpec)) || + (!isCyborg && !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup)) { + return nil, fmt.Sprintf( + "%s PodClique %q does not match the current ordinary-scheduler identity", + roleName, name, + ), nil + } + if !isCyborg { + return nil, "", nil + } + + // Cyborg publication additionally binds device claims and exact Grove revision hashes. + if !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup) { + return nil, fmt.Sprintf( + "Cyborg PodClique %q lacks the current LPU scaling-group owner identity", + clique.Name, + ), nil + } + if !hasExpectedLPXCyborgClaimReferences(snapshot.podCliqueSet, clique, plan.CyborgTemplate) { + return nil, fmt.Sprintf( + "Cyborg PodClique %q DRA Claim references changed during Grove materialization", + clique.Name, + ), nil + } + reference := &lpxv1alpha1.CyborgPodCliqueReference{Name: clique.Name, UID: string(clique.UID)} + + // Require Grove's selected PCS revision and pod-template identities. Deliberately + // do not require Cyborg ObservedGeneration: LPX must remain able to publish the + // provision request before Grove advances that compatibility signal. + if clique.Status.CurrentPodCliqueSetGenerationHash == nil || + *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash || + clique.Status.CurrentPodTemplateHash == nil || + *clique.Status.CurrentPodTemplateHash == "" { + return reference, fmt.Sprintf( + "Waiting for Grove to materialize the current PodCliqueSet generation in Cyborg PodClique %q", + clique.Name, + ), nil + } + return reference, "", nil +} + +func selectLPXPublicationPodGangs( + namespace string, + podGangs []groveschedulerv1alpha1.PodGang, + plan *lpx.MaterializationPlan, + allAgentGroups map[string]struct{}, +) (*groveschedulerv1alpha1.PodGang, *groveschedulerv1alpha1.PodGang, string) { + var lpxCandidate, ordinaryCandidate *groveschedulerv1alpha1.PodGang + var lpxCandidateCount, ordinaryCandidateCount int + ordinaryGroup := plan.ConductorClique + if ordinaryGroup == "" { + ordinaryGroup = plan.CyborgClique + } + // Count every matching gang before reporting either scheduler's ambiguity. + for index := range podGangs { + podGang := &podGangs[index] + switch podGang.Labels[grovecommon.LabelSchedulerName] { + case lpx.SchedulerName: + if hasExactLPXAgentRows(namespace, podGang, plan) { + lpxCandidate = podGang + lpxCandidateCount++ + } + case corev1.DefaultSchedulerName: + containsAgent, containsOrdinaryGroup := false, ordinaryGroup == "" + for _, group := range podGang.Spec.PodGroups { + if _, agent := allAgentGroups[group.Name]; agent { + containsAgent = true + } + if group.Name == ordinaryGroup { + containsOrdinaryGroup = true + } + } + if !containsAgent && containsOrdinaryGroup { + ordinaryCandidate = podGang + ordinaryCandidateCount++ + } + } + } + if lpxCandidateCount != 1 { + return nil, nil, fmt.Sprintf( + "Waiting for one replica-%d LPX PodGang; found %d candidates", + plan.ReplicaIndex, + lpxCandidateCount, + ) + } + if ordinaryCandidateCount != 1 { + return nil, nil, fmt.Sprintf( + "Waiting for one replica-%d ordinary-scheduler PodGang; found %d candidates", + plan.ReplicaIndex, + ordinaryCandidateCount, + ) + } + return lpxCandidate, ordinaryCandidate, "" +} + +func hasExactLPXAgentRows( + namespace string, + podGang *groveschedulerv1alpha1.PodGang, + plan *lpx.MaterializationPlan, +) bool { + if len(podGang.Spec.PodGroups) != len(plan.Agents) { + return false + } + groups := make(map[string]*groveschedulerv1alpha1.PodGroup, len(podGang.Spec.PodGroups)) + for index := range podGang.Spec.PodGroups { + group := &podGang.Spec.PodGroups[index] + if _, duplicate := groups[group.Name]; duplicate { + return false + } + groups[group.Name] = group + } + for _, agent := range plan.Agents { + group, found := groups[agent.CliqueName] + if !found || group.MinReplicas != int32(agent.Replicas) || len(group.PodReferences) != agent.Replicas { + return false + } + seen := make(map[types.NamespacedName]struct{}, agent.Replicas) + for _, reference := range group.PodReferences { + key := types.NamespacedName{Namespace: reference.Namespace, Name: reference.Name} + if reference.Namespace != namespace || strings.TrimSpace(reference.Name) == "" { + return false + } + if _, duplicate := seen[key]; duplicate { + return false + } + seen[key] = struct{}{} + } + } + return true +} + +func hasExpectedLPXCyborgClaimReferences( + pcs *grovev1alpha1.PodCliqueSet, + cyborg *grovev1alpha1.PodClique, + templateName string, +) bool { + var expected *corev1.PodSpec + for _, template := range pcs.Spec.Template.Cliques { + if template == nil || template.Name != templateName { + continue + } + if expected != nil { + return false + } + expected = &template.Spec.PodSpec + } + if expected == nil { + return false + } + observed := &cyborg.Spec.PodSpec + return apiequality.Semantic.DeepEqual(podLevelClaimReferences(expected), podLevelClaimReferences(observed)) && + apiequality.Semantic.DeepEqual(expected.ResourceClaims, observed.ResourceClaims) && + containerClaimReferencesEqual(expected.Containers, observed.Containers) && + containerClaimReferencesEqual(expected.InitContainers, observed.InitContainers) +} + +func podLevelClaimReferences(spec *corev1.PodSpec) []corev1.ResourceClaim { + if spec.Resources == nil { + return nil + } + return spec.Resources.Claims +} + +func containerClaimReferencesEqual(expected, observed []corev1.Container) bool { + if len(expected) != len(observed) { + return false + } + for index := range expected { + if expected[index].Name != observed[index].Name || + !apiequality.Semantic.DeepEqual(expected[index].Resources.Claims, observed[index].Resources.Claims) { + return false + } + } + return true +} + +func hasLPXCyborgDeviceIntent(podSpec corev1.PodSpec) bool { + if len(podSpec.ResourceClaims) != 0 { + return true + } + for _, container := range podSpec.Containers { + if container.Name != consts.MainContainerName && container.Name != "cyborg" { + continue + } + gpuCount, err := lpx.EffectiveCyborgGPUCount(container.Resources) + if err != nil { + return false + } + if gpuCount > 0 { + return true + } + } + return false +} + +// finalizeLPXRequests retires every exact owned publication for the shared Grove +// attempt. A wait error keeps the DGD finalizer until LPX's own cleanup +// finalizers have completed. +func (r *graphReconciler) finalizeLPXRequests( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) error { + requests, _, err := r.listOwnedLPXRequestsIfAvailable(ctx, deployment) + + // Stop when listing fails or no publication remains. + if err != nil || len(requests) == 0 { + return err + } + + // Start or continue retirement from the authoritative first request. + if _, err := r.retireLPXRequest(ctx, deployment, &requests[0], "LPXGraphDeployment is being deleted"); err != nil { + return err + } + return fmt.Errorf("waiting for LPX request %q to finish fail-closed retirement", requests[0].Name) +} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go new file mode 100644 index 000000000000..e63d27d7e218 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -0,0 +1,2459 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "errors" + "fmt" + "maps" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +const lpxTestOtherName = "other" + +func newLPXTestScheme(t testing.TB) *k8sruntime.Scheme { + t.Helper() + scheme := k8sruntime.NewScheme() + for _, add := range []func(*k8sruntime.Scheme) error{ + corev1.AddToScheme, resourcev1.AddToScheme, nvidiacomv1alpha1.AddToScheme, nvidiacomv1beta1.AddToScheme, + grovev1alpha1.AddToScheme, groveschedulerv1alpha1.AddToScheme, lpxv1alpha1.AddToScheme, + } { + require.NoError(t, add(scheme)) + } + return scheme +} + +type snapshotFailureRegistry struct { + *lpx.ModelRegistry + err error +} + +type downloadOrderedLPXRegistry struct { + *lpx.ModelRegistry + buildURL url.URL + downloaded bool + calls []string +} + +func (r *snapshotFailureRegistry) AcquireBuildSnapshot(context.Context, string) (*lpx.BuildSnapshot, error) { + return nil, r.err +} + +func (r *downloadOrderedLPXRegistry) BuildURL(string) (*url.URL, error) { + buildURL := r.buildURL + return &buildURL, nil +} + +func (r *downloadOrderedLPXRegistry) EnsureDownloaded(context.Context, url.URL) (bool, error) { + r.calls = append(r.calls, "download") + if len(r.calls) == 1 { + return false, nil + } + r.downloaded = true + return true, nil +} + +func (r *downloadOrderedLPXRegistry) AcquireBuildSnapshot( + ctx context.Context, + buildID string, +) (*lpx.BuildSnapshot, error) { + r.calls = append(r.calls, "snapshot") + if !r.downloaded { + return nil, errors.New("LPX snapshot acquired before Model Express download") + } + return r.ModelRegistry.AcquireBuildSnapshot(ctx, buildID) +} + +func requirePreparedLPX( + t *testing.T, + reconciler *graphReconciler, + ctx context.Context, + dgd *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, +) (*lpxMaterializing, *lpxRejected) { + t.Helper() + desired, rejected, err := reconciler.prepareLPXMaterializing(ctx, dgd, source) + require.NoError(t, err) + return desired, rejected +} + +func newPreparedLPXTestReconciler( + t *testing.T, + registry *lpx.ModelRegistry, + ctx context.Context, + dgd *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, +) (*graphReconciler, *lpxMaterializing) { + t.Helper() + + // Construct the fake client before preparing the selected LPX plan. + reconciler := newLPXTestReconciler(t, registry, dgd, source) + desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) + require.Nil(t, rejected) + return reconciler, desired +} + +func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { + t.Log("Publish the implicit hybrid runtime without interpreting stale selector annotations") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + source.Annotations[consts.KubeAnnotationLPXSchedulerBackend] = "unknown-scheduler" + source.Annotations[consts.KubeAnnotationLPXExecutionBackend] = "unknown-execution" + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + require.NotNil(t, desired) + require.Equal(t, "unknown-scheduler", source.Annotations[consts.KubeAnnotationLPXSchedulerBackend]) + require.Equal(t, "unknown-execution", source.Annotations[consts.KubeAnnotationLPXExecutionBackend]) + require.Empty(t, desired.plan.ConductorTemplate) + require.Empty(t, desired.plan.ConductorClique) + require.NotEmpty(t, desired.plan.CyborgClique) + + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + for _, object := range objects { + require.NotEmpty(t, object.GetName()) + } + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + require.NotContains(t, group.Spec.CliqueNames, "") + cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) + require.NotContains(t, cyborg.Spec.StartsAfter, "") + ordinaryPodGang := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) + require.Len(t, ordinaryPodGang.Spec.PodGroups, 1) + require.Equal(t, desired.plan.CyborgClique, ordinaryPodGang.Spec.PodGroups[0].Name) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.NotNil(t, request.Spec.CyborgPodCliqueRef) + require.Equal(t, desired.plan.CyborgClique, request.Spec.CyborgPodCliqueRef.Name) +} + +func TestSelectedLPXColdCacheDownloadsBeforeSnapshot(t *testing.T) { + t.Log("Build a selected LPX deployment backed by an initially cold Model Express cache") + child, source, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) + registry := &downloadOrderedLPXRegistry{ + ModelRegistry: baseRegistry, + buildURL: url.URL{ + Scheme: lpx.BuildSchemeGCS, + Host: "test-bucket", + Path: "/build-v2", + }, + } + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "Ready", ObservedGeneration: child.Generation}} + child.Status.ObservedGeneration = child.Generation + child.Status.ModelDownload = &nvidiacomv1beta1.ModelDownloadStatus{ + Builds: []string{registry.buildURL.String()}, + LastCheckedAt: &metav1.Time{Time: time.Now()}, + } + reconciler := newLPXTestReconciler(t, registry, child, source) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + + t.Log("Force Model Express despite the fresh cached status and avoid acquiring a build snapshot") + result, err := reconciler.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Equal(t, modelDownloadRequeueAfter, result.RequeueAfter) + require.Equal(t, []string{"download"}, registry.calls) + require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, "Ready")) + require.NotNil(t, child.Status.ModelDownload) + require.Empty(t, child.Status.ModelDownload.Builds) + sets := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, reconciler.List(t.Context(), sets)) + require.Empty(t, sets.Items) + + t.Log("Complete the download and reconcile the selected deployment again") + _, err = reconciler.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Equal(t, []string{"download", "download", "snapshot"}, registry.calls) + require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) + require.NotNil(t, child.Status.ModelDownload) + require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) +} + +func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXSpecDecodeTestDGD(t) + + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + require.Len(t, desired.requests, 3) + require.Len(t, desired.plan.Agents, 3) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + require.Len(t, requests, 3) + + requestByModel := make(map[string]lpxv1alpha1.LPUPipelineRequest, len(requests)) + for _, request := range requests { + model := request.Annotations[lpxModelAnnotation] + requestByModel[model] = request + require.Equal(t, model, request.Spec.NodeLocal.Model) + require.Equal(t, "selected-dgd-0-lpx-scheduler", request.Spec.PodGangRef.Name) + } + for _, projection := range desired.requests { + request, found := requestByModel[projection.modelProjection.Model()] + require.True(t, found) + require.Equal(t, projection.modelProjection.Digest().String(), request.Annotations[lpx.WorkloadDigestAnnotation]) + } + + for index, expected := range desired.plan.Agents { + projection := &desired.requests[index] + clique := findLPXTestClique(t, objects, expected.CliqueName) + require.Equal(t, int32(expected.Replicas), clique.Spec.Replicas) + require.Equal(t, ptr.To(int32(expected.Replicas)), clique.Spec.MinAvailable) + require.Equal(t, projection.modelProjection.Digest().String(), clique.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, projection.modelProjection.Model(), clique.Annotations[lpxv1alpha1.PodModelAnnotation]) + require.NotContains(t, clique.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + require.NotContains(t, clique.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) + } + pcs := findLPXTestPodCliqueSet(t, objects) + require.Equal(t, desired.workloadDigest, pcs.Annotations[lpx.WorkloadDigestAnnotation]) + + targetRequest := requestByModel["target"] + target := targetRequest.DeepCopy() + target.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: lpxv1alpha1.RequestPhaseUnsupported, + ObservedGeneration: ptr.To(target.Generation), + Diagnostics: []lpxv1alpha1.StatusDiagnostic{{ + Code: "TargetUnsupported", Subject: "model/target", Detail: "target projection is unsupported", + }}, + } + require.NoError(t, reconciler.Update(ctx, target)) + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) + require.Equal(t, "LPXUnsupported", lpxResult(classification).Reason) + require.Contains(t, lpxResult(classification).Message, "TargetUnsupported") +} + +func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { + t.Log("Materialize two runtime replicas and preserve their distinct publication identities") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + source.Spec.Components[0].Replicas = ptr.To(int32(2)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + require.Equal(t, int32(2), desired.plan.Replicas) + require.Len(t, desired.requests, 2) + require.NotEqual(t, desired.requests[0].requestName, desired.requests[1].requestName) + for replicaIndex := int32(0); replicaIndex < 2; replicaIndex++ { + require.Equal(t, replicaIndex, desired.requests[replicaIndex].replicaIndex) + } + + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + require.Equal(t, int32(2), scalingGroup.Spec.Replicas) + + t.Log("Leave Agent status unobserved and drift generated-child replica counts") + replicaOneAgent := findLPXTestClique(t, objects, desired.plan.ForReplica(1).Agents[0].CliqueName) + replicaOneAgent.Status.ObservedGeneration = nil + replicaOneAgent.Status.Replicas = replicaOneAgent.Spec.Replicas + replicaOneAgent.Status.UpdatedReplicas = replicaOneAgent.Spec.Replicas + replicaOneAgent.Status.ScheduleGatedReplicas = replicaOneAgent.Spec.Replicas + replicaOneAgent.Status.ScheduledReplicas = 0 + replicaOneAgent.Status.Conditions = nil + scalingGroup.Spec.Replicas = 9 + agentClique := findLPXTestClique(t, objects, desired.plan.Agents[0].CliqueName) + agentClique.Spec.Replicas = 9 + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + t.Log("Allow only the deterministic Cyborg and parent-PCSG reads; reject Pods and Agent cliques") + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + cachedClient, ok := reconciler.Client.(client.WithWatch) + require.True(t, ok) + forbiddenReads := make([]string, 0) + podGangLists := 0 + lifecycleRequestListCalls, dependencyListCalls := 0, 0 + scalingGroupGets := 0 + conductorGets := 0 + cyborgGets := 0 + forbidden := interceptor.Funcs{ + Get: func( + ctx context.Context, + delegated client.WithWatch, + key client.ObjectKey, + object client.Object, + opts ...client.GetOption, + ) error { + switch object.(type) { + case *corev1.Pod: + forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) + case *grovev1alpha1.PodCliqueScalingGroup: + if key.Name != desired.plan.LPXScalingGroup { + forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) + } else { + scalingGroupGets++ + } + case *grovev1alpha1.PodClique: + known := false + for _, request := range desired.requests { + plan := desired.plan.ForReplica(request.replicaIndex) + if key.Name == plan.ConductorClique { + conductorGets++ + known = true + break + } + if key.Name == plan.CyborgClique { + cyborgGets++ + known = true + break + } + } + if !known { + forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) + } + } + return delegated.Get(ctx, key, object, opts...) + }, + List: func( + ctx context.Context, + delegated client.WithWatch, + list client.ObjectList, + opts ...client.ListOption, + ) error { + if _, request := list.(*lpxv1alpha1.LPUPipelineRequestList); request { + lifecycleRequestListCalls++ + } else { + dependencyListCalls++ + } + switch list.(type) { + case *groveschedulerv1alpha1.PodGangList: + podGangLists++ + case *corev1.PodList, *grovev1alpha1.PodCliqueList, + *grovev1alpha1.PodCliqueScalingGroupList: + forbiddenReads = append(forbiddenReads, fmt.Sprintf("list %T", list)) + } + return delegated.List(ctx, list, opts...) + }, + } + reconciler.apiReader = interceptor.NewClient(reader, forbidden) + reconciler.Client = interceptor.NewClient(cachedClient, forbidden) + + t.Log("Publish directly from the PCS and PodGang witness") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + require.Empty(t, forbiddenReads) + require.Equal(t, 1, podGangLists) + require.Equal(t, 1, scalingGroupGets) + require.Zero(t, conductorGets) + require.Equal(t, 2, cyborgGets) + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + require.Len(t, requests, 2) + require.NotEqual(t, requests[0].Spec.PodGangRef.Name, requests[1].Spec.PodGangRef.Name) + require.NotNil(t, requests[0].Spec.CyborgPodCliqueRef) + require.NotNil(t, requests[1].Spec.CyborgPodCliqueRef) + require.NotEqual(t, requests[0].Spec.CyborgPodCliqueRef.Name, requests[1].Spec.CyborgPodCliqueRef.Name) + published := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + + t.Log("Repeat publication using the same witnesses and preserve the request") + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + require.Equal(t, published.UID, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName).UID) + require.Empty(t, forbiddenReads) + require.Equal(t, 2, scalingGroupGets) + require.Zero(t, conductorGets) + require.Equal(t, 4, cyborgGets) + + t.Log("Leave unsupported generated-child replica mutations untouched") + require.NoError(t, cachedClient.Get(ctx, client.ObjectKeyFromObject(scalingGroup), scalingGroup)) + require.Equal(t, int32(9), scalingGroup.Spec.Replicas) + require.NoError(t, cachedClient.Get(ctx, client.ObjectKeyFromObject(agentClique), agentClique)) + require.Equal(t, int32(9), agentClique.Spec.Replicas) + + t.Log("Retire the complete runtime while scheduler finalizers hold both requests") + for index := range requests { + requests[index].Finalizers = []string{"scheduling.lpu.nvidia.com/test-cleanup"} + require.NoError(t, reconciler.Update(ctx, &requests[index])) + } + _, err = reconciler.retireLPXRequest(ctx, dgd, &requests[0], "test group retirement") + require.NoError(t, err) + requests, err = reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + require.Len(t, requests, 2) + for index := range requests { + require.False(t, requests[index].DeletionTimestamp.IsZero()) + } + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, types.NamespacedName{ + Namespace: dgd.Namespace, + Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation], + }, pcs)) + require.Zero(t, pcs.Spec.Replicas) + + t.Log("Restore nonzero Grove scale while scheduler finalizers still hold both requests in deletion") + pcs.Spec.Replicas = 1 + require.NoError(t, reconciler.Update(ctx, pcs)) + lifecycleRequestListCalls, dependencyListCalls = 0, 0 + + t.Log("Re-establish scale-to-zero using only the authoritative attempt-wide request observation") + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + require.Positive(t, lifecycleRequestListCalls) + require.Zero(t, dependencyListCalls) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.Zero(t, pcs.Spec.Replicas) +} + +func TestSelectedLPXObservesLimitsOnlyCyborgGPUIntent(t *testing.T) { + t.Log("Materialize a hybrid Cyborg whose classic GPU intent exists only in limits") + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + cyborgPodSpec := &source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + cyborgPodSpec.ResourceClaims = nil + cyborgPodSpec.Containers[0].Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), + }, + } + reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) + cyborg.Spec.PodSpec = *cyborgPodSpec.DeepCopy() + cyborg.Spec.PodSpec.SchedulerName = corev1.DefaultSchedulerName + pcs := findLPXTestPodCliqueSet(t, objects) + for _, template := range pcs.Spec.Template.Cliques { + if template.Name == desired.plan.CyborgTemplate { + template.Spec.PodSpec = *cyborg.Spec.PodSpec.DeepCopy() + } + } + container := cyborg.Spec.PodSpec.Containers[0] + require.Empty(t, container.Resources.Requests) + require.NotEmpty(t, container.Resources.Limits) + + t.Log("Observe the raw Grove PodSpec without relying on Kubernetes Pod request defaulting") + createLPXTestObjects(t, t.Context(), reconciler.Client, objects...) + classification, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + + t.Log("Retire the published request if the live classic GPU intent disappears") + storedCyborg := &grovev1alpha1.PodClique{} + require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(cyborg), storedCyborg)) + storedCyborg.Spec.PodSpec.ResourceClaims = nil + for index := range storedCyborg.Spec.PodSpec.Containers { + if storedCyborg.Spec.PodSpec.Containers[index].Name == consts.MainContainerName || + storedCyborg.Spec.PodSpec.Containers[index].Name == "cyborg" { + storedCyborg.Spec.PodSpec.Containers[index].Resources = corev1.ResourceRequirements{} + } + } + require.NoError(t, reconciler.Update(t.Context(), storedCyborg)) + classification, err = reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) + require.NoError(t, err) + retiring, ok := classification.(*lpxRetiring) + require.True(t, ok) + require.Contains(t, retiring.retirementReason, "ordinary-scheduler identity") + requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) +} + +func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { + for _, test := range []struct { + name string + snapshotError error + retirementMessage string + }{ + {name: "inconsistent snapshot", snapshotError: fmt.Errorf("%w: compiler metadata changed during duplicate reads", lpx.ErrBuildSnapshotInconsistent), retirementMessage: "The model build became inconsistent while acquiring the immutable snapshot:"}, + {name: "transient snapshot", snapshotError: errors.New("temporary object-store timeout")}, + {name: "render failure", retirementMessage: "The current LPX workload can no longer be rendered safely:"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Publish a request backed by the rendered LPX workload") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + requestBefore := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + key := client.ObjectKeyFromObject(dgd) + pcsKey := client.ObjectKey{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]} + pcsBefore := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, pcsBefore)) + + t.Log("Fail the actual snapshot dependency or remove required runtime SSH configuration") + message := "node-local LPU runtime requires an MPI SSH secret name" + if test.snapshotError != nil { + reconciler.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: test.snapshotError} + message = test.snapshotError.Error() + } else { + reconciler.Config.MPI.SSHSecretName = "" + } + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + pcs := &grovev1alpha1.PodCliqueSet{} + if test.retirementMessage == "" { + t.Log("A transient snapshot failure must preserve the exact published request and PCS") + require.Equal(t, requestBefore, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name)) + require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) + require.Equal(t, pcsBefore, pcs) + } else { + t.Log("Retire the request while retaining its stopped PCS and actionable reason") + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name) + require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) + require.Zero(t, pcs.Spec.Replicas) + require.NoError(t, reconciler.Get(ctx, key, dgd)) + ready := meta.FindStatusCondition(dgd.Status.Conditions, "Ready") + require.NotNil(t, ready) + require.Equal(t, lpxRetiringReason, ready.Reason) + require.Contains(t, ready.Message, test.retirementMessage) + require.Contains(t, ready.Message, message) + + t.Log("Remove the old PCS only after its request is gone") + result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, pcs))) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + } + + t.Log("Persist the actual actionable failure and retain its wrapped cause") + require.ErrorContains(t, err, message) + if test.snapshotError != nil { + require.ErrorIs(t, err, test.snapshotError) + require.ErrorIs(t, err, lpx.ErrBuildSnapshotAcquisition) + } + require.NoError(t, reconciler.Get(ctx, key, dgd)) + failed := meta.FindStatusCondition(dgd.Status.Conditions, "Failed") + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionTrue, failed.Status) + require.Equal(t, dgd.Generation, failed.ObservedGeneration) + require.Equal(t, err.Error(), failed.Message) + }) + } +} + +func TestLPXAuthoritativeLifecycleListConsumesContinuePages(t *testing.T) { + t.Log("Create lifecycle objects spanning multiple authoritative pages") + ctx := t.Context() + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + dgd := newLPXTestDeployment(t, source) + reconciler := newLPXTestReconciler(t, nil, dgd, source) + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + + expected := make([]lpxv1alpha1.LPUPipelineRequest, lpxLifecycleListPageSize+1) + for index := range expected { + expected[index] = lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("request-%02d", index), + Namespace: dgd.Namespace, + }} + } + const nextPageToken = "lpx-request-page-2" + + t.Log("Install a paginated reader that repopulates each supplied page in place") + requestGets := 0 + requestListCalls := make([]client.ListOptions, 0, 2) + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + Get: func( + ctx context.Context, + delegated client.WithWatch, + key client.ObjectKey, + object client.Object, + opts ...client.GetOption, + ) error { + if _, request := object.(*lpxv1alpha1.LPUPipelineRequest); request { + requestGets++ + } + return delegated.Get(ctx, key, object, opts...) + }, + List: func( + ctx context.Context, + delegated client.WithWatch, + list client.ObjectList, + opts ...client.ListOption, + ) error { + requestList, request := list.(*lpxv1alpha1.LPUPipelineRequestList) + if !request { + return delegated.List(ctx, list, opts...) + } + options := *(&client.ListOptions{}).ApplyOptions(opts) + requestListCalls = append(requestListCalls, options) + switch options.Continue { + case "": + requestList.Items = append(requestList.Items[:0], expected[:lpxLifecycleListPageSize]...) + requestList.Continue = nextPageToken + case nextPageToken: + requestList.Items = append(requestList.Items[:0], expected[lpxLifecycleListPageSize:]...) + requestList.Continue = "" + default: + return fmt.Errorf("unexpected LPX request continuation token %q", options.Continue) + } + return nil + }, + }) + + t.Log("Retain raw callback pointers until every continuation page is consumed") + retained := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expected)) + require.NoError(t, visitLifecycleObjectPages( + ctx, + reconciler.apiReader, + &lpxv1alpha1.LPUPipelineRequestList{}, + func(object k8sruntime.Object) error { + request := object.(*lpxv1alpha1.LPUPipelineRequest) + retained = append(retained, request) + return nil + }, + client.InNamespace(dgd.Namespace), + )) + + t.Log("Verify retained first-page objects and exact pagination options after traversal") + require.Len(t, retained, len(expected)) + for index := range expected { + require.Equal(t, expected[index], *retained[index]) + } + require.Zero(t, requestGets, "pagination must not fall back to per-object request Gets") + require.Len(t, requestListCalls, 2, "the authoritative list must make exactly one call per returned page") + require.Equal(t, "", requestListCalls[0].Continue) + require.Equal(t, nextPageToken, requestListCalls[1].Continue) + for _, call := range requestListCalls { + require.Equal(t, int64(lpxLifecycleListPageSize), call.Limit) + require.Equal(t, dgd.Namespace, call.Namespace) + require.Nil(t, call.LabelSelector) + } +} + +func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { + t.Log("Prepare the current LPX attempt") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + prepare, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + desired := renderLPXTestPodCliqueSet(t, ctx, prepare, dgd, source, selected) + + t.Log("Seed a PCS already owned by the exact LPX child; metadata synchronization never adopts resources") + cached := desired.DeepCopy() + require.NoError(t, ctrl.SetControllerReference(dgd, cached, prepare.Scheme())) + cached.UID = "cached-pcs-uid" + cached.ResourceVersion = "7" + cached.Generation = 4 + hash, err := commoncontroller.GetSpecHash(cached) + require.NoError(t, err) + metav1.SetMetaDataAnnotation(&cached.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) + delete(cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) + cached.Annotations[lpx.DGDGenerationAnnotation] = "0" + cached.Annotations["test.example/keep"] = "kept" + + t.Log("Observe metadata writes and confirm reconciliation does not fetch another cached PCS") + reconciler := newLPXTestReconciler(t, registry, dgd, source, cached) + base, ok := reconciler.Client.(client.WithWatch) + require.True(t, ok) + gets := 0 + updates := 0 + reconciler.Client = interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + gets++ + return delegated.Get(ctx, key, object, opts...) + }, + Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { + updates++ + return delegated.Update(ctx, object, opts...) + }, + }) + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + fenceLists := 0 + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + fenceLists++ + return delegated.List(ctx, list, opts...) + }, + }) + + t.Log("Repair source generation and bookkeeping together without changing the attempt or spec") + synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) + require.NoError(t, err) + require.True(t, modified) + require.Zero(t, gets) + require.Equal(t, 1, updates) + require.Zero(t, fenceLists) + require.Equal(t, cached.UID, synced.UID) + bookkeeping := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(desired), bookkeeping)) + require.Equal(t, bookkeeping.ResourceVersion, synced.ResourceVersion) + require.True(t, metav1.IsControlledBy(synced, dgd)) + require.Equal(t, cached.Spec, synced.Spec) + require.Equal(t, desired.Annotations[lpx.DGDGenerationAnnotation], synced.Annotations[lpx.DGDGenerationAnnotation]) + require.Equal(t, desired.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, desired.Annotations[lpxDeploymentUIDAnnotation], synced.Annotations[lpxDeploymentUIDAnnotation]) + require.Equal(t, "kept", synced.Annotations["test.example/keep"]) + require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) + require.Equal(t, "4", synced.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) + require.Equal(t, "0", cached.Annotations[lpx.DGDGenerationAnnotation], "cached input must remain untouched") + require.NotContains(t, cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) + + t.Log("An unchanged observation is a no-op") + updates = 0 + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired, selected) + require.NoError(t, err) + require.False(t, modified) + require.Zero(t, updates) + require.Zero(t, fenceLists) + require.Equal(t, cached.UID, synced.UID) + + t.Log("A spec edit after observation conflicts instead of being overwritten by metadata repair") + synced.Annotations[lpx.DGDGenerationAnnotation] = "0" + require.NoError(t, base.Update(ctx, synced)) + cached = synced.DeepCopy() + concurrent := synced.DeepCopy() + concurrent.Spec.Replicas++ + concurrent.Generation++ + require.NoError(t, base.Update(ctx, concurrent)) + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) + require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) + require.Nil(t, synced) + require.False(t, modified) + require.Equal(t, 1, updates) + require.Zero(t, gets) + require.Zero(t, fenceLists) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(concurrent), bookkeeping)) + require.True(t, apiequality.Semantic.DeepEqual(concurrent.Spec, bookkeeping.Spec)) + require.Equal(t, concurrent.ResourceVersion, bookkeeping.ResourceVersion) +} + +func TestLPXPodCliqueSetPublicationFence(t *testing.T) { + t.Log("Prepare a render for publication and observe writes across the authoritative fence") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + desired := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, selected) + base, ok := reconciler.Client.(client.WithWatch) + require.True(t, ok) + writes, lists := 0, 0 + reconciler.Client = interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + writes++ + return delegated.Create(ctx, object, opts...) + }, + Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { + writes++ + return delegated.Update(ctx, object, opts...) + }, + }) + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + fenceErr := errors.New("fence list failed") + failFence := true + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if failFence { + return fenceErr + } + return delegated.List(ctx, list, opts...) + }, + }) + + t.Log("A failed publication observation prevents initial creation") + synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + require.ErrorIs(t, err, fenceErr) + require.False(t, modified) + require.Nil(t, synced) + require.Zero(t, writes) + require.Equal(t, 1, lists) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) + + t.Log("Recover the observation and create the exact child-owned PCS") + failFence = false + writes, lists = 0, 0 + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + require.NoError(t, err) + require.True(t, modified) + require.Equal(t, 1, lists) + require.Equal(t, 1, writes) + require.True(t, apiequality.Semantic.DeepEqual(desired.Spec, synced.Spec)) + require.True(t, metav1.IsControlledBy(synced, dgd)) + hash, hashErr := commoncontroller.GetSpecHash(desired) + require.NoError(t, hashErr) + require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) + require.Equal(t, "1", synced.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) + stored := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(desired), stored)) + require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) + + t.Log("A stale missing observation does not adopt an already-created PCS") + _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) + require.False(t, modified) + + t.Log("An observation outage also fences replacement without changing the current PCS") + live := synced + desired = live.DeepCopy() + desired.ResourceVersion, desired.UID = "", "" + desired.Spec.Replicas++ + desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" + failFence = true + writes, lists = 0, 0 + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired, selected) + require.ErrorIs(t, err, fenceErr) + require.False(t, modified) + require.Nil(t, synced) + require.Zero(t, writes) + require.Equal(t, 1, lists) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(live), stored)) + require.True(t, apiequality.Semantic.DeepEqual(live.Spec, stored.Spec)) + require.Equal(t, live.Annotations, stored.Annotations) +} + +func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + rendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, desired) + _, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + + pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]} + drifted := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) + require.NotEmpty(t, drifted.Spec.Template.Cliques) + drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName = "externally-mutated" + require.NoError(t, reconciler.Update(ctx, drifted)) + require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) + + _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + dgd, + drifted, + rendered, + desired, + ) + var retiring *lpxRetiring + require.ErrorAs(t, err, &retiring) + require.NotNil(t, retiring) + requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) + + t.Log("Hold the retired PCS at zero before any successor write") + require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) + require.Zero(t, drifted.Spec.Replicas) + + foreignPCS := drifted.DeepCopy() + foreignPCS.ResourceVersion = "" + require.NotEmpty(t, foreignPCS.OwnerReferences) + foreignPCS.OwnerReferences[0].UID = "foreign-owner" + foreignReconciler := newLPXTestReconciler(t, registry, dgd, source, foreignPCS) + _, err = foreignReconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil, false) + require.EqualError(t, err, fmt.Sprintf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", foreignPCS.Name)) + + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + pcsGets := 0 + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + pcsGets++ + } else { + return errors.New("retired Grove observation must not read dependencies") + } + return delegated.Get(ctx, key, object, opts...) + }, + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("retired Grove observation must not read dependencies") + }, + }) + classification, err := reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil, false) + require.NoError(t, err) + closed := requireLPXClosed(t, classification) + require.Equal(t, "The retired Grove attempt is held at zero; successor publication waits for post-retirement Grove synchronization", closed.incomplete) + require.Equal(t, 1, pcsGets) + reconciler.apiReader = reader + + require.Equal(t, "externally-mutated", drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName) + _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + dgd, + drifted, + rendered, + desired, + ) + require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) + require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is deleted") + require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, &grovev1alpha1.PodCliqueSet{}))) + requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) + + _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + dgd, + nil, + rendered, + desired, + ) + require.NoError(t, err) + + recreated := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) + for _, clique := range recreated.Spec.Template.Cliques { + require.NotEqual(t, "externally-mutated", clique.Spec.PodSpec.PriorityClassName) + } + requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) +} + +func TestGroveSpecSyncRecreatesStaleLPXAttemptBeforeSuccessor(t *testing.T) { + ctx := t.Context() + oldDGD, oldSource, oldRegistry := newLPXTestDGD(t, lpx.PipelineSingle) + oldPrepareReconciler := newLPXTestReconciler(t, oldRegistry, oldDGD, oldSource) + oldDesired, rejected := requirePreparedLPX(t, oldPrepareReconciler, ctx, oldDGD, oldSource) + require.Nil(t, rejected) + + oldPCS := findLPXTestPodCliqueSet(t, lpxMaterializedObjects(t, oldPrepareReconciler, oldDGD, oldSource, oldDesired)) + + const successorBuildID = "build-v2-successor" + successorRegistry := newLPXTestRegistryWithPartitionsAndMode( + t, + successorBuildID, + []int{7, 8, 9}, + manifestcapnpv2.CompilationMode_lpuOnly, + ) + successorDGD := oldDGD.DeepCopy() + successorSource := oldSource.DeepCopy() + successorDGD.Generation++ + successorSource.Spec.Components[0].LPX.BuildID = successorBuildID + reconciler := newLPXTestReconciler(t, successorRegistry, successorDGD, successorSource) + successorDesired, rejected := requirePreparedLPX(t, reconciler, ctx, successorDGD, successorSource) + require.Nil(t, rejected) + + createLPXTestObjects(t, ctx, reconciler.Client, oldPCS) + pcs := &grovev1alpha1.PodCliqueSet{} + pcsKey := types.NamespacedName{ + Namespace: successorDGD.Namespace, + Name: successorDGD.Annotations[dynamo.LPXPCSNameAnnotation], + } + require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) + oldCliqueCount := len(pcs.Spec.Template.Cliques) + oldAgentReplicas := int32(0) + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { + oldAgentReplicas = clique.Spec.Replicas + } + } + require.Positive(t, oldAgentReplicas) + successorRendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, successorDGD, successorSource, successorDesired) + + _, _, err := reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + successorDGD, + pcs, + successorRendered, + successorDesired, + ) + require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) + require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is scaled to zero") + + stored := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, stored)) + require.Zero(t, stored.Spec.Replicas) + require.Equal(t, oldDesired.workloadDigest, stored.Annotations[lpx.WorkloadDigestAnnotation]) + + _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + successorDGD, + stored, + successorRendered, + successorDesired, + ) + require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) + require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is deleted") + require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, &grovev1alpha1.PodCliqueSet{}))) + + _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + ctx, + successorDGD, + nil, + successorRendered, + successorDesired, + ) + require.NoError(t, err) + + recreated := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) + require.Equal(t, successorDesired.workloadDigest, recreated.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, string(successorDGD.UID), recreated.Annotations[lpxDeploymentUIDAnnotation]) + require.Equal(t, strconv.FormatInt(successorDGD.Generation, 10), recreated.Annotations[lpxDeploymentGenerationAnnotation]) + require.Len(t, recreated.Spec.Template.Cliques, oldCliqueCount) + newAgentReplicas := int32(0) + for _, clique := range recreated.Spec.Template.Cliques { + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { + newAgentReplicas = clique.Spec.Replicas + } + } + require.Greater(t, newAgentReplicas, oldAgentReplicas) +} + +func TestLPXRetirementOverridesRuntimeReadiness(t *testing.T) { + t.Log("Retirement must replace both successful and pending runtime results") + retiring := &lpxRetiring{retirementReason: "test retirement"} + for _, base := range []reconcileOutcome{{State: nvidiacomv1beta1.DGDStateSuccessful}, {State: nvidiacomv1beta1.DGDStatePending}} { + require.Equal(t, lpxResult(retiring), overlayLPXResult(base, retiring)) + } +} + +func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { + t.Log("Prepare the LPU-only runtime without adding scheduler configuration to its source") + ctx := context.Background() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + require.Nil(t, source.Spec.Scheduling) + require.NotContains(t, source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) + + t.Log("Materialize the selected runtime alongside an unrelated ordinary frontend group") + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + ordinary := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) + ordinary.Spec.PodGroups = append(ordinary.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{ + Name: "frontend", MinReplicas: 1, + PodReferences: []groveschedulerv1alpha1.NamespacedName{{Namespace: dgd.Namespace, Name: "frontend-0"}}, + }) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + t.Log("Keep an otherwise-ready runtime gated before the exact request is Bound") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + runtimeReady := reconcileOutcome{ + State: nvidiacomv1beta1.DGDStateSuccessful, + Reason: "all_resources_are_ready", + Message: "All resources are ready", + } + require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) + + t.Log("Recreate the controller and derive the same open attempt from API objects") + reconciler = &graphReconciler{ + Client: reconciler.Client, + recorder: reconciler.recorder, + apiReader: reconciler.apiReader, + runtimeConfig: reconciler.runtimeConfig, + modelRegistry: registry, + Config: reconciler.Config, + DockerSecretRetriever: reconciler.DockerSecretRetriever, + } + rederived, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) + require.Nil(t, rejected) + require.Equal(t, desired.requests[0].attemptDigest, rederived.requests[0].attemptDigest) + desired = rederived + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + + t.Log("Observe the exact current scheduler receipt after the controller restart") + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Generation = 7 + request.Status = newLPXNodeLocalBoundStatus(7, 3, "sha256:bound-plan") + require.NoError(t, reconciler.Update(ctx, request)) + + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + bound, ok := classification.(*lpxBound) + require.True(t, ok) + + t.Log("Return an exact node-local Bound receipt to the base Grove runtime result") + require.Equal(t, runtimeReady, overlayLPXResult(runtimeReady, bound)) + runtimePending := reconcileOutcome{ + State: nvidiacomv1beta1.DGDStatePending, + Reason: "some_resources_are_not_ready", + Message: "Some resources are not ready", + } + require.Equal(t, runtimePending, overlayLPXResult(runtimePending, bound)) + + t.Run("unknown backend arm is not bound", func(t *testing.T) { + t.Log("Reject an execution backend outside the scheduler API contract") + unknownBackend := request.DeepCopy() + unknownBackend.Spec.ExecutionBackend = "unknown" + unknownBackend.Status.Committed.Plan.Placement = lpxv1alpha1.PlanPlacement{ExecutionBackend: "unknown"} + require.IsType(t, &lpxSchedulerObserved{}, classifyPublishedLPX(unknownBackend)) + }) + + t.Log("Keep runtime readiness gated when the Bound receipt belongs to an older generation") + request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Status.ObservedGeneration = ptr.To(int64(6)) + require.NoError(t, reconciler.Update(ctx, request)) + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) + + t.Log("Stale failure diagnostics are not attributed to the current request generation") + request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Status.Phase = lpxv1alpha1.RequestPhaseUnsupported + request.Status.Committed = nil + request.Status.Diagnostics = []lpxv1alpha1.StatusDiagnostic{{ + Code: "StaleUnsupported", Subject: "request/old", Detail: "belongs to an older generation", + }} + require.NoError(t, reconciler.Update(ctx, request)) + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + require.NotContains(t, lpxResult(classification).Message, "StaleUnsupported") + + t.Log("Delete the stored child while scheduler cleanup holds its published request") + request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} + require.NoError(t, reconciler.Update(ctx, request)) + key := client.ObjectKeyFromObject(dgd) + require.NoError(t, reconciler.Get(ctx, key, dgd)) + require.NoError(t, reconciler.Delete(ctx, dgd)) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + waiting := fmt.Sprintf("waiting for LPX request %q to finish fail-closed retirement", request.Name) + require.EqualError(t, err, waiting) + request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, request.Name) + require.False(t, request.DeletionTimestamp.IsZero()) + require.Equal(t, []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"}, request.Finalizers) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs)) + require.Zero(t, pcs.Spec.Replicas, "child finalization must stop the workload before scheduler cleanup finishes") + + t.Log("Keep the child finalizer until the scheduler releases the exact request") + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.EqualError(t, err, waiting) + require.NoError(t, reconciler.Get(ctx, key, dgd)) + require.Contains(t, dgd.Finalizers, lpxGraphDeploymentFinalizer) + request.Finalizers = []string{lpxAttemptRecordingFinalizer} + require.NoError(t, reconciler.Update(ctx, request)) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.EqualError(t, err, waiting) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, request.Name) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) +} + +func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { + for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { + t.Run(string(pipeline), func(t *testing.T) { + ctx := t.Context() + deployment, source, registry := newLPXTestDGD(t, pipeline) + if pipeline == lpx.PipelineLPX { + source.Spec.Components[0].Replicas = ptr.To(int32(1)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + } + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) + objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + require.Equal(t, desired.workload.LPXComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) + } + } + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + pcs := findLPXTestPodCliqueSet(t, objects) + rootReads := 0 + base, ok := reconciler.Client.(client.WithWatch) + require.True(t, ok) + reader := interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + rootReads++ + } + return reader.Get(ctx, key, object, opts...) + }}) + observe := func() dynamo.GroveReadiness { + ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, source, deployment, pcs) + require.NoError(t, err) + require.Len(t, ready.ComponentStatuses, 1) + require.Zero(t, rootReads, "reuse the synchronized PCS observation") + return ready + } + t.Log("Report one complete engine, never separate GPU or Agent component capacity") + ready := observe() + require.True(t, ready.Ready) + status := ready.ComponentStatuses[desired.workload.LPXComponentName()] + require.Equal(t, nvidiacomv1beta1.ComponentKindPodCliqueScalingGroup, status.ComponentKind) + require.Equal(t, []string{desired.plan.LPXScalingGroup}, status.ComponentNames) + require.Equal(t, int32(1), status.Replicas) + require.Equal(t, int32(1), status.UpdatedReplicas) + require.Equal(t, ptr.To(int32(1)), status.AvailableReplicas) + require.Equal(t, ptr.To(int32(1)), status.ScheduledReplicas) + if desired.plan.CyborgClique != "" { + t.Log("A stale GPU-role scale cannot satisfy complete-engine readiness") + gpu := findLPXTestClique(t, objects, desired.plan.CyborgClique).DeepCopy() + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(gpu), gpu)) + gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 1, 1, 1 + gpu.Status.ReadyReplicas, gpu.Status.ScheduledReplicas = 1, 1 + require.NoError(t, reconciler.Update(ctx, gpu)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, nvidiacomv1beta1.DGDReadyReasonUpdating, ready.Classification) + t.Log("Partial GPU readiness contributes zero complete engine replicas") + gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 2, 2, 2 + gpu.Status.ScheduledReplicas = 2 + require.NoError(t, reconciler.Update(ctx, gpu)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, nvidiacomv1beta1.DGDReadyReasonPodsNotReady, ready.Classification) + require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.LPXComponentName()].AvailableReplicas) + gpu.Status.ReadyReplicas = 2 + require.NoError(t, reconciler.Update(ctx, gpu)) + require.True(t, observe().Ready) + } + t.Log("Reject stale complete-engine capacity even when every existing role is ready") + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.LPXScalingGroup}, group)) + group.Spec.Replicas = 2 + require.NoError(t, reconciler.Update(ctx, group)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, nvidiacomv1beta1.DGDReadyReasonUpdating, ready.Classification) + }) + } +} + +func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { + tests := []struct { + name string + phase lpxv1alpha1.RequestPhase + releaseReason *lpxv1alpha1.ReleaseReason + wantState nvidiacomv1beta1.DGDState + wantReason string + wantMessage string + }{ + {name: "pending", phase: lpxv1alpha1.RequestPhasePending, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXSchedulerPending"}, + {name: "no fit", phase: lpxv1alpha1.RequestPhaseNoFit, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXNoFit"}, + {name: "unsupported", phase: lpxv1alpha1.RequestPhaseUnsupported, wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXUnsupported"}, + {name: "planned", phase: lpxv1alpha1.RequestPhasePlanned, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXPlanned"}, + {name: "reserving", phase: lpxv1alpha1.RequestPhaseReserving, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReserving"}, + {name: "binding", phase: lpxv1alpha1.RequestPhaseBinding, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXBinding"}, + {name: "degraded", phase: lpxv1alpha1.RequestPhaseDegraded, wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXDegraded"}, + { + name: "failure release in progress", phase: lpxv1alpha1.RequestPhaseReleasing, + releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonBindingFailed), + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXReleasing", wantMessage: "bindingFailed", + }, + { + name: "request change release in progress", phase: lpxv1alpha1.RequestPhaseReleasing, + releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonRequestChanged), + wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReleasing", wantMessage: "requestChanged", + }, + { + name: "failure release complete", phase: lpxv1alpha1.RequestPhaseReleased, + releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonDependencyChanged), + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXReleased", wantMessage: "dependencyChanged", + }, + { + name: "deletion release complete", phase: lpxv1alpha1.RequestPhaseReleased, + releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonRequestDeleted), + wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReleased", wantMessage: "requestDeleted", + }, + { + name: "release without journal", phase: lpxv1alpha1.RequestPhaseReleasing, + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", + wantMessage: "without the required release journal", + }, + { + name: "release with unknown reason", phase: lpxv1alpha1.RequestPhaseReleased, + releaseReason: ptr.To(lpxv1alpha1.ReleaseReason("futureReason")), + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", + wantMessage: "unknown release reason", + }, + { + name: "bound without exact proof", phase: lpxv1alpha1.RequestPhaseBound, + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", wantMessage: "without the exact current-generation plan proof", + }, + { + name: "unknown phase", phase: lpxv1alpha1.RequestPhase("future"), + wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", wantMessage: `unknown phase "future"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Record a scheduler status observation for the current generation") + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{Generation: 7}, + Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: test.phase, + ObservedGeneration: ptr.To(int64(7)), + Diagnostics: []lpxv1alpha1.StatusDiagnostic{{ + Code: "ExactDiagnostic", Subject: "requests/current", Detail: "the exact scheduler detail", + }}, + }, + } + if test.releaseReason != nil { + request.Status.Committed = &lpxv1alpha1.Committed{ + Execution: lpxv1alpha1.CommittedExecution{ + Release: &lpxv1alpha1.ReleaseJournal{Reason: *test.releaseReason, RequestedAtGeneration: 7}, + }, + } + } + + t.Log("Classify the current receipt and preserve its phase, journal and diagnostic") + classification := classifyPublishedLPX(request) + observed, ok := classification.(*lpxSchedulerObserved) + require.True(t, ok) + require.Equal(t, test.phase, observed.Phase) + result := lpxResult(observed) + require.Equal(t, test.wantState, result.State) + require.Equal(t, test.wantReason, result.Reason) + require.Contains(t, result.Message, "ExactDiagnostic") + require.Contains(t, result.Message, "requests/current") + require.Contains(t, result.Message, "the exact scheduler detail") + if test.wantMessage != "" { + require.Contains(t, result.Message, test.wantMessage) + } + }) + } +} + +func TestLPXSchedulerDiagnosticMessageRespectsConditionLimit(t *testing.T) { + diagnostics := []lpxv1alpha1.StatusDiagnostic{{ + Code: strings.Repeat("c", 253), + Subject: strings.Repeat("s", 253), + Detail: strings.Repeat("界", 4096/len("界")), + }} + for range 31 { + diagnostics = append(diagnostics, diagnostics[0]) + } + + message := lpxMessageWithDiagnostics("LPX scheduler diagnostic summary", diagnostics) + + require.LessOrEqual(t, len(message), maxDGDConditionMessageSize) + require.True(t, utf8.ValidString(message)) + require.True(t, strings.HasSuffix(message, "…")) +} + +func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { + type digestInput struct { + namespace string + name string + uid types.UID + generation int64 + model string + workloadDigest string + } + digest := func(input digestInput, replicaIndex int32) string { + return digestAttemptKey( + input.namespace, + input.name, + input.uid, + input.generation, + input.model, + input.workloadDigest, + replicaIndex, + ) + } + base := digestInput{ + namespace: "ns", name: "dgd", uid: "uid-a", generation: 4, + model: "default", workloadDigest: "sha256:workload-a", + } + baseDigest := digest(base, 0) + require.Equal(t, "sha256:ad538980e64759a4f9761c61ee637e8856513d2cec2cf5ddaf645a8c88b4f5da", baseDigest) + for name, mutate := range map[string]func(*digestInput){ + "namespace": func(key *digestInput) { key.namespace = "other-ns" }, + "name": func(key *digestInput) { key.name = "other-dgd" }, + "uid": func(key *digestInput) { key.uid = "uid-b" }, + "generation": func(key *digestInput) { + key.generation++ + }, + "model": func(key *digestInput) { key.model = "model-b" }, + "projection": func(key *digestInput) { key.workloadDigest = "sha256:workload-b" }, + } { + t.Run(name, func(t *testing.T) { + changed := base + mutate(&changed) + require.NotEqual(t, baseDigest, digest(changed, 0)) + mutate(&changed) + require.NotEmpty(t, digest(changed, 0)) + }) + } + require.Equal(t, baseDigest, digest(base, 0), "change-then-revert must return to the same complete tuple digest") + require.NotEqual(t, baseDigest, digest(base, 1)) + require.LessOrEqual(t, len(lpxRequestName("a-very-long-but-readable-dynamo-graph-deployment-name", baseDigest)), 63) + ctx := context.Background() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + + t.Log("Fence exact PCS and PodGang UID annotations") + projection := &desired.requests[0] + identity := &lpxGroveIdentity{pcsUID: "pcs-uid", podGangName: "podgang", podGangUID: "podgang-uid"} + current := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection, identity)}, + Spec: projection.modelProjection.RequestSpec(dgd.Namespace, identity.podGangName, identity.cyborgClique), + } + require.NoError(t, validateCurrentLPXRequest(dgd, projection, identity, current)) + changed := *identity + changed.pcsUID = "replacement-pcs-uid" + require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") + changed = *identity + changed.podGangUID = "replacement-podgang-uid" + require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") + + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + _, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + + dgd.Generation++ + classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + + classification, _, err = reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + require.NoError(t, err) + require.Nil(t, classification) +} + +func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { + for _, scenario := range []struct{ name, message string }{ + {"LPX", "LPX integration is disabled"}, + {"Grove", "Grove is disabled"}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Publish a complete workload, discovery service and scheduling attempt") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + source.Spec.Scheduling = deadlineTestScheduling() + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) + key := client.ObjectKeyFromObject(child) + for range 4 { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, key, child)) + require.NotNil(t, child.Status.Placement) + require.NotNil(t, child.Status.Placement.LPXAttempt) + require.NotNil(t, child.Status.Placement.LPXAttempt.DeadlineAt) + placement := child.Status.Placement.DeepCopy() + before := []client.ObjectList{ + &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}, + &corev1.ServiceList{}, &lpxv1alpha1.LPUPipelineRequestList{}, + } + for _, list := range before { + require.NoError(t, r.List(ctx, list)) + require.Positive(t, meta.LenList(list)) + } + + t.Log("Disable the provider before an input edit and an expired deadline can retire the workload") + if scenario.name == "LPX" { + r.runtimeConfig.Gate.LPX = false + } else { + r.runtimeConfig.Gate.Grove = false + } + r.modelRegistry = nil + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) + originalSpec := source.Spec.DeepCopy() + source.Spec.Components[0].LPX.BuildID = "edited-while-disabled" + require.NoError(t, r.Update(ctx, source)) + child.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))) + require.NoError(t, r.Status().Update(ctx, child)) + disabledPlacement := child.Status.Placement.DeepCopy() + for range 2 { + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Zero(t, result) + } + require.NoError(t, r.Get(ctx, key, child)) + require.Equal(t, disabledPlacement, child.Status.Placement) + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionTrue, failed.Status) + require.Equal(t, "LPXUnavailable", failed.Reason) + require.Equal(t, scenario.message, failed.Message) + require.Equal(t, child.Generation, failed.ObservedGeneration) + require.Equal(t, metav1.ConditionFalse, meta.FindStatusCondition(child.Status.Conditions, "Ready").Status) + for _, list := range before { + current := list.DeepCopyObject().(client.ObjectList) + require.NoError(t, r.List(ctx, current)) + require.Equal(t, list, current) + } + + t.Log("Re-enable unchanged intent and resume the original PCS and request identities") + source.Spec = *originalSpec + require.NoError(t, r.Update(ctx, source)) + child.Status.Placement = placement + require.NoError(t, r.Status().Update(ctx, child)) + r.runtimeConfig.Gate.LPX, r.runtimeConfig.Gate.Grove = true, true + r.modelRegistry = registry + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key, child)) + require.NotEqual(t, "LPXUnavailable", meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + for _, list := range before { + current := list.DeepCopyObject().(client.ObjectList) + require.NoError(t, r.List(ctx, current)) + require.Equal(t, list, current) + } + + t.Log("Explicit deletion still retires requests and releases the child finalizer while disabled") + if scenario.name == "LPX" { + r.runtimeConfig.Gate.LPX = false + } else { + r.runtimeConfig.Gate.Grove = false + } + r.modelRegistry = nil + require.NoError(t, r.Delete(ctx, child)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.ErrorContains(t, err, "waiting for LPX request") + request := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + require.False(t, request.DeletionTimestamp.IsZero()) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: child.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs)) + require.Zero(t, pcs.Spec.Replicas) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.ErrorContains(t, err, "waiting for LPX request") + requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) + }) + } +} + +func TestLPXControllerRetiresPublishedWorkOutsideGrove(t *testing.T) { + t.Log("Retire a published attempt when its source leaves the Grove provider") + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) + createLPXTestObjects(t, t.Context(), reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + _, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) + require.NoError(t, err) + + require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent + require.NoError(t, reconciler.Update(t.Context(), source)) + result, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) + require.NoError(t, err) + require.Equal(t, lpxRetirementRequeueAfter, result.RequeueAfter) + requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) +} + +func TestLPUOnlyWaitsForCurrentConductorWitness(t *testing.T) { + tests := []struct{ name, defect, want string }{ + {name: "stale owner", defect: "owner", want: "ordinary-scheduler identity"}, + {name: "wrong PodGang", defect: "podgang", want: "ordinary-scheduler identity"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + conductor := findLPXTestClique(t, objects, desired.plan.ConductorClique) + switch test.defect { + case "owner": + conductor.OwnerReferences[0].UID = "previous-scaling-group-uid" + case "podgang": + conductor.Labels[grovecommon.LabelPodGang] = "other-podgang" + } + createLPXTestObjects(t, t.Context(), reconciler.Client, objects...) + classification, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) + require.NoError(t, err) + closed := requireLPXClosed(t, classification) + require.Contains(t, closed.incomplete, test.want) + requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + }) + } +} + +func TestNodeLocalHybridPublishesAndFencesCurrentCyborgMaterialization(t *testing.T) { + tests := []struct { + name string + generation int64 + observedGeneration *int64 + }{ + { + name: "absent generation then claim drift", + generation: 1, + observedGeneration: nil, + }, + { + name: "lagging generation then UID replacement", + generation: 2, + observedGeneration: ptr.To(int64(1)), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Arrange current Cyborg materialization evidence without an observed generation") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) + cyborg.Generation = test.generation + cyborg.Status.ObservedGeneration = test.observedGeneration + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + t.Log("Publish the NodeLocal request with the exact Cyborg name and UID") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + request := getLPXRequest( + t, + ctx, + reconciler.Client, + dgd.Namespace, + desired.requests[0].requestName, + ) + require.NotNil(t, request.Spec.CyborgPodCliqueRef) + require.Equal(t, cyborg.Name, request.Spec.CyborgPodCliqueRef.Name) + require.Equal(t, string(cyborg.UID), request.Spec.CyborgPodCliqueRef.UID) + publishedUID := request.UID + + storedCyborg := &grovev1alpha1.PodClique{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(cyborg), storedCyborg)) + var retirementReason string + if test.observedGeneration == nil { + t.Log("Drift a live Claim reference without an observed generation") + require.NotEmpty(t, storedCyborg.Spec.PodSpec.ResourceClaims) + storedCyborg.Generation++ + storedCyborg.Spec.PodSpec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("stale-gpu") + require.NoError(t, reconciler.Update(ctx, storedCyborg)) + retirementReason = "DRA Claim references changed" + } else { + t.Log("Advance only the global generation without rotating the published request") + storedCyborg.Status.ObservedGeneration = ptr.To(test.generation) + require.NoError(t, reconciler.Update(ctx, storedCyborg)) + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.Equal(t, publishedUID, request.UID) + + t.Log("Replace the Cyborg clique at the same name with a different UID") + require.NoError(t, reconciler.Delete(ctx, storedCyborg)) + replacement := storedCyborg.DeepCopy() + replacement.ResourceVersion = "" + replacement.UID = "replacement-cyborg-uid" + replacement.DeletionTimestamp = nil + replacement.CreationTimestamp = metav1.Time{} + require.NoError(t, reconciler.Create(ctx, replacement)) + retirementReason = "immutable intent" + } + + t.Log("Retire the request when its exact published materialization no longer holds") + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + retiring, ok := classification.(*lpxRetiring) + require.True(t, ok) + require.Contains(t, retiring.retirementReason, retirementReason) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + }) + } +} + +func TestNodeLocalHybridWaitsForCurrentCyborgWitness(t *testing.T) { + tests := []struct { + name string + defect string + wantMessage string + }{ + { + name: "missing current scaling group", + defect: "missing-scaling-group", + wantMessage: "current LPU PodCliqueScalingGroup", + }, + { + name: "stale scaling-group owner", + defect: "stale-scaling-group-owner", + wantMessage: "current PodCliqueSet owner identity", + }, + { + name: "stale Cyborg owner", + defect: "stale-cyborg-owner", + wantMessage: "current LPU scaling-group owner identity", + }, + { + name: "missing PCS generation hash", + defect: "missing-pcs-hash", + wantMessage: "current PodCliqueSet generation", + }, + { + name: "mismatched PCS generation hash", + defect: "mismatched-pcs-hash", + wantMessage: "current PodCliqueSet generation", + }, + { + name: "missing Pod template hash", + defect: "missing-template-hash", + wantMessage: "current PodCliqueSet generation", + }, + { + name: "empty Pod template hash", + defect: "empty-template-hash", + wantMessage: "current PodCliqueSet generation", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Arrange one incomplete hybrid Cyborg witness") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) + switch test.defect { + case "missing-scaling-group": + scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + objects = slices.DeleteFunc(objects, func(object client.Object) bool { return object == scalingGroup }) + case "stale-scaling-group-owner": + scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + require.NotEmpty(t, scalingGroup.OwnerReferences) + scalingGroup.OwnerReferences[0].UID = "previous-pcs-uid" + case "stale-cyborg-owner": + require.NotEmpty(t, cyborg.OwnerReferences) + cyborg.OwnerReferences[0].UID = "previous-scaling-group-uid" + case "missing-pcs-hash": + cyborg.Status.CurrentPodCliqueSetGenerationHash = nil + case "mismatched-pcs-hash": + cyborg.Status.CurrentPodCliqueSetGenerationHash = ptr.To("previous-pcs-generation") + case "missing-template-hash": + cyborg.Status.CurrentPodTemplateHash = nil + case "empty-template-hash": + cyborg.Status.CurrentPodTemplateHash = ptr.To("") + default: + require.FailNow(t, "unknown materialization defect", "defect: %s", test.defect) + } + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + t.Log("Keep publication pending until a dependency watch supplies current evidence") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + closed := requireLPXClosed(t, classification) + require.Contains(t, closed.incomplete, test.wantMessage) + require.Equal(t, nvidiacomv1beta1.DGDStatePending, lpxResult(closed).State) + requireLPXRequestNotFound( + t, + ctx, + reconciler.Client, + dgd.Namespace, + desired.requests[0].requestName, + ) + }) + } +} + +func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { + const ( + crossReplicaAgent = "other replica Agent enters ordinary gang" + missingLPXGang = "Waiting for one replica-0 LPX PodGang; found 0 candidates" + missingOrdinary = "Waiting for one replica-0 ordinary-scheduler PodGang; found 0 candidates" + ) + tests := []struct { + name, wantMessage string + }{ + {"LPX gang unavailable", missingLPXGang}, + {"LPX gang is stale", "Waiting for Grove to publish current-generation PodGangs"}, + {"LPX gang has incomplete references", missingLPXGang}, + {"LPX gang has an extra group", missingLPXGang}, + {"LPX gang is ambiguous", "Waiting for one replica-0 LPX PodGang; found 2 candidates"}, + {"Agent enters ordinary gang", missingOrdinary}, + {crossReplicaAgent, missingOrdinary}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Materialize the complete hybrid runtime and its scheduler-scoped gangs") + ctx := context.Background() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) + if test.name == crossReplicaAgent { + source.Spec.Components[0].Replicas = ptr.To(int32(2)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(1)) + } + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + + t.Log("Introduce only the selected gang or startup-witness defect") + gang := findLPXTestPodGang(t, objects, lpx.SchedulerName) + switch test.name { + case "LPX gang unavailable": + gang.Labels[consts.KubeLabelDynamoGraphDeploymentName] = lpxTestOtherName + case "LPX gang is stale": + gang.Annotations[lpxDeploymentGenerationAnnotation] = "0" + case "LPX gang has incomplete references": + gang.Spec.PodGroups[0].PodReferences = nil + case "LPX gang has an extra group": + gang.Spec.PodGroups = append(gang.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{Name: "unexpected"}) + case "LPX gang is ambiguous": + duplicate := gang.DeepCopy() + duplicate.Name, duplicate.UID, duplicate.ResourceVersion = duplicate.Name+"-duplicate", "duplicate-uid", "" + objects = append(objects, duplicate) + case "Agent enters ordinary gang", crossReplicaAgent: + ordinary := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) + agent := desired.plan.Agents[0] + if test.name == crossReplicaAgent { + agent = desired.plan.ForReplica(1).Agents[0] + } + ordinary.Spec.PodGroups = append(ordinary.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{ + Name: agent.CliqueName, MinReplicas: 1, + PodReferences: []groveschedulerv1alpha1.NamespacedName{{Namespace: ordinary.Namespace, Name: agent.CliqueName + "-0"}}, + }) + } + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + t.Log("Keep publication closed for the actual defect, without creating an LPR") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxClosed{}, classification) + require.Equal(t, test.wantMessage, classification.(*lpxClosed).incomplete) + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + require.Empty(t, requests) + }) + } +} + +func TestLPXLiveRequestSizeGrowthRetiresPublication(t *testing.T) { + ctx := context.Background() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + require.NotNil(t, desired) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + + t.Log("Publish the request, then model API-server metadata growth") + classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.IsType(t, &lpxOpen{}, classification) + storedClient := reconciler.Client + storedWithWatch, ok := storedClient.(client.WithWatch) + require.True(t, ok) + reconciler.apiReader = interceptor.NewClient(storedWithWatch, interceptor.Funcs{ + List: func( + ctx context.Context, + delegated client.WithWatch, + list client.ObjectList, + opts ...client.ListOption, + ) error { + if err := delegated.List(ctx, list, opts...); err != nil { + return err + } + requests, ok := list.(*lpxv1alpha1.LPUPipelineRequestList) + if !ok { + return nil + } + for index := range requests.Items { + request := &requests.Items[index] + if request.Name != desired.requests[0].requestName { + continue + } + request.ManagedFields = []metav1.ManagedFieldsEntry{{ + Manager: "metadata-growth-test", + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{ + Raw: []byte(`{"padding":"` + strings.Repeat("x", 850*1024) + `"}`), + }, + }} + } + return nil + }, + }) + + t.Log("Retire the now-oversized publication") + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + sizeRejected, ok := classification.(*lpxRetiring) + require.True(t, ok) + require.Contains(t, sizeRejected.retirementReason, "scheduler size budget") + requireLPXRequestNotFound(t, ctx, storedClient, dgd.Namespace, desired.requests[0].requestName) +} + +func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t *testing.T) { + t.Log("Give foreign requests the desired names in the opposite lexical order") + ctx := t.Context() + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + dgd := newLPXTestDeployment(t, source) + foreign := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "z-foreign-request", + Namespace: dgd.Namespace, + UID: "foreign-collision-uid", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta1.GroupVersion.String(), + Kind: nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind, + Name: lpxTestOtherName, + UID: "other-uid", + Controller: ptr.To(true), + }}, + }, + } + second := foreign.DeepCopy() + second.Name, second.UID = "a-foreign-request", "second-foreign-collision-uid" + desired := &lpxMaterializing{requests: []lpxModelMaterializing{ + {requestName: foreign.Name}, {requestName: second.Name}, + }} + reconciler := newLPXTestReconciler(t, nil, dgd, source) + + t.Log("Complete an empty preflight fence before foreign requests occupy the desired names") + currents, retiring, err := reconciler.reconcileLPXAttemptFence(ctx, dgd, desired) + require.NoError(t, err) + require.Empty(t, currents) + require.Nil(t, retiring) + for _, request := range []*lpxv1alpha1.LPUPipelineRequest{foreign, second} { + require.NoError(t, reconciler.Create(ctx, request)) + } + + t.Log("Exclude foreign requests from owned publications and the known-intent retirement fence") + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + require.NoError(t, err) + require.Empty(t, requests) + classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + require.NoError(t, err) + require.Nil(t, classification) + + t.Log("Observe exact-name collisions with one bounded authoritative list and no per-request Gets") + reader, ok := reconciler.apiReader.(client.WithWatch) + require.True(t, ok) + requestGets := 0 + requestListCalls := make([]client.ListOptions, 0) + reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ + Get: func( + ctx context.Context, + delegated client.WithWatch, + key client.ObjectKey, + object client.Object, + opts ...client.GetOption, + ) error { + if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + requestGets++ + } + return delegated.Get(ctx, key, object, opts...) + }, + List: func( + ctx context.Context, + delegated client.WithWatch, + list client.ObjectList, + opts ...client.ListOption, + ) error { + if _, ok := list.(*lpxv1alpha1.LPUPipelineRequestList); ok { + requestListCalls = append(requestListCalls, *(&client.ListOptions{}).ApplyOptions(opts)) + } + return delegated.List(ctx, list, opts...) + }, + }) + + currents, retiring, err = reconciler.reconcileLPXAttemptFence(ctx, dgd, desired) + require.EqualError(t, err, fmt.Sprintf("LPX request name %q is occupied by a foreign owner", foreign.Name)) + require.Nil(t, currents) + require.Nil(t, retiring) + require.Zero(t, requestGets, "exact-name collision detection must not issue one request Get per desired projection") + require.Len(t, requestListCalls, 1) + require.Equal(t, int64(lpxLifecycleListPageSize), requestListCalls[0].Limit) + require.Equal(t, dgd.Namespace, requestListCalls[0].Namespace) + require.True(t, requestListCalls[0].LabelSelector == nil || requestListCalls[0].LabelSelector.Empty()) + + t.Log("Refresh publication state and refuse the foreign requests created after the empty preflight") + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.ErrorContains(t, err, "refusing to adopt foreign LPX request") + require.Nil(t, classification) + require.Len(t, requestListCalls, 2, "final publication must refresh the authoritative request snapshot") + for _, request := range []*lpxv1alpha1.LPUPipelineRequest{foreign, second} { + stored := &lpxv1alpha1.LPUPipelineRequest{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(request), stored)) + require.Equal(t, request.UID, stored.UID) + require.Equal(t, request.OwnerReferences, stored.OwnerReferences) + require.True(t, stored.DeletionTimestamp.IsZero()) + } + allRequests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, reconciler.List(ctx, allRequests, client.InNamespace(dgd.Namespace))) + require.Len(t, allRequests.Items, 2, "the fresh final snapshot must not publish alongside the collisions") +} + +func requireLPXClosed(t *testing.T, classification lpxClassification) *lpxClosed { + t.Helper() + closed, ok := classification.(*lpxClosed) + require.True(t, ok, "classification=%T", classification) + return closed +} + +func newLPXNodeLocalBoundStatus(generation, revision int64, planDigest lpxv1alpha1.PlanDigest) *lpxv1alpha1.LPUPipelineRequestStatus { + // Construct a fresh exact-generation NodeLocal scheduler receipt. + return &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: lpxv1alpha1.RequestPhaseBound, + ObservedGeneration: ptr.To(generation), + LastPlanRevision: revision, + Committed: &lpxv1alpha1.Committed{ + Execution: lpxv1alpha1.CommittedExecution{ + AcceptedGeneration: ptr.To(generation), + NodeLocal: &lpxv1alpha1.NodeLocalExecution{}, + }, + Plan: lpxv1alpha1.CommittedPlan{ + PlannedFromGeneration: generation, + Revision: revision, + PlanDigest: planDigest, + Placement: lpxv1alpha1.PlanPlacement{ + ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, + NodeLocal: &lpxv1alpha1.NodeLocalPlacement{}, + }, + }, + }, + } +} + +func findLPXTestClique(t *testing.T, objects []client.Object, name string) *grovev1alpha1.PodClique { + t.Helper() + for _, object := range objects { + if clique, ok := object.(*grovev1alpha1.PodClique); ok && clique.Name == name { + return clique + } + } + t.Fatalf("PodClique %q not found", name) + return nil +} + +func findLPXTestScalingGroup(t *testing.T, objects []client.Object, name string) *grovev1alpha1.PodCliqueScalingGroup { + t.Helper() + for _, object := range objects { + if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && group.Name == name { + return group + } + } + t.Fatalf("PodCliqueScalingGroup %q not found", name) + return nil +} + +func findLPXTestPodGang(t *testing.T, objects []client.Object, schedulerName string) *groveschedulerv1alpha1.PodGang { + t.Helper() + for _, object := range objects { + if gang, ok := object.(*groveschedulerv1alpha1.PodGang); ok && gang.Labels[grovecommon.LabelSchedulerName] == schedulerName { + return gang + } + } + t.Fatalf("PodGang for scheduler %q not found", schedulerName) + return nil +} + +func findLPXTestPodCliqueSet(t *testing.T, objects []client.Object) *grovev1alpha1.PodCliqueSet { + t.Helper() + for _, object := range objects { + if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + return pcs + } + } + t.Fatal("PodCliqueSet not found") + return nil +} + +func renderLPXTestPodCliqueSet( + t *testing.T, ctx context.Context, reconciler *graphReconciler, + dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, desired *lpxMaterializing, +) *grovev1alpha1.PodCliqueSet { + t.Helper() + rendered, _, err := renderPodCliqueSet(ctx, source, reconciler.Config, reconciler.runtimeConfig, + reconciler.Client, reconciler.DockerSecretRetriever, desired.workload, desired.plan, dgd) + require.NoError(t, err) + return rendered +} + +func newLPXTestDGD(t *testing.T, pipeline lpx.Pipeline) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, *lpx.ModelRegistry) { + t.Helper() + compilationMode := manifestcapnpv2.CompilationMode_lpuOnly + if pipeline == lpx.PipelineLPX { + compilationMode = manifestcapnpv2.CompilationMode_lpx + } + const buildID = "build-v2" + registry := newLPXTestRegistryWithPartitionsAndMode(t, buildID, []int{7, 8}, compilationMode) + source := newLPXTestSource(pipeline, buildID) + return newLPXTestDeployment(t, source), source, registry +} + +func newLPXTestDeployment(t *testing.T, source *nvidiacomv1beta1.DynamoGraphDeployment) *nvidiacomv1alpha1.LPXGraphDeployment { + t.Helper() + revision, err := dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + return &nvidiacomv1alpha1.LPXGraphDeployment{ + TypeMeta: metav1.TypeMeta{APIVersion: nvidiacomv1alpha1.GroupVersion.String(), Kind: "LPXGraphDeployment"}, + ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: types.UID("lpx-" + string(source.UID)), Generation: 1, + Annotations: map[string]string{dynamo.LPXPCSNameAnnotation: dynamo.PCSNameForLPX(source), lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + Finalizers: []string{lpxGraphDeploymentFinalizer}, + }, + Spec: nvidiacomv1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, + } +} + +func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.DynamoGraphDeployment { + one := int32(1) + source := &nvidiacomv1beta1.DynamoGraphDeployment{ + TypeMeta: metav1.TypeMeta{APIVersion: nvidiacomv1beta1.GroupVersion.String(), Kind: "DynamoGraphDeployment"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-source", Namespace: "test", UID: "source-uid", Generation: 3, + Annotations: map[string]string{consts.KubeAnnotationLPXSchedulerBackend: consts.LPXSchedulerBackend}, + }, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, Replicas: &one, + LPX: &nvidiacomv1beta1.LPXConfig{ + BuildID: buildID, + Settings: &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":true}`)}, + }, + Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLeader}, { + Name: nvidiacomv1beta1.ComponentRoleWorker, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: consts.MainContainerName, Image: "lpu-runtime", + VolumeMounts: []corev1.VolumeMount{{ + Name: consts.ModelStorageVolumeName, MountPath: "/models", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: consts.ModelStorageVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }}, + }}, + }}, + }}, + }, + } + if pipeline == lpx.PipelineLPX { + *source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader) = nvidiacomv1beta1.ComponentRoleSpec{ + Name: nvidiacomv1beta1.ComponentRoleLeader, + Replicas: &one, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + ResourceClaims: []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("gpu")}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "cyborg-runtime"}}, + }}, + } + } + return source +} + +func newLPXSpecDecodeTestDGD(t *testing.T) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, *lpx.ModelRegistry) { + t.Helper() + source := newLPXSpecDecodeTestSource() + root := t.TempDir() + writeLPXTestBuild(t, root, "draft-build", []int{7, 8}, manifestcapnpv2.CompilationMode_lpuOnly) + writeLPXTestBuild(t, root, "target-build", []int{9, 10}, manifestcapnpv2.CompilationMode_lpuOnly) + registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() + registry, err := lpx.NewModelRegistry(registryURL, nil) + require.NoError(t, err) + return newLPXTestDeployment(t, source), source, registry +} + +func newLPXSpecDecodeTestSource() *nvidiacomv1beta1.DynamoGraphDeployment { + source := newLPXTestSource(lpx.PipelineSingle, "target-build") + target := &source.Spec.Components[0] + draft := target.DeepCopy() + draft.ComponentName, draft.LPX.BuildID, draft.Replicas = "draft", "draft-build", ptr.To(int32(2)) + draft.Roles = []nvidiacomv1beta1.ComponentRoleSpec{*draft.ComponentRole(nvidiacomv1beta1.ComponentRoleWorker)} + source.Spec.Components = append(source.Spec.Components, *draft) + return source +} + +func newLPXTestRegistryWithPartitionsAndMode( + t *testing.T, + buildID string, + partitionIDs []int, + compilationMode manifestcapnpv2.CompilationMode, +) *lpx.ModelRegistry { + t.Helper() + root := t.TempDir() + writeLPXTestBuild(t, root, buildID, partitionIDs, compilationMode) + registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() + registry, err := lpx.NewModelRegistry(registryURL, nil) + require.NoError(t, err) + return registry +} + +func writeLPXTestBuild( + t *testing.T, + root string, + buildID string, + partitionIDs []int, + compilationMode manifestcapnpv2.CompilationMode, +) { + t.Helper() + require.NotEmpty(t, partitionIDs) + buildDir := filepath.Join(root, buildID) + require.NoError(t, os.Mkdir(buildDir, 0o700)) + + t.Log("Encode the V2 manifest header and the tokenizer consumed by rendered runtimes") + message, manifest := newTestGraphManifest(t) + + t.Log("Retain the opaque compiler provenance without the removed V1 publication contract") + populateTestGraphBuild(t, manifest, buildID) + + t.Log("Describe the same one-batch runtime with explicit V2 runtime I/O and prop-sync evidence") + deployment, program := newTestGraphProgram(t, manifest, compilationMode, uint32(len(partitionIDs)*2), 8192) + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + chains, err := deployment.NewSelectedPropSyncChains(1) + require.NoError(t, err) + selectedIDs, err := chains.At(0).NewPartitionIds(int32(len(partitionIDs))) + require.NoError(t, err) + for index, partitionID := range partitionIDs { + selectedIDs.Set(index, uint32(partitionID)) + } + + t.Log("Package the LPU partitions and the hybrid CUDA marker in the flat V2 artifact inventory") + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitionCount := len(partitionIDs) + if compilationMode == manifestcapnpv2.CompilationMode_lpx { + partitionCount++ + } + partitions, err := artifacts.NewPartitions(int32(partitionCount)) + require.NoError(t, err) + for index, partitionID := range partitionIDs { + partition := partitions.At(index) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(uint32(partitionID)) + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", partitionID))) + require.NoError(t, detail.SetTopology("URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA")) + detail.SetNumChips(16) + detail.SetDevicesPerNode(8) + } + if compilationMode == manifestcapnpv2.CompilationMode_lpx { + partition := partitions.At(len(partitionIDs)) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_cuda) + ref.SetPartitionId(uint32(len(partitionIDs))) + _, err = partition.Detail().NewCuda() + require.NoError(t, err) + } + + t.Log("Publish the required V2 compiler manifest") + payload, err := message.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), payload, 0o600)) +} + +func newLPXTestReconciler( + t *testing.T, + registry lpxModelRegistry, + dgd *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + objects ...client.Object, +) *graphReconciler { + t.Helper() + revision, err := dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + dgd.Spec.InputRevision = revision + dgd.Annotations[dynamo.LPXPCSNameAnnotation] = dynamo.PCSNameForLPX(source) + if dgd.ResourceVersion == "" { + dgd.ResourceVersion = "1" + } + scheme := newLPXTestScheme(t) + seed := append([]client.Object{dgd.DeepCopy(), source.DeepCopy()}, objects...) + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). + WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimRefIndex, lpxDRAClaimReferences(false)). + WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimTemplateRefIndex, lpxDRAClaimReferences(true)). + WithIndex(&resourcev1.ResourceClaim{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences). + WithIndex(&resourcev1.ResourceClaimTemplate{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences). + WithStatusSubresource(&nvidiacomv1beta1.DynamoGraphDeployment{}, &nvidiacomv1alpha1.LPXGraphDeployment{}). + WithObjects(seed...). + Build() + nextLPRUID := 0 + wrapped := interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + if request.UID == "" { + nextLPRUID++ + request.UID = types.UID(fmt.Sprintf("lpr-%s-%d", request.Name, nextLPRUID)) + } + if request.Generation == 0 { + request.Generation = 1 + } + if request.CreationTimestamp.IsZero() { + request.CreationTimestamp = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) + } + } + return delegated.Create(ctx, object, opts...) + }, + }) + recorder := events.NewFakeRecorder(100) + config := &configv1alpha1.OperatorConfiguration{ + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + } + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, DRA: true, LPX: true}} + return &graphReconciler{ + Client: wrapped, + recorder: recorder, + apiReader: wrapped, + runtimeConfig: runtimeConfig, + modelRegistry: registry, + Config: config, + } +} + +func lpxMaterializedObjects( + t *testing.T, + reconciler *graphReconciler, + dgd *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + desired *lpxMaterializing, +) []client.Object { + t.Helper() + t.Log("Render LPX intent once and emulate the API-server and Grove observations") + pcs := renderLPXTestPodCliqueSet(t, t.Context(), reconciler, dgd, source, desired) + pcs.TypeMeta = metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet"} + pcs.UID, pcs.Generation = "pcs-uid", 1 + pcs.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)} + const generationHash = "pcs-generation-hash" + pcs.Status = grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(pcs.Generation), CurrentGenerationHash: ptr.To(generationHash), + } + + // Grove materializes the rendered scaling-group configuration once. + groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].DeepCopy() + group := &grovev1alpha1.PodCliqueScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: desired.plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: "lpu-group-uid", + Generation: 1, Labels: maps.Clone(pcs.Labels), Annotations: groupTemplate.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: ptr.Deref(groupTemplate.Replicas, 1), MinAvailable: groupTemplate.MinAvailable, + CliqueNames: groupTemplate.CliqueNames, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + }, + } + group.Labels[grovecommon.LabelPartOfKey] = pcs.Name + group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" + objects := []client.Object{pcs, group} + + // Each Grove replica owns independent cliques and one gang per scheduler. + for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { + gangs := make(map[string]*groveschedulerv1alpha1.PodGang, 2) + for _, scheduler := range []string{corev1.DefaultSchedulerName, lpx.SchedulerName} { + name := fmt.Sprintf("selected-dgd-%d-%s", replicaIndex, scheduler) + gangs[scheduler] = &groveschedulerv1alpha1.PodGang{ + TypeMeta: metav1.TypeMeta{APIVersion: groveschedulerv1alpha1.SchemeGroupVersion.String(), Kind: "PodGang"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), + Labels: maps.Clone(pcs.Labels), Annotations: maps.Clone(pcs.Annotations), + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + } + gangs[scheduler].Labels[grovecommon.LabelPartOfKey] = pcs.Name + gangs[scheduler].Labels[grovecommon.LabelSchedulerName] = scheduler + } + parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} + for _, template := range pcs.Spec.Template.Cliques { + rendered := template.DeepCopy() + name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) + gang := gangs[rendered.Spec.PodSpec.SchedulerName] + clique := &grovev1alpha1.PodClique{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, + Labels: rendered.Labels, Annotations: rendered.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }, + Spec: rendered.Spec, + Status: grovev1alpha1.PodCliqueStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + CurrentPodTemplateHash: ptr.To(rendered.Name + "-pod-template-hash"), + }, + } + clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name + clique.Labels[grovecommon.LabelPodGang] = gang.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) + for index, dependency := range clique.Spec.StartsAfter { + clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) + } + objects = append(objects, clique) + + // Gang references, not fake Pods, witness each clique's rendered cardinality. + podGroup := groveschedulerv1alpha1.PodGroup{Name: name, MinReplicas: ptr.Deref(clique.Spec.MinAvailable, clique.Spec.Replicas)} + for podIndex := int32(0); podIndex < clique.Spec.Replicas; podIndex++ { + podGroup.PodReferences = append(podGroup.PodReferences, groveschedulerv1alpha1.NamespacedName{ + Namespace: pcs.Namespace, Name: fmt.Sprintf("%s-%d", name, podIndex), + }) + } + gang.Spec.PodGroups = append(gang.Spec.PodGroups, podGroup) + } + objects = append(objects, gangs[corev1.DefaultSchedulerName], gangs[lpx.SchedulerName]) + } + return objects +} + +func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.Client, objects ...client.Object) { + t.Helper() + for _, object := range objects { + require.NoError(t, kubeClient.Create(ctx, object.DeepCopyObject().(client.Object))) + } +} + +func getLPXRequest(t *testing.T, ctx context.Context, kubeClient client.Reader, namespace, name string) *lpxv1alpha1.LPUPipelineRequest { + t.Helper() + request := &lpxv1alpha1.LPUPipelineRequest{} + require.NoError(t, kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request)) + return request +} + +func requireLPXRequestNotFound( + t *testing.T, + ctx context.Context, + kubeClient client.Reader, + namespace string, + name string, +) { + t.Helper() + request := &lpxv1alpha1.LPUPipelineRequest{} + err := kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request) + require.True(t, apierrors.IsNotFound(err), "expected LPX request %s/%s to be absent, got %v", namespace, name, err) +} diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go new file mode 100644 index 000000000000..4b0392744128 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "errors" + "fmt" + "net/url" + "sort" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + modelDownloadRequeueAfter = 5 * time.Second + modelDownloadPendingReason string = "model_download_in_progress" + modelDownloadPendingMessage string = "Waiting for model downloads to complete" + + modelDownloadRefreshInterval = 24 * time.Hour + modelDownloadRefreshTimeout = 30 * time.Second +) + +type lpxModelRegistry interface { + BuildURL(id string) (*url.URL, error) + EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) + AcquireBuildSnapshot(ctx context.Context, id string) (*lpx.BuildSnapshot, error) +} + +// newLPXModelRegistry constructs a registry from the non-nil operator configuration. +func newLPXModelRegistry(config *configv1alpha1.OperatorConfiguration) (*lpx.ModelRegistry, error) { + // Configure the optional Model Express client. + var mxClient modelpb.ModelServiceClient + var err error + if config.Infrastructure.ModelExpressURL != "" { + mxClient, err = lpx.NewModelExpressClient(config.Infrastructure.ModelExpressURL) + if err != nil { + return nil, fmt.Errorf("unable to create Model Express client: %w", err) + } + log.Log.WithName("setup").Info("LPX Model Express client configured", "modelExpressURL", config.Infrastructure.ModelExpressURL) + } + + // Construct the registry with the configured download client. + registry, err := lpx.NewModelRegistry(config.LPX.ModelRegistryURL, mxClient) + if err != nil { + return nil, fmt.Errorf("unable to create LPX model registry client: %w", err) + } + return registry, nil +} + +// reconcileModelDownloads checks the nonnil DGD's remote builds and updates the +// owned, nonnil child's download status without mutating the DGD. forceCheck +// bypasses the refresh interval until an immutable LPX selection exists. +func (r *graphReconciler) reconcileModelDownloads( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + forceCheck bool, +) (bool, error) { + current := deployment.Status.ObservedGeneration == deployment.Generation + previous := deployment.Status.ModelDownload + if !forceCheck && current && previous != nil && previous.LastCheckedAt != nil && + time.Since(previous.LastCheckedAt.Time) < modelDownloadRefreshInterval { + return true, nil + } + running := !forceCheck && current && meta.IsStatusConditionTrue(deployment.Status.Conditions, "Ready") + if running { + refreshCtx, cancel := context.WithTimeout(ctx, modelDownloadRefreshTimeout) + defer cancel() + ctx = refreshCtx + } + + downloaded, ready, err := ensureModelsDownloaded(ctx, dgd, r.modelRegistry) + checkSucceeded := ready && err == nil + // Retain the running child's last observation when refresh is incomplete. + if !running || checkSucceeded { + deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: downloaded} + } else if previous == nil { + deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{} + } + if running || checkSucceeded { + checkedAt := metav1.Now() + deployment.Status.ModelDownload.LastCheckedAt = &checkedAt + } + if !running { + return ready, err + } + if err != nil { + log.FromContext(ctx).Error(err, "failed to refresh model downloads", "dgd", dgd.Name) + } + return true, nil +} + +func ensureModelsDownloaded( + ctx context.Context, + dgd *v1beta1.DynamoGraphDeployment, + registry lpxModelRegistry, +) ([]string, bool, error) { + builds, err := collectBuilds(dgd, registry) + if err != nil { + return nil, false, err + } + + downloaded := make([]string, 0, len(builds)) + var downloadErrors []error + for i, buildURL := range builds { + build := buildURL.String() + buildCtx := ctx + cancel := func() {} + if deadline, ok := ctx.Deadline(); ok { + perBuildTimeout := time.Until(deadline) / time.Duration(len(builds)-i) + buildCtx, cancel = context.WithTimeout(ctx, perBuildTimeout) + } + buildDownloaded, err := registry.EnsureDownloaded(buildCtx, buildURL) + cancel() + if err != nil { + downloadErrors = append(downloadErrors, fmt.Errorf("ensure model %q is downloaded: %w", build, err)) + continue + } + if !buildDownloaded { + continue + } + + downloaded = append(downloaded, build) + } + + return downloaded, len(downloaded) == len(builds), errors.Join(downloadErrors...) +} + +func collectBuilds(dgd *v1beta1.DynamoGraphDeployment, registry lpxModelRegistry) ([]url.URL, error) { + buildsByKey := make(map[string]url.URL) + + for _, component := range dgd.Spec.Components { + config := component.LPX + if config == nil { + continue + } + + // Each LPX component owns one build, including an agent-only draft. + buildURL, err := registry.BuildURL(config.BuildID) + if err != nil { + return nil, fmt.Errorf("resolve LPX build URL %q: %w", config.BuildID, err) + } + + if buildURL.Scheme == lpx.BuildSchemeGCS { + buildsByKey[buildURL.String()] = *buildURL + } + } + + builds := make([]url.URL, 0, len(buildsByKey)) + for _, build := range buildsByKey { + builds = append(builds, build) + } + sort.Slice(builds, func(i, j int) bool { + return builds[i].String() < builds[j].String() + }) + + return builds, nil +} diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go new file mode 100644 index 000000000000..88e814ee851c --- /dev/null +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -0,0 +1,355 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "fmt" + "net/url" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" +) + +const ( + modelDownloadTestBuildID = "gs://bucket/model/build" + modelDownloadTestSecondBuildID = "gs://bucket/model/second-build" + modelDownloadTestLocalBuildID = "file:///models/local-build" + modelDownloadTestDGDName = "test-dgd" + modelDownloadTestNamespace = "default" +) + +func TestEnsureModelsDownloaded(t *testing.T) { + t.Log("Use the real registry to check missing Model Express configuration") + withoutModelExpress, err := lpx.NewModelRegistry("", nil) + require.NoError(t, err) + + t.Log("Own a mixed graph with reversed build order and a previously cached source") + pending := newModelDownloadDGD(modelDownloadTestSecondBuildID, modelDownloadTestBuildID) + pending.Spec.Components = append([]v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "worker", ComponentType: v1beta1.ComponentTypeWorker, + }}, pending.Spec.Components...) + cached := newModelDownloadDGD(modelDownloadTestBuildID, modelDownloadTestSecondBuildID) + cached.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{modelDownloadTestBuildID}}, + } + tests := []struct { + name string + dgd *v1beta1.DynamoGraphDeployment + registry lpxModelRegistry + wantReady bool + wantErr string + wantCalls []string + wantBuilds []string + }{ + { + name: "returns Model Express client error for remote builds when Model Express is not configured", + dgd: newModelDownloadDGD(modelDownloadTestBuildID), + registry: withoutModelExpress, + wantErr: "Model Express client is required for GCS model downloads", + }, + { + name: "local build does not call Model Express", + dgd: newModelDownloadDGD(modelDownloadTestLocalBuildID), + registry: newModelDownloadRegistry(t, nil, nil), + wantReady: true, + }, + { + name: "deduplicates the same build across draft and target", + dgd: newModelDownloadDGD(modelDownloadTestBuildID, modelDownloadTestBuildID), + registry: newModelDownloadRegistry(t, map[string]bool{modelDownloadTestBuildID: true}, nil), + wantReady: true, + wantCalls: []string{modelDownloadTestBuildID}, + wantBuilds: []string{modelDownloadTestBuildID}, + }, + { + name: "checks every remote build regardless of cached status", + dgd: cached, + registry: newModelDownloadRegistry(t, map[string]bool{ + modelDownloadTestBuildID: true, + modelDownloadTestSecondBuildID: true, + }, nil), + wantReady: true, + wantCalls: []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, + wantBuilds: []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, + }, + { + name: "skips ordinary components, sorts remote builds and records partial progress", + dgd: pending, + registry: newModelDownloadRegistry(t, map[string]bool{ + modelDownloadTestBuildID: true, + }, nil), + wantReady: false, + wantCalls: []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, + wantBuilds: []string{modelDownloadTestBuildID}, + }, + { + name: "returns build URL resolution error", + dgd: newModelDownloadDGD("relative-build-id"), + registry: newModelDownloadRegistry(t, nil, nil), + wantErr: "model registry URL is not configured", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Check every selected remote build without acquiring its snapshot") + builds, ready, err := ensureModelsDownloaded(t.Context(), tt.dgd, tt.registry) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.wantReady, ready) + require.True(t, slices.Equal(builds, tt.wantBuilds), "downloaded builds = %v, want %v", builds, tt.wantBuilds) + if registry, ok := tt.registry.(*fakeModelDownloadRegistry); ok { + require.Equal(t, tt.wantCalls, registry.calls) + require.Zero(t, registry.acquireBuildSnapshotCalls) + } + }) + } +} + +func TestEnsureModelsDownloadedSharesDeadlineAcrossBuilds(t *testing.T) { + t.Log("Give both builds a shared deadline and let the first download fail") + dgd := newModelDownloadDGD(modelDownloadTestBuildID, modelDownloadTestSecondBuildID) + registry := newModelDownloadRegistry(t, + map[string]bool{modelDownloadTestBuildID: true, modelDownloadTestSecondBuildID: true}, + map[string]error{modelDownloadTestBuildID: fmt.Errorf("download failed: %w", context.DeadlineExceeded)}, + ) + ctx, cancel := context.WithTimeout(t.Context(), time.Hour) + defer cancel() + + t.Log("Continue checking later builds and retain their completed progress") + builds, ready, err := ensureModelsDownloaded(ctx, dgd, registry) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "download failed") + require.False(t, ready) + require.Equal(t, []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, registry.calls) + require.Equal(t, []string{modelDownloadTestSecondBuildID}, builds) + require.Zero(t, registry.acquireBuildSnapshotCalls) + + t.Log("Reserve time for the second build without extending the parent's budget") + deadline, _ := ctx.Deadline() + require.Len(t, registry.deadlines, 2) + require.False(t, registry.deadlines[0].IsZero()) + require.True(t, registry.deadlines[0].Before(registry.deadlines[1])) + require.Equal(t, deadline, registry.deadlines[1]) +} + +func TestRunningLPXModelDownloadRefresh(t *testing.T) { + tests := []struct { + name string + checkedAt time.Time + ready bool + err error + wantCalls int + }{ + { + name: "fresh check skips ModelExpress", + checkedAt: time.Now(), + ready: true, + }, + { + name: "stale check calls ModelExpress", + checkedAt: time.Now().Add(-modelDownloadRefreshInterval), + ready: true, + wantCalls: 1, + }, + { + name: "stale check remains ready when ModelExpress is down", + checkedAt: time.Now().Add(-modelDownloadRefreshInterval), + err: fmt.Errorf("ModelExpress unavailable"), + wantCalls: 1, + }, + { + name: "stale check remains ready while an evicted model redownloads", + checkedAt: time.Now().Add(-modelDownloadRefreshInterval), + wantCalls: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Refresh a running workload while preserving its downloaded builds") + dgd := newModelDownloadDGD(modelDownloadTestBuildID) + child := newLPXTestDeployment(t, dgd) + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ + Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: &metav1.Time{Time: tt.checkedAt}, + } + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}} + child.Status.ObservedGeneration = child.Generation + registry := newModelDownloadRegistry(t, + map[string]bool{modelDownloadTestBuildID: tt.ready}, + map[string]error{modelDownloadTestBuildID: tt.err}, + ) + + lpx := &graphReconciler{modelRegistry: registry} + for range 2 { + ready, err := lpx.reconcileModelDownloads(t.Context(), child, dgd, false) + require.NoError(t, err) + require.True(t, ready) + } + if len(registry.calls) != tt.wantCalls { + t.Fatalf("ModelExpress calls = %d, want %d", len(registry.calls), tt.wantCalls) + } + if tt.wantCalls > 0 && !child.Status.ModelDownload.LastCheckedAt.After(tt.checkedAt) { + t.Fatalf("last check = %s, want after %s", child.Status.ModelDownload.LastCheckedAt, tt.checkedAt) + } + if !slices.Equal(child.Status.ModelDownload.Builds, []string{modelDownloadTestBuildID}) { + t.Fatalf("downloaded builds = %v, want preserved", child.Status.ModelDownload.Builds) + } + }) + } +} + +func TestModelDownloadSpecChangeRemainsFailClosed(t *testing.T) { + t.Log("Keep the source's download status independent of the unobserved child") + source := newModelDownloadDGD(modelDownloadTestBuildID) + source.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{modelDownloadTestBuildID}}, + } + child := newLPXTestDeployment(t, source) + child.Status.ObservedGeneration = child.Generation - 1 + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}} + child.Status.ModelDownload = source.Status.LPX.ModelDownload.DeepCopy() + child.Status.ModelDownload.LastCheckedAt = ptr.To(metav1.Now()) + registry := newModelDownloadRegistry(t, nil, map[string]error{modelDownloadTestBuildID: fmt.Errorf("ModelExpress unavailable")}) + lifecycle := &graphReconciler{modelRegistry: registry} + t.Log("Do not reuse a fresh download check for an unobserved LPX generation") + ready, err := lifecycle.reconcileModelDownloads(t.Context(), child.DeepCopy(), source, false) + require.ErrorContains(t, err, "ModelExpress unavailable") + require.False(t, ready) + t.Log("The child controller persists failed download progress without publishing a PCS") + r := newLPXTestReconciler(t, registry, child, source) + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.ErrorContains(t, err, "ModelExpress unavailable") + updated := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), updated)) + require.NotNil(t, updated.Status.ModelDownload) + require.Empty(t, updated.Status.ModelDownload.Builds) + require.False(t, meta.IsStatusConditionTrue(updated.Status.Conditions, "Ready")) + require.Equal(t, []string{modelDownloadTestBuildID}, source.Status.LPX.ModelDownload.Builds) +} + +func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { + t.Log("A new revision retains old placement and download observations without a scheduling deadline") + source := newModelDownloadDGD(modelDownloadTestSecondBuildID) + source.Generation++ + child := newLPXTestDeployment(t, source) + child.Status.ObservedGeneration = child.Generation + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ + Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: ptr.To(metav1.Now()), + } + child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ + ObservedGeneration: child.Generation, + DeadlineAt: ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))), + Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "previous-request", UID: "previous-uid"}}, + }} + child.Generation++ + previousGeneration := child.Status.ObservedGeneration + require.Nil(t, lpxAttemptDeadlineSeconds(source)) + registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestSecondBuildID: true}, nil) + r := newLPXTestReconciler(t, registry, child, source) + r.runtimeConfig.Gate = features.Gates{LPX: true} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + previousPlacement := child.Status.Placement.DeepCopy() + + t.Log("Disabled reconciliation reports the new failure without marking stale placement as observed") + for range 2 { + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, previousGeneration, child.Status.ObservedGeneration) + require.Nil(t, child.Status.ModelDownload) + require.Equal(t, previousPlacement, child.Status.Placement) + require.Empty(t, registry.calls, "the early exit has not checked either build") + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + require.Equal(t, child.Generation, failed.ObservedGeneration) + require.Equal(t, "LPXUnavailable", failed.Reason) + } + + t.Log("Resumed download checking cannot reuse the previous revision's cached success") + r.runtimeConfig.Gate.Grove = true + ready, err := r.reconcileModelDownloads(t.Context(), child, source, false) + require.NoError(t, err) + require.True(t, ready) + require.Equal(t, []string{modelDownloadTestSecondBuildID}, registry.calls) + require.Equal(t, []string{modelDownloadTestSecondBuildID}, child.Status.ModelDownload.Builds) + require.NotNil(t, child.Status.ModelDownload.LastCheckedAt) +} + +type fakeModelDownloadRegistry struct { + *lpx.ModelRegistry + ready map[string]bool + err map[string]error + calls []string + deadlines []time.Time + acquireBuildSnapshotCalls int +} + +func (r *fakeModelDownloadRegistry) AcquireBuildSnapshot(ctx context.Context, id string) (*lpx.BuildSnapshot, error) { + r.acquireBuildSnapshotCalls++ + return r.ModelRegistry.AcquireBuildSnapshot(ctx, id) +} + +func (r *fakeModelDownloadRegistry) EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) { + build := buildURL.String() + r.calls = append(r.calls, build) + deadline, _ := ctx.Deadline() + r.deadlines = append(r.deadlines, deadline) + return r.ready[build], r.err[build] +} + +func newModelDownloadRegistry(t *testing.T, ready map[string]bool, errs map[string]error) *fakeModelDownloadRegistry { + t.Helper() + + registry, err := lpx.NewModelRegistry("", nil) + require.NoError(t, err) + return &fakeModelDownloadRegistry{ModelRegistry: registry, ready: ready, err: errs} +} + +func newModelDownloadDGD(buildIDs ...string) *v1beta1.DynamoGraphDeployment { + dgd := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: modelDownloadTestDGDName, + Namespace: modelDownloadTestNamespace, + Generation: 1, + UID: types.UID("test-dgd-uid"), + }, + } + // Each build belongs to a distinct public component; only the target serves. + for i, buildID := range buildIDs { + component := v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "lpx-worker", ComponentType: v1beta1.ComponentTypeLPX, Replicas: ptr.To(int32(1)), + LPX: &v1beta1.LPXConfig{BuildID: buildID}, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime"}}, + }}}}, + } + if i == len(buildIDs)-1 { + component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLeader}) + } else { + component.ComponentName = "draft" + } + dgd.Spec.Components = append(dgd.Spec.Components, component) + } + return dgd +} diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go new file mode 100644 index 000000000000..4d6f425a57d3 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lpx + +import ( + "context" + "errors" + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "k8s.io/apimachinery/pkg/util/validation/field" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" +) + +// reconcileLPXSafetyPreflight classifies and fences scheduler-owned state +// after the LPX controller validates source identity and the Grove provider. +func (r *graphReconciler) reconcileLPXSafetyPreflight( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + requests []lpxv1alpha1.LPUPipelineRequest, +) (*lpxMaterializing, lpxClassification, error) { + classification, requests, err := r.reconcileLPXKnownIntentFence(ctx, deployment, source, requests) + if err != nil { + return nil, nil, fmt.Errorf("failed to fence prior LPX intent: %w", err) + } + if classification != nil { + return nil, classification, nil + } + + // The serving component owns every Grove name, including the shared draft cliques. + serving := dynamolpx.ServingComponent(source) + for index := range source.Spec.Components { + component := &source.Spec.Components[index] + if component != serving { + continue + } + combinedLength := len(dynamo.PCSNameForLPX(source)) + dynamo.LPXComponentNameBudget(component.ComponentName) + if combinedLength > consts.MaxCombinedGroveResourceNameLength { + err := field.Invalid(field.NewPath("spec", "components").Index(index).Child("name"), component.ComponentName, + fmt.Sprintf("combined Grove resource name length %d exceeds the %d-character limit; shorten the deployment or component name", + combinedLength, consts.MaxCombinedGroveResourceNameLength)) + return nil, &lpxRejected{reason: err.Error()}, nil + } + break + } + + if len(requests) == 0 { + if err := dynamolpx.ValidateSelectedIntent(source).ToAggregate(); err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } + return nil, nil, nil + } + + return r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source) +} + +// reconcileSelectedLPXSafetyPreflight resolves and validates one selected LPX +// workload after its caller establishes the required download order. +func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, +) (*lpxMaterializing, lpxClassification, error) { + // Resolve one immutable workload projection for selection and rendering. + selectedLPX, rejected, prepareErr := r.prepareLPXMaterializing(ctx, deployment, source) + if prepareErr != nil { + // Only inconsistent metadata invalidates publication; a transient read failure leaves it intact. + if errors.Is(prepareErr, dynamolpx.ErrBuildSnapshotInconsistent) { + retiring, fenceErr := r.retireInvalidLPXWorkload(ctx, deployment, fmt.Sprintf("The model build became inconsistent while acquiring the immutable snapshot: %v", prepareErr)) + if fenceErr != nil { + return nil, nil, fmt.Errorf("failed to retire the LPX publication after build snapshot inconsistency: %w", fenceErr) + } + if retiring != nil { + return nil, retiring, nil + } + } + return nil, nil, fmt.Errorf("failed to resolve the LPX workload: %w", prepareErr) + } + if rejected != nil { + requests, listErr := r.listOwnedLPXRequests(ctx, deployment) + if listErr != nil { + return nil, nil, fmt.Errorf("failed to list LPX requests after workload resolution: %w", listErr) + } + if len(requests) > 0 { + retiring, retireErr := r.retireLPXRequest(ctx, deployment, &requests[0], rejected.reason) + if retireErr != nil { + return nil, nil, fmt.Errorf("failed to retire LPX attempt after workload resolution: %w", retireErr) + } + return nil, retiring, nil + } + return nil, rejected, nil + } + + // Fence stale attempts before any Grove resource can be rendered or written. + currentLPXRequests, retiring, fenceErr := r.reconcileLPXAttemptFence(ctx, deployment, selectedLPX) + if fenceErr != nil { + return nil, nil, fmt.Errorf("failed to fence stale LPX attempt: %w", fenceErr) + } + if retiring != nil { + return nil, retiring, nil + } + + // Validate the complete existing request set now; first publication follows Grove sync. + classification, reconcileErr := r.reconcileSelectedLPXFromCurrentRequests( + ctx, + deployment, + selectedLPX, + currentLPXRequests, + false, + ) + if reconcileErr != nil { + return nil, nil, fmt.Errorf("failed to validate selected LPX lifecycle: %w", reconcileErr) + } + switch classification.(type) { + case *lpxOpen, *lpxBound, *lpxSchedulerObserved, *lpxClosed: + return selectedLPX, nil, nil + default: + return nil, classification, nil + } +} diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go new file mode 100644 index 000000000000..dafd60856042 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/render.go @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "strconv" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// renderPodCliqueSet renders the exact workload selected by controller preflight. +// The source, child, workload and plan must be non-nil and validated by that +// preflight; the plan must use the child's PodCliqueSet identity. The renderer +// does not mutate its inputs. secretsRetriever may be nil when no secrets are used. +func renderPodCliqueSet( + ctx context.Context, + source *v1beta1.DynamoGraphDeployment, + operatorConfig *configv1alpha1.OperatorConfiguration, + runtimeConfig *controller_common.RuntimeConfig, + kubeClient client.Reader, + secretsRetriever dynamo.SecretsRetriever, + workload *dynamolpx.SelectedWorkload, + plan *dynamolpx.MaterializationPlan, + deployment *v1alpha1.LPXGraphDeployment, +) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { + // Apply shared Dynamo defaults before materializing the independently owned LPX roles. + pcs, input, err := dynamo.RenderLPXBasePodCliqueSet(ctx, source, operatorConfig, runtimeConfig, + kubeClient, secretsRetriever, workload, plan.PodCliqueSetName) + if err != nil { + return nil, nil, err + } + resources, err := dynamolpx.RenderSelectedNodeLocal(pcs, workload, plan, *input) + if err != nil { + return nil, nil, err + } + + // Stamp the selected child revision and identify the engine's serving endpoint. + stampLPXIdentity(deployment, pcs, resources) + serving := plan.ConductorTemplate + if serving == "" { + serving = plan.CyborgTemplate + } + + // The child retains the last delivered LPX restart while other components restart. + restartToken := deployment.Annotations[dynamo.LPXRestartAnnotation] + for _, clique := range pcs.Spec.Template.Cliques { + if restartToken != "" { + clique.Annotations[commonconsts.RestartAnnotation] = restartToken + } + delete(clique.Labels, dynamo.LPXServingLabel) + if clique.Name == serving { + clique.Labels[dynamo.LPXServingLabel] = commonconsts.KubeLabelValueTrue + } else { + // Auxiliary roles must be private in both Service and Pod discovery modes. + delete(clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) + delete(clique.Labels, commonconsts.KubeLabelDynamoDiscoveryBackend) + delete(clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) + } + } + + // Enforce the size budget after ownership, scheduling and discovery metadata are final. + serialized, err := json.Marshal(pcs) + if err != nil { + return nil, nil, fmt.Errorf("serializing selected LPX PodCliqueSet: %w", err) + } + if len(serialized) > dynamolpx.MaxRenderedPodCliqueSetBytes { + return nil, nil, fmt.Errorf("rendered LPX PodCliqueSet is %d bytes; maximum is %d", + len(serialized), dynamolpx.MaxRenderedPodCliqueSetBytes) + } + return pcs, resources, nil +} + +// stampLPXIdentity separates attempt ownership from the scheduler's source-DGD +// identity. The source generation is frozen when effective LPX input changes. +func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { + // Preflight validates the exact source controller owner before rendering resources. + sourceOwner := metav1.GetControllerOf(deployment) + identity := map[string]string{ + lpxDeploymentUIDAnnotation: string(deployment.UID), + lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), + dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, + dynamolpx.DGDUIDAnnotation: string(sourceOwner.UID), + dynamolpx.DGDGenerationAnnotation: deployment.Annotations[dynamolpx.DGDGenerationAnnotation], + } + stamp := func(annotations *map[string]string) { + if *annotations == nil { + *annotations = make(map[string]string) + } + maps.Copy(*annotations, identity) + } + stamp(&pcs.Annotations) + for _, clique := range pcs.Spec.Template.Cliques { + stamp(&clique.Annotations) + } + for i := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { + stamp(&pcs.Spec.Template.PodCliqueScalingGroupConfigs[i].Annotations) + } + for _, resource := range resources { + annotations := resource.GetAnnotations() + stamp(&annotations) + resource.SetAnnotations(annotations) + } +} diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go new file mode 100644 index 000000000000..28f0cca02ec5 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -0,0 +1,1595 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "testing" + "time" + + "capnproto.org/go/capnp/v3" + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + "gotest.tools/v3/golden" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +func newLPXRenderDeployment(t *testing.T, source *v1beta1.DynamoGraphDeployment) *v1alpha1.LPXGraphDeployment { + t.Helper() + revision, err := dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + return &v1alpha1.LPXGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: "lpx-render-uid", Generation: 1, + Annotations: map[string]string{dynamo.LPXPCSNameAnnotation: dynamo.PCSNameForLPX(source), lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, + }, + Spec: v1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, + } +} + +func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { + t.Log("Resolve a real LPU workload including inherited scheduler metadata") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + source.Annotations = map[string]string{"kai.scheduler/padding": ""} + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + require.NoError(t, err) + config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} + kube := newTestLPXClient(t) + runtimeConfig := &controller_common.RuntimeConfig{} + pcs, _, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + selected, plan, newLPXRenderDeployment(t, source)) + require.NoError(t, err) + serialized, err := json.Marshal(pcs) + require.NoError(t, err) + + t.Log("Accept exactly one MiB including final identity, discovery and scheduler metadata") + source.Annotations["kai.scheduler/padding"] = strings.Repeat("x", lpx.MaxRenderedPodCliqueSetBytes-len(serialized)) + pcs, _, err = renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + selected, plan, newLPXRenderDeployment(t, source)) + require.NoError(t, err) + serialized, err = json.Marshal(pcs) + require.NoError(t, err) + require.Len(t, serialized, lpx.MaxRenderedPodCliqueSetBytes) + for _, clique := range pcs.Spec.Template.Cliques { + require.NotContains(t, clique.Annotations, "kai.scheduler/padding") + } + + t.Log("Reject one additional final-metadata byte as a selected-render failure before publication") + source.Annotations["kai.scheduler/padding"] += "x" + pcs, resources, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + selected, plan, newLPXRenderDeployment(t, source)) + require.ErrorContains(t, err, "rendered LPX PodCliqueSet is 1048577 bytes; maximum is 1048576") + require.Nil(t, pcs) + require.Nil(t, resources) +} + +func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { + t.Log("Create immutable native builds for this story's hybrid cases") + registry := newTestDataModelRegistry(t, t.TempDir()) + + for _, fixture := range []string{"node-local-v2-hybrid", "node-local-v3-hx-hybrid"} { + t.Run(fixture, func(t *testing.T) { + t.Log("Resolve a hybrid engine using a real ClusterTopologyBinding for KV transfer") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + fixture + ".input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + source.Spec.Experimental = &v1beta1.DynamoGraphDeploymentExperimentalSpec{ + KvTransferPolicy: &v1beta1.KvTransferPolicy{ClusterTopologyName: "fabric", Domain: "rack"}, + } + binding := &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "fabric"}, + Spec: grovev1alpha1.ClusterTopologyBindingSpec{Levels: []grovev1alpha1.TopologyLevel{ + {Domain: "zone", Key: "topology.kubernetes.io/zone"}, {Domain: "rack", Key: "nvidia.com/rack"}, + }}, + } + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, grovev1alpha1.AddToScheme(scheme)) + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(binding).Build() + child := newLPXRenderDeployment(t, source) + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + require.NoError(t, err) + config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} + pcs, _, err := renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) + require.NoError(t, err) + + t.Log("The Cyborg role receives every topology domain, not an empty mounted directory") + items := map[string]string{} + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name != selected.CyborgTemplateName() { + continue + } + require.Equal(t, binding.Name, clique.Annotations[commonconsts.KubeAnnotationTopologyClusterTopologyName]) + for _, volume := range clique.Spec.PodSpec.Volumes { + if volume.Name == "topology-labels" { + require.NotNil(t, volume.DownwardAPI) + for _, item := range volume.DownwardAPI.Items { + items[item.Path] = item.FieldRef.FieldPath + } + } + } + } + require.Equal(t, map[string]string{ + "zone": "metadata.labels['" + commonconsts.DynamoTopologyLabelKey("zone") + "']", + "rack": "metadata.labels['" + commonconsts.DynamoTopologyLabelKey("rack") + "']", + }, items) + + t.Log("Binding changes and deletion must invalidate the render instead of silently losing routing metadata") + binding.Spec.Levels = binding.Spec.Levels[:1] + require.NoError(t, kube.Update(t.Context(), binding)) + _, _, err = renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) + require.ErrorContains(t, err, `domain "rack" does not exist`) + require.NoError(t, kube.Delete(t.Context(), binding)) + _, _, err = renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) + require.ErrorContains(t, err, "was not found") + }) + } +} + +func TestLPXRenderingPreservesInputs(t *testing.T) { + const restartToken = "2026-09-08T00:00:00Z" + t.Log("Create immutable native builds while keeping each rendering's mutable inputs separate") + registry := newTestDataModelRegistry(t, t.TempDir()) + + for _, test := range []struct { + name string + wantFamily lpxv1alpha1.TargetFamily + wantMode lpxv1alpha1.WorkloadMode + }{ + {"node-local-v2-lpu-only", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, + {"node-local-v2-hybrid", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2StrictHybrid}, + {"node-local-v2-specdecode", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, + {"node-local-v3-hx-lpu-only", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, + {"node-local-v3-hx-hybrid", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, + {"node-local-v3-hx-specdecode", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Resolve the authored engine and its exact child without changing either input") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + test.name + ".input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + source.UID, source.Generation = "source-dgd-uid", 7 + source.Labels = map[string]string{ + "priorityClassName": "inference", "kai.scheduler/preemptibility": "NonPreemptible", + "project": "test", "unrelated": "ignored", + } + metav1.SetMetaDataAnnotation(&source.ObjectMeta, "kai.scheduler/topology", "source-topology") + metav1.SetMetaDataAnnotation(&source.ObjectMeta, "unrelated", "ignored") + metav1.SetMetaDataAnnotation(&source.ObjectMeta, commonconsts.KubeAnnotationEnableMetrics, "false") + source.Spec.Labels = map[string]string{"shared-label": "source"} + source.Spec.Annotations = map[string]string{ + "shared-annotation": "source", "kai.scheduler/topology": "explicit-pcs-topology", + "selected-metadata": "from-dgd", + lpx.DGDUIDAnnotation: "stale", + lpx.DGDGenerationAnnotation: "stale", + } + source.Spec.Env = []corev1.EnvVar{{Name: "LPX_DGD_ENV", Value: test.name}} + for _, component := range lpx.Components(source) { + component.GlobalDynamoNamespace = true + component.RuntimeVersionOverride = "1.5.0" + for _, role := range component.Roles { + if role.PodTemplate != nil { + metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, "selected-metadata", "from-component") + metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, commonconsts.RestartAnnotation, "2026-09-07T00:00:00Z") + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, "shared-label", "from-component") + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, commonconsts.KubeLabelDynamoNamespace, "authored-namespace") + } + } + } + component := lpx.ServingComponent(source) + component.ModelRef = &v1beta1.ModelReference{Name: "test/model"} + component.MinAvailable = ptr.To(int32(1)) + hybrid := strings.HasSuffix(test.name, "-hybrid") + singleXT := test.name == "node-local-v2-lpu-only" + if hybrid { + component.Replicas = ptr.To(int32(3)) + component.MinAvailable = ptr.To(int32(3)) + component.ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + } + if singleXT { + t.Log("Keep XT's authored readonly mount at the canonical config path") + main := &component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0] + main.VolumeMounts = append(main.VolumeMounts, + corev1.VolumeMount{Name: "config", MountPath: "/configs", ReadOnly: true}) + } + + t.Log("Preserve role metadata and replica units through alpha/beta conversion before freezing the child") + alpha := &v1alpha1.DynamoGraphDeployment{} + require.NoError(t, alpha.ConvertFrom(source)) + source = &v1beta1.DynamoGraphDeployment{} + require.NoError(t, alpha.ConvertTo(source)) + component = lpx.ServingComponent(source) + child := newLPXRenderDeployment(t, source) + child.Annotations[dynamo.LPXRestartAnnotation] = restartToken + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, restartToken) + require.NoError(t, err) + source.Generation++ + before, childBefore := source.DeepCopy(), child.DeepCopy() + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + require.NoError(t, err) + config := &configv1alpha1.OperatorConfiguration{ + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, + } + + t.Log("Project every model into the expected scheduler family, mode and physical Agent shape") + require.Equal(t, lpx.BuildFamily(test.wantFamily), selected.BuildFamily()) + var cyborgRef *lpxv1alpha1.CyborgPodCliqueReference + if hybrid { + cyborgRef = &lpxv1alpha1.CyborgPodCliqueReference{Name: "cyborg", UID: "uid"} + } + projections := selected.ModelProjections() + require.Len(t, plan.Agents, len(projections)) + projectionIndices := make(map[string]int, len(projections)) + for index, projection := range projections { + projectionIndices[plan.Agents[index].TemplateName] = index + request := projection.RequestSpec(source.Namespace, "agents", cyborgRef) + require.Equal(t, lpxv1alpha1.ExecutionBackendNodeLocal, request.ExecutionBackend) + require.NotNil(t, request.NodeLocal) + require.Equal(t, test.wantFamily, request.TargetFamily) + require.Equal(t, test.wantMode, request.WorkloadMode) + require.NotEmpty(t, request.Partitions) + require.Len(t, request.NodeLocal.PartitionMappings, len(request.Partitions)) + require.Equal(t, hybrid, request.CyborgPodCliqueRef != nil) + if test.wantFamily == lpxv1alpha1.TargetFamilyHx16x8x2x3 { + require.Len(t, request.Partitions, 1) + require.Equal(t, int64(1), request.Partitions[0].CompilerPartitionID) + require.Equal(t, []int64{16, 1, 1, 1}, *request.Partitions[0].Extent) + require.Empty(t, request.PropSyncConnectors) + require.Equal(t, 1, plan.Agents[index].Replicas) + } + } + + t.Log("Render twice and require identical resources with no input or shared-metadata mutation") + first, firstResources, err := renderPodCliqueSet(t.Context(), source, config, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, child) + require.NoError(t, err) + second, secondResources, err := renderPodCliqueSet(t.Context(), source, config, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, child) + require.NoError(t, err) + require.Equal(t, first, second) + require.Equal(t, firstResources, secondResources) + require.Equal(t, "inference", first.Labels["priorityClassName"]) + require.Equal(t, "NonPreemptible", first.Labels["kai.scheduler/preemptibility"]) + require.Equal(t, "test", first.Labels["project"]) + require.Equal(t, "explicit-pcs-topology", first.Annotations["kai.scheduler/topology"]) + require.NotContains(t, first.Annotations, commonconsts.RestartAnnotation) + require.NotContains(t, first.Labels, "unrelated") + require.NotContains(t, first.Annotations, "unrelated") + require.Equal(t, selected.Digest().String(), first.Annotations[lpx.WorkloadDigestAnnotation]) + + t.Log("Stamp frozen source identity on every final resource despite a newer ordinary DGD generation") + require.NotEqual(t, source.UID, child.UID) + annotationMaps := []map[string]string{first.Annotations} + for _, clique := range first.Spec.Template.Cliques { + require.Equal(t, restartToken, clique.Annotations[commonconsts.RestartAnnotation]) + annotationMaps = append(annotationMaps, clique.Annotations) + } + for _, group := range first.Spec.Template.PodCliqueScalingGroupConfigs { + require.NotContains(t, group.Annotations, commonconsts.RestartAnnotation) + annotationMaps = append(annotationMaps, group.Annotations) + } + for _, resource := range firstResources { + require.NotContains(t, resource.GetAnnotations(), commonconsts.RestartAnnotation) + annotationMaps = append(annotationMaps, resource.GetAnnotations()) + } + for _, annotations := range annotationMaps { + require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) + require.Equal(t, "7", annotations[lpx.DGDGenerationAnnotation]) + require.Equal(t, string(child.UID), annotations[dynamo.LPXDeploymentUIDAnnotation]) + require.Equal(t, "1", annotations[dynamo.LPXDeploymentGenerationAnnotation]) + } + + t.Log("Component and role order preserve the exact child, names and resources") + equivalent := source.DeepCopy() + slices.Reverse(equivalent.Spec.Components) + for index := range equivalent.Spec.Components { + slices.Reverse(equivalent.Spec.Components[index].Roles) + } + equivalentBefore := equivalent.DeepCopy() + revision, err := dynamo.LPXInputRevision(equivalent, restartToken) + require.NoError(t, err) + require.Equal(t, child.Spec.InputRevision, revision) + equivalentSelected, err := lpx.ResolveSelectedWorkload(t.Context(), equivalent, registry) + require.NoError(t, err) + equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(equivalent)) + require.NoError(t, err) + require.Equal(t, selected.Digest(), equivalentSelected.Digest()) + require.Equal(t, plan, equivalentPlan) + equivalentPCS, equivalentResources, err := renderPodCliqueSet(t.Context(), equivalent, config, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, equivalentSelected, equivalentPlan, child) + require.NoError(t, err) + require.Equal(t, first, equivalentPCS) + require.Equal(t, firstResources, equivalentResources) + require.Equal(t, equivalentBefore, equivalent) + + t.Log("Connect every model projection to its rendered Agent while preserving shared runtime metadata and replica units") + require.Len(t, first.Spec.Template.PodCliqueScalingGroupConfigs, 1) + group := first.Spec.Template.PodCliqueScalingGroupConfigs[0] + require.Equal(t, component.Replicas, group.Replicas) + require.Equal(t, component.MinAvailable, group.MinAvailable) + var conductor, gpuClique *grovev1alpha1.PodCliqueTemplateSpec + serving := 0 + for index, clique := range first.Spec.Template.Cliques { + if clique.Labels[dynamo.LPXServingLabel] == commonconsts.KubeLabelValueTrue { + serving++ + require.NotEqual(t, lpxv1alpha1.PodRoleAgent, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) + require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) + } else { + require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) + require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) + } + require.Equal(t, ptr.To(clique.Spec.Replicas), clique.Spec.MinAvailable) + require.NotEmpty(t, clique.Spec.PodSpec.Containers) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_DGD_ENV", Value: test.name}) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: commonconsts.DynamoNamespaceEnvVar, Value: commonconsts.GlobalDynamoNamespace}) + require.Equal(t, commonconsts.GlobalDynamoNamespace, clique.Labels[commonconsts.KubeLabelDynamoNamespace]) + require.Equal(t, "from-component", clique.Labels["shared-label"]) + require.Equal(t, "false", clique.Annotations[commonconsts.KubeAnnotationEnableMetrics]) + require.Equal(t, "kubernetes", clique.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend]) + switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { + case lpxv1alpha1.PodRoleAgent: + index, found := projectionIndices[clique.Name] + require.True(t, found, "unexpected or duplicate Agent %q", clique.Name) + require.Equal(t, projections[index].Model(), clique.Annotations[lpxv1alpha1.PodModelAnnotation]) + require.Equal(t, projections[index].Digest().String(), clique.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, int32(plan.Agents[index].Replicas), clique.Spec.Replicas) + if singleXT { + require.Contains(t, clique.Spec.PodSpec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: "config", MountPath: "/configs", ReadOnly: true}) + } + delete(projectionIndices, clique.Name) + case lpxv1alpha1.PodRoleConductor: + require.Nil(t, conductor) + conductor = clique + case lpxv1alpha1.PodRoleCyborgWorker: + require.Nil(t, gpuClique) + gpuClique = clique + } + + t.Log("Mutate this rendered role without changing the second rendering or authored input") + clique.Spec.PodSpec.Containers[0].Image = "rendered:changed" + clique.Labels["shared-label"] = "rendered-role" + clique.Annotations["shared-annotation"] = "rendered-role" + require.NotEqual(t, clique.Spec.PodSpec.Containers[0].Image, second.Spec.Template.Cliques[index].Spec.PodSpec.Containers[0].Image) + require.Equal(t, "from-component", second.Spec.Template.Cliques[index].Labels["shared-label"]) + require.Equal(t, "source", second.Spec.Template.Cliques[index].Annotations["shared-annotation"]) + } + require.Empty(t, projectionIndices) + require.Equal(t, 1, serving) + if hybrid { + require.Nil(t, conductor) + require.NotNil(t, gpuClique) + require.Equal(t, int32(2), gpuClique.Spec.Replicas) + require.Equal(t, selected.Digest().String(), gpuClique.Annotations[lpx.WorkloadDigestAnnotation]) + require.Contains(t, gpuClique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "true"}) + } else { + require.NotNil(t, conductor) + require.Equal(t, "from-component", conductor.Annotations["selected-metadata"]) + require.Equal(t, selected.Digest().String(), conductor.Annotations[lpx.WorkloadDigestAnnotation]) + require.Nil(t, gpuClique) + } + first.Labels["shared-label"] = "rendered" + first.Annotations["shared-annotation"] = "rendered" + require.Equal(t, "source", second.Labels["shared-label"]) + require.Equal(t, "source", second.Annotations["shared-annotation"]) + require.Equal(t, before, source) + require.Equal(t, childBefore, child) + }) + } +} + +func TestLPXSpecDecodeConductorTemplate(t *testing.T) { + t.Log("Create immutable native builds for the explicit and inherited conductor cases") + registry := newTestDataModelRegistry(t, t.TempDir()) + + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) { + t.Log("Give the speculative components distinct templates and select the target conductor") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + for _, component := range lpx.Components(source) { + agent := component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate + agent.Spec.Containers[0].Image = component.ComponentName + "-runtime" + agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} + } + target := lpx.ServingComponent(source) + wantImage := target.ComponentName + "-runtime" + if explicit { + template := target.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.DeepCopy() + wantImage = "independent-conductor-runtime" + template.Spec.Containers[0].Image = wantImage + template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} + template.Labels = map[string]string{"owner": "explicit-conductor"} + template.Spec.NodeSelector = map[string]string{"runtime-role": "conductor"} + template.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"conductor"}}}, + }}}, + }} + target.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate = template + } + before := source.DeepCopy() + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + require.NoError(t, err) + + t.Log("Render one shared conductor without changing either component's Agent template") + pcs, _, err := renderPodCliqueSet(t.Context(), source, + &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}}, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, source)) + require.NoError(t, err) + + t.Log("Preserve independent role fields while retaining the Nova launch") + require.Len(t, pcs.Spec.Template.Cliques, len(plan.Agents)+1) + conductors := 0 + for _, clique := range pcs.Spec.Template.Cliques { + container := clique.Spec.PodSpec.Containers[0] + if clique.Name == plan.ConductorTemplate { + conductors++ + require.Equal(t, wantImage, container.Image) + require.Equal(t, []string{"/bin/bash"}, container.Command) + require.GreaterOrEqual(t, len(container.Args), 3) + require.Equal(t, "-c", container.Args[0]) + require.Contains(t, container.Args[1], `; exec /bin/nova "$@"`) + require.Equal(t, "--", container.Args[2]) + if explicit { + template := target.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate + require.Equal(t, "explicit-conductor", clique.Labels["owner"]) + require.Equal(t, template.Spec.NodeSelector, clique.Spec.PodSpec.NodeSelector) + require.Equal(t, template.Spec.Affinity, clique.Spec.PodSpec.Affinity) + require.Contains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) + require.NotContains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) + } + } else { + component := clique.Labels[commonconsts.KubeLabelDynamoComponent] + require.Equal(t, component+"-runtime", container.Image) + require.Contains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) + require.NotContains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) + } + } + require.Equal(t, 1, conductors) + require.Equal(t, before, source) + }) + } +} + +func newTestDataModelRegistry(t *testing.T, registryRoot string) *lpx.ModelRegistry { + t.Helper() + + v2Builds := map[string]testV2GraphManifestFixture{ + "node-local-v2-connected": { + topology: "URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA", + partitionCount: 2, + numChips: 16, + devicesPerNode: 8, + selectedPropSyncChain: []uint32{0, 1}, + }, + "node-local-v2-connected-lpx": { + topology: "URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA", + partitionCount: 2, + numChips: 16, + devicesPerNode: 8, + selectedPropSyncChain: []uint32{0, 1}, + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + }, + "node-local-v2-cpu-embeddings": { + topology: "URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA", + partitionCount: 3, + numChips: 16, + devicesPerNode: 8, + selectedPropSyncChain: []uint32{0, 1, 2}, + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + standaloneTokenEmbeddings: true, + supportsCPUEmbeddings: true, + runtimeTokenEmbeddingsPath: "runtime/text_embeddings.npz", + }, + "llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c": { + topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", + partitionCount: 17, + numChips: 8, + devicesPerNode: 8, + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + }, + "llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c": { + topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", + partitionCount: 17, + numChips: 8, + devicesPerNode: 8, + }, + } + for buildID, fixture := range v2Builds { + writeTestGraphBuild(t, registryRoot, buildID, testV2GraphManifestCapnp(t, buildID, fixture)) + } + + v3Builds := map[string]testV3GraphManifestFixture{ + "node-local-v3-hx-connected": {}, + "node-local-v3-hx-connected-lpx": { + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + }, + "node-local-v3-hx-sd-draft": {}, + "node-local-v3-hx-sd-target": {}, + } + for buildID, fixture := range v3Builds { + writeTestGraphBuild(t, registryRoot, buildID, testV3GraphManifestCapnp(t, buildID, fixture)) + } + + registry, err := lpx.NewModelRegistry(registryRoot, nil) + require.NoError(t, err) + return registry +} + +type testV2GraphManifestFixture struct { + topology string + partitionCount int + numChips uint32 + devicesPerNode uint32 + selectedPropSyncChain []uint32 + compilationMode manifestcapnpv2.CompilationMode + nonLPUDeviceTypes []manifestcapnpv2.DeviceType + standaloneTokenEmbeddings bool + supportsCPUEmbeddings bool + runtimeTokenEmbeddingsPath string +} + +func writeTestGraphBuild(t *testing.T, registryRoot, buildID string, manifest []byte) { + t.Helper() + + buildDir := filepath.Join(registryRoot, buildID) + require.NoError(t, os.MkdirAll(buildDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), manifest, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.json"), []byte(`{}`), 0o600)) +} + +func newTestGraphManifest(t *testing.T) (*capnp.Message, manifestcapnpv2.Manifest) { + t.Helper() + + t.Log("Create the shared manifest header and tokenizer model") + message, segment := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnpv2.NewRootManifest(segment) + require.NoError(t, err) + manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) + model, err := manifest.NewModel() + require.NoError(t, err) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stopTokens, err := tokenizer.NewStopTokens(1) + require.NoError(t, err) + stopTokens.Set(0, 1) + return message, manifest +} + +func populateTestGraphBuild(t *testing.T, manifest manifestcapnpv2.Manifest, wireBuildID string) { + t.Helper() + + t.Log("Populate the compiler-owned build identity used only as opaque provenance") + build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) + require.NoError(t, err) + require.NoError(t, build.SetText(11, wireBuildID)) + require.NoError(t, manifest.SetReserved3(build.ToPtr())) +} + +func newTestGraphProgram( + t *testing.T, + manifest manifestcapnpv2.Manifest, + mode manifestcapnpv2.CompilationMode, + numLPUNodes uint32, + sequenceLength uint32, +) (manifestcapnpv2.DeploymentInfo, manifestcapnpv2.ProgramConfig) { + t.Helper() + + t.Log("Allocate the deployment, program, and runtime I/O contract shared by graph fixtures") + deployment, err := manifest.NewDeployment() + require.NoError(t, err) + deployment.SetCompilationMode(mode) + deployment.SetNumLpuNodes(numLPUNodes) + program, err := deployment.NewProgram() + require.NoError(t, err) + program.SetBatchSize(1) + program.SetSequenceLength(sequenceLength) + program.SetInputSize(1) + program.SetOutputSize(1) + runtimeIO, err := deployment.NewRuntimeIo() + require.NoError(t, err) + runtimeIO.SetProtocol(0) + runtimeIO.SetReserved1(1) + runtimeIO.SetIoFpgaCount(1) + runtimeIO.SetFanoutFactor(1) + return deployment, program +} + +func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphManifestFixture) []byte { + t.Helper() + + message, manifest := newTestGraphManifest(t) + populateTestGraphBuild(t, manifest, filepath.Base(buildID)) + + deployment, program := newTestGraphProgram(t, manifest, fixture.compilationMode, uint32(fixture.partitionCount)*fixture.numChips/fixture.devicesPerNode, 8192) + if len(fixture.selectedPropSyncChain) != 0 { + chains, err := deployment.NewSelectedPropSyncChains(1) + require.NoError(t, err) + partitionIDs, err := chains.At(0).NewPartitionIds(int32(len(fixture.selectedPropSyncChain))) + require.NoError(t, err) + for index, partitionID := range fixture.selectedPropSyncChain { + partitionIDs.Set(index, partitionID) + } + } + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + program.SetSupportsCpuEmbeddings(fixture.supportsCPUEmbeddings) + program.SetStandaloneTokenEmbeddings(fixture.standaloneTokenEmbeddings) + + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + if fixture.runtimeTokenEmbeddingsPath != "" { + runtimeAssets, err := artifacts.NewRuntimeAssets() + require.NoError(t, err) + require.NoError(t, runtimeAssets.SetTokenEmbeddingsPath(fixture.runtimeTokenEmbeddingsPath)) + } + partitions, err := artifacts.NewPartitions(int32(fixture.partitionCount + len(fixture.nonLPUDeviceTypes))) + require.NoError(t, err) + for index := range fixture.partitionCount { + partition := partitions.At(index) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(uint32(index)) + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", index))) + require.NoError(t, detail.SetTopology(fixture.topology)) + detail.SetNumChips(fixture.numChips) + detail.SetDevicesPerNode(fixture.devicesPerNode) + } + for index, deviceType := range fixture.nonLPUDeviceTypes { + partition := partitions.At(fixture.partitionCount + index) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(deviceType) + ref.SetPartitionId(uint32(fixture.partitionCount + index)) + switch deviceType { + case manifestcapnpv2.DeviceType_cuda: + _, err = partition.Detail().NewCuda() + case manifestcapnpv2.DeviceType_cpu: + _, err = partition.Detail().NewCpu() + default: + t.Fatalf("unsupported non-LPU test device type %q", deviceType) + } + require.NoError(t, err) + } + + data, err := message.Marshal() + require.NoError(t, err) + return data +} + +func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { + t.Helper() + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(dgd)) + require.NoError(t, err) + return plan +} + +type testV3GraphManifestFixture struct { + compilationMode manifestcapnpv2.CompilationMode + nonLPUDeviceTypes []manifestcapnpv2.DeviceType +} + +func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphManifestFixture) []byte { + t.Helper() + + message, manifest := newTestGraphManifest(t) + populateTestGraphBuild(t, manifest, buildID) + + _, program := newTestGraphProgram(t, manifest, fixture.compilationMode, 2, 8192) + program.SetNumKvCaches(1) + + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(int32(1 + len(fixture.nonLPUDeviceTypes))) + require.NoError(t, err) + partition := partitions.At(0) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(1) + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath("part-1")) + require.NoError(t, detail.SetTopology("opaque-v3-topology")) + detail.SetNumChips(16) + detail.SetDevicesPerNode(16) + for offset, deviceType := range fixture.nonLPUDeviceTypes { + partition := partitions.At(1 + offset) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(deviceType) + ref.SetPartitionId(uint32(2 + offset)) + switch deviceType { + case manifestcapnpv2.DeviceType_cuda: + _, err = partition.Detail().NewCuda() + case manifestcapnpv2.DeviceType_cpu: + _, err = partition.Detail().NewCpu() + default: + t.Fatalf("unsupported non-LPU test device type %q", deviceType) + } + require.NoError(t, err) + } + + data, err := message.Marshal() + require.NoError(t, err) + return data +} + +func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { + controllerConfig := &configv1alpha1.OperatorConfiguration{ + Orchestrators: configv1alpha1.OrchestratorConfiguration{ + Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, + }, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + } + const buildID = "single-v2-manifest-defaults/build_manifest_defaults" + registryRoot := t.TempDir() + writeTestGraphBuild(t, registryRoot, buildID, testGbuildManifestCapnp(t, buildID)) + registry, err := lpx.NewModelRegistry(registryRoot, nil) + require.NoError(t, err) + + type renderedConfig struct { + modelConfig string + hash string + } + t.Log("Render manifest-derived, equivalent explicit, and overridden runtime settings") + configs := make([]renderedConfig, 0, 3) + for _, settingsJSON := range []string{"", `{"batch_size":1,"swa":{"chunked":false}}`, `{"sequence_length":65536}`} { + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml") + require.NoError(t, err) + + var deployment v1beta1.DynamoGraphDeployment + require.NoError(t, yaml.Unmarshal(payload, &deployment)) + require.Len(t, deployment.Spec.Components, 1) + require.NotNil(t, deployment.Spec.Components[0].LPX) + require.Len(t, deployment.Spec.Components[0].Roles, 2) + if settingsJSON != "" { + deployment.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(settingsJSON)} + } + + selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) + require.NoError(t, err) + podCliqueSet, resources, err := renderPodCliqueSet( + t.Context(), + &deployment, + controllerConfig, + &controller_common.RuntimeConfig{}, + nil, nil, + selected, + mustPlanSelectedLPX(t, &deployment, selected), + newLPXRenderDeployment(t, &deployment), + ) + require.NoError(t, err) + + configMap := getResource[*corev1.ConfigMap](t, resources, "dgd-single-v2-manifest-defaults-lpu") + var hash string + for _, clique := range podCliqueSet.Spec.Template.Cliques { + if value := clique.Annotations[commonconsts.AnnotationExtraResourcesHash]; value != "" { + hash = value + break + } + } + require.NotEmpty(t, hash) + configs = append(configs, renderedConfig{modelConfig: configMap.Data["model_config.toml"], hash: hash}) + } + + t.Log("Preserve derived config and hash until an explicit value changes the output") + derived, explicitButEqual, overridden := configs[0], configs[1], configs[2] + + buildPath := filepath.Join(registryRoot, buildID) + require.Equal(t, fmt.Sprintf(`type = 'Single' + +[iop] +batch_size = 1 +cpu_embeddings = true +dkvc = true +input_size = 1 +model_path = '%s' +num_dkvc_blocks = 256 +num_kv_caches = 1 +num_layers = 24 +output_size = 1 +prop_sync = false +sequence_length = 131072 +stop_tokens = [200002, 199999, 200012] +tokenizer_path = '%s' +vocab_size = 201088 + +[iop.swa] +chunked = false +num_swa_dkvc_blocks = 1 +swa_ctx_len = 128 +swa_num_users = 8 +swa_padding_len = 0 + +[scheduler] + +[setup] +resolved_partitions_dir = '/configs' +setup_ops_format = 'agent_v2' +`, buildPath, filepath.Join(buildPath, "tokenizer")), derived.modelConfig) + require.Equal(t, derived.modelConfig, explicitButEqual.modelConfig) + require.Equal(t, derived.hash, explicitButEqual.hash) + require.NotEqual(t, derived.modelConfig, overridden.modelConfig) + require.NotEqual(t, derived.hash, overridden.hash) +} + +func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { + t.Helper() + + const topology = "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106" + + msg, seg := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnpv2.NewRootManifest(seg) + require.NoError(t, err) + manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) + populateTestGraphBuild(t, manifest, filepath.Base(registryDir)) + + model, err := manifest.NewModel() + require.NoError(t, err) + arch, err := model.NewArch() + require.NoError(t, err) + arch.SetNLayers(24) + arch.SetVocabSize(201088) + swa, err := arch.NewSwa() + require.NoError(t, err) + swa.SetMaxSeqLen(128) + swa.SetPadding(0) + swaNumUsers, err := swa.NewNumUsers() + require.NoError(t, err) + swaNumUsers.SetValue(8) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stopTokens, err := tokenizer.NewStopTokens(3) + require.NoError(t, err) + stopTokens.Set(0, 200002) + stopTokens.Set(1, 199999) + stopTokens.Set(2, 200012) + + deployment, program := newTestGraphProgram(t, manifest, manifestcapnpv2.CompilationMode_lpuOnly, 1, 131072) + _, err = deployment.NewSelectedPropSyncChains(0) + require.NoError(t, err) + dkvc, err := program.NewDkvc() + require.NoError(t, err) + dkvc.SetNumBlocksPerKvCache(256) + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + program.SetBatchFolding(false) + program.SetSupportsCpuEmbeddings(true) + program.SetSwaChunked(false) + program.SetNumSwaDkvcBlocks(1) + + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(1) + require.NoError(t, err) + partition := partitions.At(0) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(0) + lpuDetail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, lpuDetail.SetPath("assemble/part-0")) + require.NoError(t, lpuDetail.SetTopology(topology)) + lpuDetail.SetNumChips(8) + lpuDetail.SetDevicesPerNode(8) + + data, err := msg.Marshal() + require.NoError(t, err) + return data +} + +// Update this golden with: +// go test ./internal/controller/lpx -run TestGenerateGrovePodCliqueSet_FromDGDYaml -count=1 -args -update +func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { + controllerConfig := &configv1alpha1.OperatorConfiguration{ + Infrastructure: configv1alpha1.InfrastructureConfiguration{ + ETCDAddress: "etcd-address", + NATSAddress: "nats-address", + }, + Orchestrators: configv1alpha1.OrchestratorConfiguration{ + Grove: configv1alpha1.GroveConfiguration{ + TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}, + }, + }, + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + } + + registryRoot := t.TempDir() + testdataModelRegistry := newTestDataModelRegistry(t, registryRoot) + kubeClient := newTestLPXClient(t) + runtimeConfig := &controller_common.RuntimeConfig{} + + tests := []string{ + "from_dgd_yaml/lpx_v2_vllm", + "from_dgd_yaml/node-local-v2-lpu-only", + "from_dgd_yaml/node-local-v2-hybrid", + "from_dgd_yaml/node-local-v2-specdecode", + "from_dgd_yaml/node-local-v3-hx-lpu-only", + "from_dgd_yaml/node-local-v3-hx-specdecode", + "from_dgd_yaml/node-local-v3-hx-hybrid", + "from_dgd_yaml/single_v2", + } + + for _, name := range tests { + t.Run(name, func(t *testing.T) { + t.Log("Load the authored input before selection") + b, err := os.ReadFile("../../dynamo/testdata/" + name + ".input.yaml") + require.NoError(t, err) + + var dynamoDeployment v1beta1.DynamoGraphDeployment + require.NoError(t, yaml.Unmarshal(b, &dynamoDeployment)) + if dynamoDeployment.UID == "" { + dynamoDeployment.UID = "11111111-1111-4111-8111-111111111111" + } + if dynamoDeployment.Generation == 0 { + dynamoDeployment.Generation = 1 + } + selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(&dynamoDeployment)) + require.NoError(t, err) + t.Log("Render all LPX roles with the separate LPX PCS identity") + got, extraResources, err := renderPodCliqueSet( + t.Context(), &dynamoDeployment, controllerConfig, runtimeConfig, + kubeClient, nil, selected, + plan, + newLPXRenderDeployment(t, &dynamoDeployment), + ) + require.NoError(t, err) + for _, clique := range got.Spec.Template.Cliques { + component := dynamoDeployment.GetComponentByName(clique.Labels[commonconsts.KubeLabelDynamoComponent]) + require.NotNil(t, component) + require.True(t, component.IsLPX()) + require.Equal(t, component.ComponentName, clique.Labels[lpx.StageLabel]) + } + + t.Log("Render conventional components independently from their ordinary-only input") + ordinary := dynamoDeployment.DeepCopy() + ordinary.Spec.Components = slices.DeleteFunc(ordinary.Spec.Components, func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.IsLPX() + }) + normal, err := dynamo.GenerateGrovePodCliqueSet( + t.Context(), ordinary, controllerConfig, runtimeConfig, + kubeClient, nil, nil, nil, nil, + ) + require.NoError(t, err) + require.NotEqual(t, normal.Name, got.Name) + for _, clique := range normal.Spec.Template.Cliques { + require.NotEqual(t, selected.LPXComponentName(), clique.Labels[commonconsts.KubeLabelDynamoComponent]) + } + podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} + if len(normal.Spec.Template.Cliques) > 0 { + podCliqueSets = append(podCliqueSets, normal) + } + + for _, pcs := range podCliqueSets { + sort.Slice(pcs.Spec.Template.Cliques, func(i, j int) bool { + return pcs.Spec.Template.Cliques[i].Name < pcs.Spec.Template.Cliques[j].Name + }) + for _, clique := range pcs.Spec.Template.Cliques { + for i := range clique.Spec.PodSpec.Containers { + slices.SortFunc(clique.Spec.PodSpec.Containers[i].Env, func(a, b corev1.EnvVar) int { + return strings.Compare(a.Name, b.Name) + }) + } + } + extraResources = append(extraResources, pcs) + } + + var out []string + for _, resource := range extraResources { + b, err := yaml.Marshal(resource) + if err != nil { + t.Errorf("Marshal() error = %v", err) + } + out = append(out, string(b)) + } + + t.Log("Normalize temporary paths and their content hashes only at the golden boundary; incorrect hash annotations still fail") + replacements := []string{registryRoot, "/testdata"} + for _, resource := range extraResources { + config, ok := resource.(*corev1.ConfigMap) + if !ok { + continue + } + actualHash := lpx.LPUConfigMapHash(config) + normalized := config.DeepCopy() + for key, value := range normalized.Data { + normalized.Data[key] = strings.ReplaceAll(value, registryRoot, "/testdata") + } + normalizedHash := lpx.LPUConfigMapHash(normalized) + replacements = append(replacements, actualHash, normalizedHash) + } + goldenPath, err := filepath.Abs("../../dynamo/testdata/" + name + ".yaml") + require.NoError(t, err) + golden.Assert(t, strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) + }) + } +} + +func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *testing.T) { + t.Log("Load the pre-scheduler V2 hybrid DGD shape without scheduler intent or an image-owned command") + payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + require.NoError(t, err) + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + delete(dgd.Annotations, commonconsts.KubeAnnotationLPXSchedulerBackend) + dgd.Spec.Scheduling = nil + lpxComponent := dgd.GetComponentByName("lpu") + require.NotNil(t, lpxComponent) + require.NotNil(t, lpxComponent.LPX) + lpxComponent.LPX.BuildID = "node-local-v2-cpu-embeddings" + lpxComponent.ModelRef = &v1beta1.ModelReference{Name: "test/model"} + + t.Log("Customize the independent Cyborg batch while keeping its sidecar separate") + cyborgRole := lpxComponent.ComponentRole(v1beta1.ComponentRoleLeader) + require.NotNil(t, cyborgRole) + authoredCyborgMain := &cyborgRole.PodTemplate.Spec.Containers[0] + require.Equal(t, commonconsts.MainContainerName, authoredCyborgMain.Name) + authoredCyborgMain.Env = append( + authoredCyborgMain.Env, + corev1.EnvVar{Name: lpx.CyborgBatchSizeEnv, Value: "2"}, + corev1.EnvVar{Name: "USER_BATCH_REF", Value: "$(CYBORG_BATCH_SIZE)"}, + ) + + t.Log("Keep a sidecar named cyborg separate from the main runtime container") + sidecar := corev1.Container{Name: "cyborg", Image: "sidecar", Args: []string{"observe"}} + cyborgRole.PodTemplate.Spec.Containers = append(cyborgRole.PodTemplate.Spec.Containers, sidecar) + + t.Log("Customize Agent metadata and placement while leaving its launch to the runtime") + agentTemplate := lpxComponent.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate + agentMain := &agentTemplate.Spec.Containers[0] + require.NotEmpty(t, agentMain.Command) + require.NotEmpty(t, agentMain.Args) + agentMain.Command = nil + agentMain.Args = []string{"--instance-model-name", "test/model", "--agent-env-vars", "PIPELINE_ENV=kept"} + agentMain.Env = append(agentMain.Env, + corev1.EnvVar{Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, + corev1.EnvVar{Name: "MAIN_CPU", ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, Resource: "requests.cpu", + }}}, + ) + agentTemplate.Spec.Tolerations = []corev1.Toleration{ + {Key: "cluster.example/custom", Operator: corev1.TolerationOpExists}, + {Key: "lpu.nvidia.com/node", Operator: corev1.TolerationOpExists}, + } + agentMain.VolumeMounts = append(agentMain.VolumeMounts, + corev1.VolumeMount{Name: "config", MountPath: "/custom", ReadOnly: true}, + ) + customConfigVolume := corev1.Volume{ + Name: "config", + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: dgd.Name + "-lpu"}, + DefaultMode: ptr.To[int32](0o440), + }}, + } + agentTemplate.Spec.Volumes = append(agentTemplate.Spec.Volumes, customConfigVolume) + + t.Log("Project the scheduler-defaulted workload and preserve its conductorless LPU runtime") + registry := newTestDataModelRegistry(t, t.TempDir()) + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) + require.NoError(t, err) + projections := selected.ModelProjections() + require.Len(t, projections, 1) + projection := projections[0] + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(dgd)) + require.NoError(t, err) + require.Empty(t, plan.ConductorTemplate) + require.Len(t, plan.Agents, 1) + request := projection.RequestSpec(dgd.Namespace, "test", nil) + require.Len(t, request.Partitions, 2) + require.Equal(t, int64(1), request.Partitions[0].CompilerPartitionID) + require.Equal(t, int64(2), request.Partitions[1].CompilerPartitionID) + + controllerConfig := &configv1alpha1.OperatorConfiguration{ + Infrastructure: configv1alpha1.InfrastructureConfiguration{ETCDAddress: "etcd", NATSAddress: "nats"}, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + } + pcs, extraResources, err := renderPodCliqueSet( + t.Context(), dgd, controllerConfig, &controller_common.RuntimeConfig{}, + newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, dgd), + ) + require.NoError(t, err) + + t.Log("Verify LPX places the worker while the generated backend owns the complete process contract") + var agent, cyborg *grovev1alpha1.PodCliqueTemplateSpec + serving := 0 + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Labels[dynamo.LPXServingLabel] == commonconsts.KubeLabelValueTrue { + serving++ + require.Equal(t, lpxv1alpha1.PodRoleCyborgWorker, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) + require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) + } else { + require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) + require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) + } + switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { + case lpxv1alpha1.PodRoleConductor: + t.Fatalf("implicit V2 hybrid runtime unexpectedly rendered conductor %q", clique.Name) + case lpxv1alpha1.PodRoleAgent: + agent = clique + case lpxv1alpha1.PodRoleCyborgWorker: + cyborg = clique + } + } + require.NotNil(t, agent) + require.NotNil(t, cyborg) + require.Equal(t, 1, serving) + require.Equal(t, lpx.SchedulerName, agent.Spec.PodSpec.SchedulerName) + require.Equal(t, int32(plan.Agents[0].Replicas), agent.Spec.Replicas) + require.Contains(t, cyborg.Spec.StartsAfter, agent.Name) + require.NotContains(t, cyborg.Spec.StartsAfter, "lpu-ldr") + + main := agent.Spec.PodSpec.Containers[0] + require.Equal(t, "agent", main.Name) + require.Equal(t, []string{"/bin/bash"}, main.Command) + require.Len(t, main.Args, 2) + require.Equal(t, "-c", main.Args[0]) + require.Contains(t, main.Args[1], "exec numactl") + require.NotContains(t, main.Args, "--agent-env-vars") + require.NotNil(t, main.SecurityContext) + require.True(t, *main.SecurityContext.Privileged) + require.True(t, main.Stdin) + require.True(t, main.TTY) + require.NotNil(t, main.StartupProbe) + require.NotNil(t, main.ReadinessProbe) + runtimeEnv := make(map[string]string, len(main.Env)) + for _, variable := range main.Env { + runtimeEnv[variable.Name] = variable.Value + } + require.Equal(t, "agent", runtimeEnv["CONTAINER_NAME"]) + require.Equal(t, "agent", envValueSource(main.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) + require.Equal(t, "1024", runtimeEnv["GLUE_RDMA_PATH_MTU"]) + require.Equal(t, "mlx5_0", runtimeEnv["NIC_NAME"]) + require.Equal(t, "19878", runtimeEnv["READINESS_PORT"]) + require.Equal(t, "default", runtimeEnv["LPU_MODEL_NAME"]) + gasDirSource := envValueSource(main.Env, "GAS_DIR") + require.NotNil(t, gasDirSource) + require.NotNil(t, gasDirSource.ConfigMapKeyRef) + require.Equal(t, dgd.Name+"-lpu", gasDirSource.ConfigMapKeyRef.Name) + require.Equal(t, "gas_dir", gasDirSource.ConfigMapKeyRef.Key) + require.Equal(t, "topologies", envValueSource(main.Env, "TOPOLOGIES").ConfigMapKeyRef.Key) + podIPSource := envValueSource(main.Env, "POD_IP") + require.NotNil(t, podIPSource) + require.NotNil(t, podIPSource.FieldRef) + require.Equal(t, "status.podIP", podIPSource.FieldRef.FieldPath) + require.NotContains(t, runtimeEnv, "PARTITION_ID") + require.NotContains(t, runtimeEnv, "RANK_IN_PARTITION") + + t.Log("Preserve custom tolerations and mounts alongside the required Agent networking and storage") + require.True(t, agent.Spec.PodSpec.HostIPC) + require.True(t, agent.Spec.PodSpec.HostNetwork) + require.Equal(t, corev1.DNSClusterFirstWithHostNet, agent.Spec.PodSpec.DNSPolicy) + require.Equal(t, []corev1.Toleration{ + {Key: "cluster.example/custom", Operator: corev1.TolerationOpExists}, + {Key: "lpu.nvidia.com/node", Operator: corev1.TolerationOpExists}, + {Key: "lpu.nvidia.com/lpu", Operator: corev1.TolerationOpExists}, + {Key: "lpu.nvidia.com/node-v2", Operator: corev1.TolerationOpExists}, + }, agent.Spec.PodSpec.Tolerations) + for _, name := range []string{"config", "tmp", "hugepages", "host-dev", "host-sys", "ssh-secret"} { + require.True(t, slices.ContainsFunc(agent.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }), "missing volume %s", name) + } + require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "host-dev" })) + require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "host-sys" })) + require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "hugepages" })) + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom", ReadOnly: true}) + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) + require.Contains(t, agent.Spec.PodSpec.Volumes, customConfigVolume) + require.Equal(t, resource.MustParse("62"), main.Resources.Requests[corev1.ResourceCPU]) + require.Equal(t, resource.MustParse("4096Mi"), main.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"]) + + t.Log("Verify Cyborg receives its generated config and InfiniBand bindings instead of fallback PVCs") + require.Contains(t, cyborg.Spec.PodSpec.Containers, sidecar) + require.NotEmpty(t, cyborg.Spec.PodSpec.Containers) + cyborgMain := cyborg.Spec.PodSpec.Containers[0] + require.Equal(t, []string{"/bin/sh", "-ec"}, cyborgMain.Command) + require.Contains(t, cyborgMain.Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) + require.Equal(t, []string{"--", "/opt/gpu-runtime", "serve"}, cyborgMain.Args[1:]) + require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "config" })) + require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) + + volumes := make(map[string]corev1.Volume, len(cyborg.Spec.PodSpec.Volumes)) + for _, volume := range cyborg.Spec.PodSpec.Volumes { + volumes[volume.Name] = volume + } + configVolume := volumes["config"] + require.NotNil(t, configVolume.ConfigMap) + require.Equal(t, dgd.Name+"-decode", configVolume.ConfigMap.Name) + require.Nil(t, configVolume.PersistentVolumeClaim) + infinibandVolume := volumes["infiniband"] + require.NotNil(t, infinibandVolume.HostPath) + require.Equal(t, "/dev/infiniband", infinibandVolume.HostPath.Path) + require.Nil(t, infinibandVolume.PersistentVolumeClaim) + + t.Log("Verify Cyborg receives every runtime value consumed by its startup command") + cyborgEnv := make(map[string]string, len(cyborgMain.Env)) + for _, variable := range cyborgMain.Env { + cyborgEnv[variable.Name] = variable.Value + } + require.Equal(t, "19877", cyborgEnv["RDMA_PORT"]) + require.Equal(t, "/configs/lpu_servers", cyborgEnv["SERVER_HOSTS_FILE"]) + require.Equal(t, "1", cyborgEnv["TOTAL_REPLICAS"]) + require.Equal(t, "2", cyborgEnv[lpx.CyborgBatchSizeEnv]) + require.NotContains(t, cyborgEnv, "CYBORG_SWA_CACHE_IDS") + batchIndex := slices.IndexFunc(cyborgMain.Env, func(variable corev1.EnvVar) bool { + return variable.Name == "CYBORG_BATCH_SIZE" + }) + userIndex := slices.IndexFunc(cyborgMain.Env, func(variable corev1.EnvVar) bool { + return variable.Name == "USER_BATCH_REF" + }) + require.NotEqual(t, -1, batchIndex) + require.NotEqual(t, -1, userIndex) + require.Less(t, batchIndex, userIndex) + var tokenizerEnv *corev1.EnvVar + for index := range cyborgMain.Env { + if cyborgMain.Env[index].Name == "TOKENIZER_DIR" { + tokenizerEnv = &cyborgMain.Env[index] + break + } + } + require.NotNil(t, tokenizerEnv) + require.NotNil(t, tokenizerEnv.ValueFrom) + require.NotNil(t, tokenizerEnv.ValueFrom.ConfigMapKeyRef) + require.Equal(t, dgd.Name+"-decode", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) + require.Equal(t, "tokenizer_dir", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Key) + + t.Log("Verify the preserved runtime and LPX projection use the same filtered physical partition view") + lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, dgd.Name+"-lpu") + require.Equal(t, "1", lpuConfig.Data["partition_ids"]) + require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", lpuConfig.Data["gas_dir"]) + + t.Log("Verify generated Cyborg and LPU config resource order and contents") + decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, dgd.Name+"-decode") + require.Len(t, extraResources, 2) + require.Equal(t, dgd.Name+"-decode", extraResources[0].GetName()) + require.Equal(t, dgd.Name+"-lpu", extraResources[1].GetName()) + require.Equal(t, dgd.Namespace, decodeConfig.Namespace) + require.Len(t, decodeConfig.Data, 2) + require.Equal(t, []string{"lpu-0-lpu-wkr-m-0-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) + require.NotEmpty(t, decodeConfig.Data["tokenizer_dir"]) + decodeHash := lpx.LPUConfigMapHash(decodeConfig) + require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) +} + +func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testing.T) { + t.Log("Share the immutable build registry across entrypoint cases") + registry := newTestDataModelRegistry(t, t.TempDir()) + kubeClient := newTestLPXClient(t) + modes := []struct { + name string + file string + hybrid bool + single bool + }{ + {name: "Single V2", file: "single_v2.input.yaml", single: true}, + {name: "V2 LPU-only", file: "node-local-v2-lpu-only.input.yaml", single: true}, + {name: "V2 hybrid", file: "node-local-v2-hybrid.input.yaml", hybrid: true}, + } + intents := []struct { + name string + command []string + args []string + cyborgCommand []string + cyborgArgs []string + hybridOnly bool + }{ + {name: "image entrypoint and image command"}, + {name: "image entrypoint with user arguments", args: []string{"serve"}}, + {name: "explicit executable with image arguments", command: []string{"/opt/custom-runtime"}}, + {name: "explicit executable and arguments", command: []string{"/opt/custom-runtime"}, args: []string{"serve"}}, + { + name: "explicit Cyborg shell remains user-owned", + command: []string{"/opt/custom-runtime"}, + args: []string{"serve"}, + cyborgCommand: []string{"/bin/sh", "-c"}, + cyborgArgs: []string{"exec /opt/cyborg-runtime serve"}, + hybridOnly: true, + }, + { + name: "detected SGLang Cyborg remains selected no-op", + cyborgCommand: []string{"python3"}, + cyborgArgs: []string{"-m", "dynamo.sglang"}, + hybridOnly: true, + }, + { + name: "detected TensorRT-LLM Cyborg remains selected no-op", + cyborgCommand: []string{"python3"}, + cyborgArgs: []string{"-m", "dynamo.trtllm"}, + hybridOnly: true, + }, + } + + t.Log("Exercise both merge algorithms; shared policy tests cover default selection") + strategies := []struct { + name string + componentStrategy v1beta1.ExtraPodSpecMergeStrategy + defaultStrategy v1alpha1.ExtraPodSpecMergeStrategy + }{ + { + name: "component override", + componentStrategy: v1beta1.ExtraPodSpecMergeStrategyOverride, + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, + }, + { + name: "component strategic", + componentStrategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, + }, + } + + for _, mode := range modes { + t.Run(mode.name, func(t *testing.T) { + payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", mode.file)) + require.NoError(t, err) + + for _, strategy := range strategies { + t.Run(strategy.name, func(t *testing.T) { + for _, intent := range intents { + if intent.hybridOnly && !mode.hybrid { + continue + } + t.Run(intent.name, func(t *testing.T) { + t.Log("Render the LPX LPU and GPU roles from the same command intent") + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + component := dgd.GetComponentByName("lpu") + require.NotNil(t, component) + component.ExtraPodSpecMergeStrategy = strategy.componentStrategy + agent := component.ComponentRole(v1beta1.ComponentRoleWorker) + lpxMain := &agent.PodTemplate.Spec.Containers[0] + lpxMain.VolumeMounts = append(lpxMain.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) + templates := []*corev1.PodTemplateSpec{agent.PodTemplate} + if mode.hybrid { + templates = append(templates, component.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate) + } + for roleIndex, template := range templates { + main := &template.Spec.Containers[0] + require.Equal(t, commonconsts.MainContainerName, main.Name) + command := intent.command + args := intent.args + if roleIndex == 1 && intent.cyborgCommand != nil { + command = intent.cyborgCommand + args = intent.cyborgArgs + } + main.Command = slices.Clone(command) + main.Args = slices.Clone(args) + } + + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) + require.NoError(t, err) + pcs, _, err := renderPodCliqueSet( + t.Context(), + dgd, + &configv1alpha1.OperatorConfiguration{ + PodGeneration: configv1alpha1.PodGenerationConfiguration{ + DefaultExtraPodSpecMergeStrategy: strategy.defaultStrategy, + }, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + }, + &controller_common.RuntimeConfig{}, + kubeClient, + nil, + selected, + mustPlanSelectedLPX(t, dgd, selected), + newLPXRenderDeployment(t, dgd), + ) + if mode.single && len(intent.command) == 0 && len(intent.args) > 0 { + require.ErrorContains(t, err, "only supports --instance-model-name") + return + } + require.NoError(t, err) + + t.Log("Verify explicit commands are preserved and direct Agents receive their launch defaults") + var agents, conductors, cyborgs int + for _, clique := range pcs.Spec.Template.Cliques { + main := clique.Spec.PodSpec.Containers[0] + switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { + case lpxv1alpha1.PodRoleAgent: + agents++ + require.True(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) + if len(intent.command)+len(intent.args) == 0 { + require.Equal(t, []string{"/bin/bash"}, main.Command) + require.Len(t, main.Args, 2) + require.Equal(t, "-c", main.Args[0]) + require.Contains(t, main.Args[1], "GROVE_PCLQ_POD_INDEX") + } else { + wantCommand := intent.command + if mode.hybrid && len(wantCommand) == 0 { + wantCommand = []string{"/bin/bash"} + } + require.Equal(t, wantCommand, main.Command) + require.True(t, slices.Equal(intent.args, main.Args)) + } + case lpxv1alpha1.PodRoleConductor: + conductors++ + if len(intent.command)+len(intent.args) == 0 { + require.Equal(t, []string{"/bin/bash"}, main.Command) + allocation := slices.Index(main.Args, "--allocation") + require.GreaterOrEqual(t, allocation, 0) + require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) + } else { + wantCommand, argsStart := intent.command, 0 + if len(intent.command) == 0 { + wantCommand, argsStart = []string{"/bin/bash"}, 3 + } + require.Equal(t, wantCommand, main.Command) + require.GreaterOrEqual(t, len(main.Args), argsStart+len(intent.args)) + require.True(t, slices.Equal(intent.args, main.Args[argsStart:argsStart+len(intent.args)])) + allocation := slices.Index(main.Args, "--allocation") + require.GreaterOrEqual(t, allocation, 0) + require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) + } + case lpxv1alpha1.PodRoleCyborgWorker: + cyborgs++ + wantCommand := intent.command + wantArgs := intent.args + if intent.cyborgCommand != nil { + wantCommand = intent.cyborgCommand + wantArgs = intent.cyborgArgs + } + require.Equal(t, wantCommand, main.Command) + require.Equal(t, wantArgs, main.Args) + require.NotNil(t, main.LivenessProbe) + require.NotNil(t, main.ReadinessProbe) + require.NotNil(t, main.StartupProbe) + require.NotEmpty(t, main.Ports) + } + } + require.Positive(t, agents) + if mode.hybrid { + require.Zero(t, conductors) + require.Equal(t, 1, cyborgs) + } else { + require.Equal(t, 1, conductors) + require.Zero(t, cyborgs) + } + }) + } + }) + } + }) + } +} + +func newTestLPXClient(t *testing.T) client.Client { + t.Helper() + return fake.NewClientBuilder().Build() +} + +func envValueSource(envs []corev1.EnvVar, name string) *corev1.EnvVarSource { + for index := range envs { + if envs[index].Name == name { + return envs[index].ValueFrom + } + } + return nil +} + +func getResource[T any](t *testing.T, resources []client.Object, name string) T { + t.Helper() + i := slices.IndexFunc(resources, func(resource client.Object) bool { return resource.GetName() == name }) + require.GreaterOrEqual(t, i, 0) + return resources[i].(T) +} + +func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { + t.Log("Load an authored hybrid engine and override its independent leader template") + payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + leader := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate + leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: "config", MountPath: "/custom-config", ReadOnly: true}, + {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, + } + leader.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, + {Name: "TOTAL_REPLICAS", Value: "9"}, + {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, + } + leader.Spec.Volumes = []corev1.Volume{{ + Name: "config", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }} + + t.Log("Render the complete LPX workload with strategic merge enabled by operator default") + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) + require.NoError(t, err) + pcs, _, err := renderPodCliqueSet(t.Context(), source, + &configv1alpha1.OperatorConfiguration{ + PodGeneration: configv1alpha1.PodGenerationConfiguration{ + DefaultExtraPodSpecMergeStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, + }, + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + }, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, + mustPlanSelectedLPX(t, source, selected), newLPXRenderDeployment(t, source)) + require.NoError(t, err) + cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { + return clique.Name == selected.CyborgTemplateName() + }) + require.GreaterOrEqual(t, cliqueIndex, 0) + podSpec := pcs.Spec.Template.Cliques[cliqueIndex].Spec.PodSpec + + t.Log("Keep one authored config volume without the generated ConfigMap union member") + matches := 0 + var configVolume *corev1.Volume + for i := range podSpec.Volumes { + if podSpec.Volumes[i].Name == "config" { + matches++ + configVolume = &podSpec.Volumes[i] + } + } + require.Equal(t, 1, matches) + require.NotNil(t, configVolume) + require.NotNil(t, configVolume.VolumeSource.EmptyDir) + require.Nil(t, configVolume.VolumeSource.ConfigMap) + + t.Log("Preserve authored mounts and runtime environment over the generated Cyborg defaults") + main := podSpec.Containers[0] + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom-config", ReadOnly: true}) + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}) + env := make(map[string]string, len(main.Env)) + for _, variable := range main.Env { + env[variable.Name] = variable.Value + } + require.Equal(t, "/custom-tokenizer", env["TOKENIZER_DIR"]) + require.Equal(t, "9", env["TOTAL_REPLICAS"]) + require.Equal(t, "/custom-servers", env["SERVER_HOSTS_FILE"]) +} diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go new file mode 100644 index 000000000000..f27d1e42e69f --- /dev/null +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "fmt" + "maps" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" +) + +func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). + For(&nvidiacomv1alpha1.LPXGraphDeployment{}, builder.WithPredicates(lpxDeploymentPrimaryPredicate())). + Named("lpxgraphdeployment"). + Watches(&nvidiacomv1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []ctrl.Request { + return []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(obj)}} + }), builder.WithPredicates(lpxSourcePredicate())). + WithEventFilter(r.lpxControllerEventFilter()) + // Primary/source events and finalizer retries are sufficient while disabled. + if r.unavailableReason() != "" { + return ctrlBuilder.Complete(r) + } + + // Active integration observes its workload and scheduling dependencies. + ctrlBuilder = ctrlBuilder.Owns(&corev1.ConfigMap{}). + Owns(&corev1.Service{}). + Owns(&grovev1alpha1.PodCliqueSet{}). + Watches(&grovev1alpha1.PodClique{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxPodCliqueEventPredicates())). + Watches(&grovev1alpha1.PodCliqueScalingGroup{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxScalingGroupEventPredicates())) + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences); err != nil { + return fmt.Errorf("register LPX topology reference index: %w", err) + } + ctrlBuilder = r.addLPXWatches(ctrlBuilder) + if r.runtimeConfig.Gate.Enabled(features.DRA) { + // Index authored references at each hop without reading other cached objects. + for _, index := range []struct { + object client.Object + name string + values client.IndexerFunc + }{ + {&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimRefIndex, lpxDRAClaimReferences(false)}, + {&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimTemplateRefIndex, lpxDRAClaimReferences(true)}, + {&resourcev1.ResourceClaim{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences}, + {&resourcev1.ResourceClaimTemplate{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences}, + } { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), index.object, index.name, index.values); err != nil { + return fmt.Errorf("register LPX %T index %s: %w", index.object, index.name, err) + } + } + for _, dependency := range []client.Object{&resourcev1.ResourceClaim{}, &resourcev1.ResourceClaimTemplate{}, &resourcev1.DeviceClass{}} { + ctrlBuilder = ctrlBuilder.Watches(dependency, handler.EnqueueRequestsFromMapFunc(r.mapLPXDRADependencyToRequests), builder.WithPredicates(predicate.GenerationChangedPredicate{})) + } + } + return ctrlBuilder.Complete(r) +} + +// lpxDeploymentPrimaryPredicate observes every field that grants publication authority. +func lpxDeploymentPrimaryPredicate() predicate.Predicate { + ownerChanged := predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { + oldDeployment := e.ObjectOld.(*nvidiacomv1alpha1.LPXGraphDeployment) + newDeployment := e.ObjectNew.(*nvidiacomv1alpha1.LPXGraphDeployment) + return !apiequality.Semantic.DeepEqual(oldDeployment.OwnerReferences, newDeployment.OwnerReferences) + }} + return predicate.Or(commoncontroller.GenerationOrDeletionChangedPredicate(), predicate.AnnotationChangedPredicate{}, ownerChanged) +} + +func (r *graphReconciler) addLPXWatches(ctrlBuilder *builder.Builder) *builder.Builder { + ctrlBuilder = ctrlBuilder. + Owns(&lpxv1alpha1.LPUPipelineRequest{}). + Watches( + &grovev1alpha1.ClusterTopologyBinding{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request { + return r.indexedLPXDependencyRequests(ctx, obj, lpxTopologyBindingRefIndex) + }), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). + Watches( + &groveschedulerv1alpha1.PodGang{}, + handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), + builder.WithPredicates(lpxPodGangPredicate()), + ) + return ctrlBuilder +} + +func (r *graphReconciler) lpxControllerEventFilter() predicate.Predicate { + // The topology mapper applies namespace filtering to each consuming DGD. + return predicate.NewPredicateFuncs(func(obj client.Object) bool { + if _, deviceClass := obj.(*resourcev1.DeviceClass); deviceClass { + return true + } + if _, topology := obj.(*grovev1alpha1.ClusterTopologyBinding); topology && obj.GetNamespace() == "" { + return true + } + return commoncontroller.NamespaceAllowed(r.Config, r.runtimeConfig, obj, obj.GetNamespace()) + }) +} + +// mapLPXDRADependencyToRequests receives non-nil claim, template and DeviceClass +// events from their registered informers, keeping LPX watches out of DGD. +func (r *graphReconciler) mapLPXDRADependencyToRequests(ctx context.Context, obj client.Object) []ctrl.Request { + switch dependency := obj.(type) { + case *resourcev1.ResourceClaim: + return r.indexedLPXDependencyRequests(ctx, dependency, lpxResourceClaimRefIndex) + case *resourcev1.ResourceClaimTemplate: + return r.indexedLPXDependencyRequests(ctx, dependency, lpxResourceClaimTemplateRefIndex) + case *resourcev1.DeviceClass: + return r.mapLPXDeviceClassToRequests(ctx, dependency) + default: + return nil + } +} + +// mapLPXChildToRequests receives non-nil, namespaced Grove informer objects. +func mapLPXChildToRequests(_ context.Context, obj client.Object) []ctrl.Request { + dgdName := obj.GetLabels()[consts.KubeLabelDynamoGraphDeploymentName] + if dgdName == "" { + return nil + } + return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: dgdName}}} +} + +// lpxPodGangPredicate receives non-nil PodGangs from the registered informer. +func lpxPodGangPredicate() predicate.Predicate { + isSchedulerWitness := func(obj client.Object) bool { + if obj.GetLabels()[consts.KubeLabelDynamoGraphDeploymentName] == "" { + return false + } + schedulerName := obj.GetLabels()[grovecommon.LabelSchedulerName] + return schedulerName == dynamolpx.SchedulerName || schedulerName == corev1.DefaultSchedulerName + } + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return isSchedulerWitness(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return isSchedulerWitness(e.Object) }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldGang := e.ObjectOld.(*groveschedulerv1alpha1.PodGang) + newGang := e.ObjectNew.(*groveschedulerv1alpha1.PodGang) + if !isSchedulerWitness(oldGang) && !isSchedulerWitness(newGang) { + return false + } + return oldGang.UID != newGang.UID || + oldGang.Generation != newGang.Generation || + !maps.Equal(oldGang.Labels, newGang.Labels) || + !maps.Equal(oldGang.Annotations, newGang.Annotations) || + !apiequality.Semantic.DeepEqual(oldGang.OwnerReferences, newGang.OwnerReferences) || + !apiequality.Semantic.DeepEqual(oldGang.DeletionTimestamp, newGang.DeletionTimestamp) + }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } +} + +// OwnsPodClique verifies the LPX owner chain before suppressing ordinary DGD readiness events. +// Annotations are authorable hints; missing cached owners must not hide updates. +func OwnsPodClique(ctx context.Context, reader client.Reader, clique *grovev1alpha1.PodClique) bool { + if clique == nil || reader == nil || !lpxPodCliqueMaterializationEvent(clique) { + return false + } + owner := metav1.GetControllerOf(clique) + if owner == nil || owner.Kind != groveconstants.KindPodCliqueScalingGroup || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { + return false + } + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, group); err != nil || group.UID != owner.UID { + return false + } + owner = metav1.GetControllerOf(group) + if owner == nil || owner.Kind != groveconstants.KindPodCliqueSet || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { + return false + } + pcs := &grovev1alpha1.PodCliqueSet{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, pcs); err != nil || pcs.UID != owner.UID { + return false + } + owner = metav1.GetControllerOf(pcs) + return owner != nil && owner.Kind == nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind && + owner.APIVersion == nvidiacomv1alpha1.GroupVersion.String() && owner.UID != "" && + owner.Name == clique.Labels[consts.KubeLabelDynamoGraphDeploymentName] && + string(owner.UID) == clique.Annotations[dynamo.LPXDeploymentUIDAnnotation] +} + +func lpxPodCliqueMaterializationEvent(obj client.Object) bool { + clique := obj.(*grovev1alpha1.PodClique) + role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] + return clique.Annotations[dynamolpx.WorkloadDigestAnnotation] != "" && + (role == lpxv1alpha1.PodRoleAgent || role == lpxv1alpha1.PodRoleConductor || role == lpxv1alpha1.PodRoleCyborgWorker) +} + +func lpxPodCliqueScalingGroupMaterializationEvent(obj client.Object) bool { + group := obj.(*grovev1alpha1.PodCliqueScalingGroup) + return group.Annotations[dynamolpx.WorkloadDigestAnnotation] != "" +} + +func lpxPodCliqueEventPredicates() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return lpxPodCliqueMaterializationEvent(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return lpxPodCliqueMaterializationEvent(e.Object) }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldClique := e.ObjectOld.(*grovev1alpha1.PodClique) + newClique := e.ObjectNew.(*grovev1alpha1.PodClique) + if !lpxPodCliqueMaterializationEvent(oldClique) && !lpxPodCliqueMaterializationEvent(newClique) { + return false + } + return commoncontroller.PodCliqueStatusChangeIsSignificant(oldClique, newClique) || + !ptr.Equal(oldClique.Status.CurrentPodTemplateHash, newClique.Status.CurrentPodTemplateHash) || + oldClique.Generation != newClique.Generation || + !maps.Equal(oldClique.Annotations, newClique.Annotations) || + !maps.Equal(oldClique.Labels, newClique.Labels) || + !apiequality.Semantic.DeepEqual(oldClique.OwnerReferences, newClique.OwnerReferences) || + !apiequality.Semantic.DeepEqual(oldClique.DeletionTimestamp, newClique.DeletionTimestamp) + }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } +} + +func lpxScalingGroupEventPredicates() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return lpxPodCliqueScalingGroupMaterializationEvent(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return lpxPodCliqueScalingGroupMaterializationEvent(e.Object) }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldGroup := e.ObjectOld.(*grovev1alpha1.PodCliqueScalingGroup) + newGroup := e.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) + if !lpxPodCliqueScalingGroupMaterializationEvent(oldGroup) && !lpxPodCliqueScalingGroupMaterializationEvent(newGroup) { + return false + } + return commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldGroup, newGroup) || + oldGroup.Generation != newGroup.Generation || + !maps.Equal(oldGroup.Annotations, newGroup.Annotations) || + !maps.Equal(oldGroup.Labels, newGroup.Labels) || + !apiequality.Semantic.DeepEqual(oldGroup.OwnerReferences, newGroup.OwnerReferences) || + !apiequality.Semantic.DeepEqual(oldGroup.DeletionTimestamp, newGroup.DeletionTimestamp) + }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } +} + +// lpxSourcePredicate ignores ordinary scaling/status traffic but wakes the child +// for relevant intent, source identity/deletion, and persisted restart selection. +// Its registered informer supplies non-nil DGDs. +func lpxSourcePredicate() predicate.Predicate { + filter := predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.(*nvidiacomv1beta1.DynamoGraphDeployment).HasLPXComponent() + }) + filter.UpdateFunc = func(e event.UpdateEvent) bool { + oldSource := e.ObjectOld.(*nvidiacomv1beta1.DynamoGraphDeployment) + newSource := e.ObjectNew.(*nvidiacomv1beta1.DynamoGraphDeployment) + if !oldSource.HasLPXComponent() && !newSource.HasLPXComponent() { + return false + } + if oldSource.UID != newSource.UID || !apiequality.Semantic.DeepEqual(oldSource.DeletionTimestamp, newSource.DeletionTimestamp) { + return true + } + // Spec edits advance generation; only the selected restart depends on status. + // Avoid conversion and hashing for ordinary status-only events. + oldRestart, newRestart := dynamo.LPXRestartToken(oldSource, ""), dynamo.LPXRestartToken(newSource, "") + if oldSource.Generation == newSource.Generation && oldRestart == newRestart && + maps.Equal(oldSource.Labels, newSource.Labels) && maps.Equal(oldSource.Annotations, newSource.Annotations) { + return false + } + oldRevision, oldErr := dynamo.LPXInputRevision(oldSource, oldRestart) + newRevision, newErr := dynamo.LPXInputRevision(newSource, newRestart) + return oldErr != nil || newErr != nil || oldRevision != newRevision + } + return filter +} diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go new file mode 100644 index 000000000000..68dd190dabc1 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -0,0 +1,406 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "testing" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +func TestLPXDeploymentPrimaryPredicateTracksOwnerIdentity(t *testing.T) { + t.Log("Build one current private handoff and its primary-resource predicate") + deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "graph", Namespace: "workloads", Generation: 1, + OwnerReferences: []metav1.OwnerReference{{Name: "graph", UID: "source-uid", Controller: ptr.To(true)}}, + }, + } + filter := lpxDeploymentPrimaryPredicate() + + t.Log("Ignore status-only updates that do not change publication authority") + statusOnly := deployment.DeepCopy() + statusOnly.Status.ObservedGeneration = 1 + require.False(t, filter.Update(event.UpdateEvent{ObjectOld: deployment, ObjectNew: statusOnly})) + + t.Log("Wake when the controller owner identity changes without advancing generation") + ownerChanged := deployment.DeepCopy() + ownerChanged.OwnerReferences[0].UID = "replacement-source-uid" + require.True(t, filter.Update(event.UpdateEvent{ObjectOld: deployment, ObjectNew: ownerChanged})) +} + +func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { + t.Log("Build matching, unrelated, and out-of-scope graph policies") + scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) + matching := topologyPolicyDGD("matching", "workloads", "fabric") + unrelated := topologyPolicyDGD("unrelated", "workloads", "other-fabric") + foreign := topologyPolicyDGD("foreign", "other", "fabric") + ordinary := topologyPolicyDGD("ordinary", "workloads", "fabric") + for _, source := range []*nvidiacomv1beta1.DynamoGraphDeployment{matching, unrelated, foreign} { + source.Spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX}} + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(matching, unrelated, foreign, ordinary). + WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). + WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + require.NotNil(t, options.FieldSelector, "topology events must use an index, not scan DGDs") + return delegated.List(ctx, list, opts...) + }}).Build() + reconciler := &graphReconciler{ + Client: kube, + runtimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{LPX: true}}, + Config: &configv1alpha1.OperatorConfiguration{ + Namespace: configv1alpha1.NamespaceConfiguration{Restricted: "workloads"}, + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + }, + } + + t.Log("Accept namespaced graph events only in the restricted namespace") + filter := reconciler.lpxControllerEventFilter() + require.True(t, filter.Create(event.CreateEvent{Object: matching})) + require.False(t, filter.Create(event.CreateEvent{Object: foreign})) + + t.Log("Admit cluster-scoped topology bindings for namespace-aware mapping") + binding := &grovev1alpha1.ClusterTopologyBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "fabric"}, + } + require.True(t, filter.Create(event.CreateEvent{Object: binding})) + require.False(t, filter.Create(event.CreateEvent{Object: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "foreign-cluster-object"}, + }})) + + t.Log("Requeue only the allowed graph that references the changed binding") + requests := reconciler.indexedLPXDependencyRequests(context.Background(), binding, lpxTopologyBindingRefIndex) + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "matching"}}}, requests) + + t.Log("Ignore bindings no graph consumes") + require.Empty(t, reconciler.indexedLPXDependencyRequests(context.Background(), &grovev1alpha1.ClusterTopologyBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "unused"}, + }, lpxTopologyBindingRefIndex)) + + t.Log("Editing a source policy removes the old dependency and indexes the new one") + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(matching), matching)) + matching.Spec.Experimental.KvTransferPolicy.ClusterTopologyName = "new-fabric" + require.NoError(t, kube.Update(t.Context(), matching)) + require.Empty(t, reconciler.indexedLPXDependencyRequests(t.Context(), &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "fabric"}}, lpxTopologyBindingRefIndex)) + require.Len(t, reconciler.indexedLPXDependencyRequests(t.Context(), &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "new-fabric"}}, lpxTopologyBindingRefIndex), 1) +} + +func topologyPolicyDGD(name, namespace, bindingName string) *nvidiacomv1beta1.DynamoGraphDeployment { + return &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Experimental: &nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec{ + KvTransferPolicy: &nvidiacomv1beta1.KvTransferPolicy{ClusterTopologyName: bindingName}, + }, + }, + } +} + +func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { + t.Log("Accept scheduler-scoped publication witnesses and map their exact namespaced graph") + predicate := lpxPodGangPredicate() + gang := &groveschedulerv1alpha1.PodGang{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agents", Namespace: "workloads", UID: "uid", Generation: 1, + Labels: map[string]string{ + grovecommon.LabelSchedulerName: dynamolpx.SchedulerName, + consts.KubeLabelDynamoGraphDeploymentName: "graph", + }, + }, + } + require.True(t, predicate.Create(event.CreateEvent{Object: gang})) + require.True(t, predicate.Delete(event.DeleteEvent{Object: gang})) + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "graph"}}}, + mapLPXChildToRequests(t.Context(), gang)) + + t.Log("Ignore status-only scheduler updates") + statusOnly := gang.DeepCopy() + statusOnly.Status.Phase = groveschedulerv1alpha1.PodGangPhase("scheduled") + require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: statusOnly})) + + t.Log("Wake on PodGang spec and mirrored attempt-annotation convergence") + specChange := gang.DeepCopy() + specChange.Generation++ + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: specChange})) + annotationChange := gang.DeepCopy() + annotationChange.Annotations = map[string]string{lpxDeploymentGenerationAnnotation: "2"} + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: annotationChange})) + + t.Log("Accept the ordinary scheduler witness and reject unrelated PodGangs") + ordinary := gang.DeepCopy() + ordinary.Labels[grovecommon.LabelSchedulerName] = corev1.DefaultSchedulerName + require.True(t, predicate.Create(event.CreateEvent{Object: ordinary})) + unrelated := ordinary.DeepCopy() + delete(unrelated.Labels, consts.KubeLabelDynamoGraphDeploymentName) + require.False(t, predicate.Create(event.CreateEvent{Object: unrelated})) + require.Empty(t, mapLPXChildToRequests(t.Context(), unrelated)) +} + +func TestLPXWorkloadEventPredicates(t *testing.T) { + for _, test := range []struct { + name string + object client.Object + predicate predicate.Predicate + observe func(client.Object) + }{ + {"clique", &grovev1alpha1.PodClique{}, lpxPodCliqueEventPredicates(), func(o client.Object) { o.(*grovev1alpha1.PodClique).Status.ReadyReplicas++ }}, + {"scaling group", &grovev1alpha1.PodCliqueScalingGroup{}, lpxScalingGroupEventPredicates(), func(o client.Object) { + o.(*grovev1alpha1.PodCliqueScalingGroup).Status.ObservedGeneration = ptr.To(int64(2)) + }}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Observe creation, deletion and all metadata used to fence materialization") + old := test.object + old.SetAnnotations(map[string]string{ + dynamolpx.WorkloadDigestAnnotation: "sha256:workload", + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + }) + require.True(t, test.predicate.Create(event.CreateEvent{Object: old})) + require.True(t, test.predicate.Delete(event.DeleteEvent{Object: old})) + require.False(t, test.predicate.Generic(event.GenericEvent{Object: old})) + + t.Log("Ordinary readiness traffic does not wake the LPX controller") + unrelated := old.DeepCopyObject().(client.Object) + unrelated.SetAnnotations(nil) + require.False(t, test.predicate.Create(event.CreateEvent{Object: unrelated})) + observed := unrelated.DeepCopyObject().(client.Object) + test.observe(observed) + require.False(t, test.predicate.Update(event.UpdateEvent{ObjectOld: unrelated, ObjectNew: observed})) + + t.Log("All compiled roles and PodClique template-hash convergence remain observable") + if clique, ok := old.(*grovev1alpha1.PodClique); ok { + for _, role := range []string{lpxv1alpha1.PodRoleAgent, lpxv1alpha1.PodRoleConductor, lpxv1alpha1.PodRoleCyborgWorker} { + current := clique.DeepCopy() + current.Annotations[lpxv1alpha1.PodRoleAnnotation] = role + require.True(t, test.predicate.Create(event.CreateEvent{Object: current})) + } + current := clique.DeepCopy() + current.Status.CurrentPodTemplateHash = ptr.To("current-template") + require.True(t, test.predicate.Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: current})) + } + + t.Log("Wake on role readiness, independently observed group generation and materialization fences") + for _, change := range []struct { + name string + mutate func(client.Object) + want bool + }{ + {"unchanged", func(client.Object) {}, false}, + {"status noise", func(o client.Object) { o.SetResourceVersion("2") }, false}, + {"status observed", test.observe, true}, + {"stamp lost", func(o client.Object) { o.SetAnnotations(nil) }, true}, + {"attempt changed", func(o client.Object) { o.GetAnnotations()[lpxDeploymentGenerationAnnotation] = "2" }, true}, + {"labels changed", func(o client.Object) { o.SetLabels(map[string]string{"changed": "true"}) }, true}, + {"owner changed", func(o client.Object) { o.SetOwnerReferences([]metav1.OwnerReference{{UID: "new-owner"}}) }, true}, + {"deleting", func(o client.Object) { now := metav1.Now(); o.SetDeletionTimestamp(&now) }, true}, + {"spec generation changed", func(o client.Object) { o.SetGeneration(o.GetGeneration() + 1) }, true}, + } { + t.Run(change.name, func(t *testing.T) { + updated := old.DeepCopyObject().(client.Object) + change.mutate(updated) + require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: updated})) + require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: updated, ObjectNew: old})) + }) + } + }) + } +} + +func TestLPXSourceWatchIgnoresOrdinaryChurn(t *testing.T) { + t.Log("Ordinary capacity and DGD status edits must not rerun LPX compilation") + source := newLPXTestSource(dynamolpx.PipelineSingle, "build-v2") + source.Spec.Components = append(source.Spec.Components, nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", Replicas: ptr.To(int32(1))}) + changed := source.DeepCopy() + changed.Generation++ + changed.Spec.Components[1].Replicas = ptr.To(int32(4)) + changed.Status.State = nvidiacomv1beta1.DGDStatePending + predicate := lpxSourcePredicate() + require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + + t.Log("Ignore ordinary creation and deletion, but keep LPX creation, deletion and deselection") + ordinary := source.DeepCopy() + ordinary.Spec.Components = ordinary.Spec.Components[1:] + ordinary.Generation++ + require.False(t, predicate.Create(event.CreateEvent{Object: ordinary})) + require.False(t, predicate.Delete(event.DeleteEvent{Object: ordinary})) + require.True(t, predicate.Create(event.CreateEvent{Object: source})) + require.True(t, predicate.Delete(event.DeleteEvent{Object: source})) + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: ordinary})) + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: source})) + + t.Log("LPX input changes and persisted restart selection wake the child") + changed.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "new-runtime" + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed = source.DeepCopy() + changed.Spec.Restart = &nvidiacomv1beta1.Restart{ID: "restart"} + require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed.Status.Restart = &nvidiacomv1beta1.RestartStatus{ObservedID: "restart", Phase: nvidiacomv1beta1.RestartPhaseRestarting, InProgress: []string{"lpx"}} + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed = source.DeepCopy() + changed.UID = "replacement" + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + + t.Log("Inherited scheduler metadata still wakes the child without a new source generation") + changed = source.DeepCopy() + changed.Labels = map[string]string{"priorityClassName": "inference"} + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed = source.DeepCopy() + changed.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) +} + +func BenchmarkLPXSourceWatchStatusChurn(b *testing.B) { + source := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "inference", Namespace: "workloads", Generation: 1}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, + LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "model/build"}, + Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLeader}, { + Name: nvidiacomv1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime:test"}}}, + }, + }}, + }}, + }, + } + updated := source.DeepCopy() + updated.Status.State = nvidiacomv1beta1.DGDStatePending + update := event.UpdateEvent{ObjectOld: source, ObjectNew: updated} + filter := lpxSourcePredicate() + b.ReportAllocs() + for b.Loop() { + if filter.Update(update) { + b.Fatal("status-only updates must not enqueue reconciliation") + } + } +} + +func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { + t.Log("Index consumed regular and init-container references, deduplicating shared aliases") + source := newLPXTestSource(dynamolpx.PipelineLPX, "build-v2") + child := newLPXTestDeployment(t, source) + pod := &dynamolpx.ServingComponent(source).ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "direct"}, {Name: "template"}} + pod.Containers = append(pod.Containers, corev1.Container{Name: "sidecar", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "direct"}}}}) + pod.InitContainers = []corev1.Container{ + {Name: "prepare", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "init"}}}}, + {Name: "native-sidecar", RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "init-template"}}}}, + } + pod.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "direct", ResourceClaimName: ptr.To("shared")}, + {Name: "template", ResourceClaimTemplateName: ptr.To("shared")}, + {Name: "init", ResourceClaimName: ptr.To("init-claim")}, + {Name: "init-template", ResourceClaimTemplateName: ptr.To("init-template")}, + {Name: "unused", ResourceClaimName: ptr.To("unused-claim"), ResourceClaimTemplateName: ptr.To("unused-template")}, + } + require.ElementsMatch(t, []string{"shared", "init-claim"}, lpxDRAClaimReferences(false)(source)) + require.ElementsMatch(t, []string{"shared", "init-template"}, lpxDRAClaimReferences(true)(source)) + + t.Log("Reference same-named claims and templates from two namespaces, excluding unused aliases") + foreign := source.DeepCopy() + foreign.Name, foreign.Namespace = "foreign", "other-namespace" + unused := source.DeepCopy() + unused.Name = "unused" + unusedPod := &dynamolpx.ServingComponent(unused).ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + unusedPod.Containers = unusedPod.Containers[:1] + unusedPod.Containers[0].Resources.Claims, unusedPod.InitContainers = nil, nil + claim := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: source.Namespace}, + Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ + Name: "gpu", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: "gpu"}, + }}}}, + } + claimTemplate := &resourcev1.ResourceClaimTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: source.Namespace}, + Spec: resourcev1.ResourceClaimTemplateSpec{Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ + Name: "gpu", FirstAvailable: []resourcev1.DeviceSubRequest{{Name: "first", DeviceClassName: "gpu"}, {Name: "second", DeviceClassName: "alternative-gpu"}}, + }}}}}, + } + foreignTemplate := claimTemplate.DeepCopy() + foreignTemplate.Namespace = foreign.Namespace + r := newLPXTestReconciler(t, nil, child, source, foreign, unused, claim, claimTemplate, foreignTemplate) + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + require.NotNil(t, options.FieldSelector, "DRA events must use indexes at both hops") + require.False(t, options.FieldSelector.Empty()) + return delegated.List(ctx, list, opts...) + }}) + sourceRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)} + foreignRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(foreign)} + + t.Log("Claim events remain namespace-local and unused aliases never enqueue a graph") + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claim)) + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) + require.Equal(t, []ctrl.Request{foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), foreignTemplate)) + require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.ResourceClaimTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: source.Namespace, Name: "unrelated"}})) + require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "unrelated"}})) + + t.Log("Class events join exact and alternative references and enqueue each graph once") + for _, class := range []string{"gpu", "alternative-gpu"} { + require.ElementsMatch(t, []ctrl.Request{sourceRequest, foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) + } + r.Config.Namespace.Restricted = "unwatched-namespace" + require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) + r.Config.Namespace.Restricted = source.Namespace + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) + + t.Log("Editing only a direct claim updates its DeviceClass dependency") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claim), claim)) + claim.Spec.Devices.Requests[0].Exactly.DeviceClassName = "replacement-claim-gpu" + require.NoError(t, r.Update(t.Context(), claim)) + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-claim-gpu"}})) + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) + + t.Log("A template-only edit removes its old class references without a DGD edit") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claimTemplate), claimTemplate)) + claimTemplate.Spec.Spec.Devices.Requests[0].FirstAvailable = []resourcev1.DeviceSubRequest{{Name: "replacement", DeviceClassName: "replacement-template-gpu"}} + require.NoError(t, r.Update(t.Context(), claimTemplate)) + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) + for _, class := range []string{"gpu", "alternative-gpu"} { + require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) + } + + t.Log("Missing templates remain indexed, but no longer contribute a DeviceClass dependency") + require.NoError(t, r.Delete(t.Context(), claimTemplate)) + require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) + require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) + + t.Log("Dependency events never mutate the source or its revision; ordinary components are not indexed") + stored := &nvidiacomv1beta1.DynamoGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), stored)) + require.Equal(t, source.Spec, stored.Spec) + require.Equal(t, source.Generation, stored.Generation) + source.Spec.Components[0].ComponentType = nvidiacomv1beta1.ComponentTypeDecode + require.Empty(t, lpxDRAClaimReferences(false)(source)) + require.Empty(t, lpxDRAClaimReferences(true)(source)) +} diff --git a/deploy/operator/internal/controller/primary_resource_predicate.go b/deploy/operator/internal/controller/primary_resource_predicate.go index fc4d0d0f1de6..437344434bf4 100644 --- a/deploy/operator/internal/controller/primary_resource_predicate.go +++ b/deploy/operator/internal/controller/primary_resource_predicate.go @@ -18,21 +18,15 @@ package controller import ( - "sigs.k8s.io/controller-runtime/pkg/event" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "sigs.k8s.io/controller-runtime/pkg/predicate" ) -func generationOrDeletionChangedPredicate() predicate.Predicate { +// DGD renderers consume metadata as well as spec; status alone is not input. +func dgdPrimaryPredicate() predicate.Predicate { return predicate.Or( - predicate.GenerationChangedPredicate{}, - predicate.Funcs{ - CreateFunc: func(event.CreateEvent) bool { return false }, - DeleteFunc: func(event.DeleteEvent) bool { return false }, - GenericFunc: func(event.GenericEvent) bool { return false }, - UpdateFunc: func(update event.UpdateEvent) bool { - return update.ObjectOld.GetDeletionTimestamp().IsZero() && - !update.ObjectNew.GetDeletionTimestamp().IsZero() - }, - }, + commoncontroller.GenerationOrDeletionChangedPredicate(), + predicate.AnnotationChangedPredicate{}, + predicate.LabelChangedPredicate{}, ) } diff --git a/deploy/operator/internal/controller/primary_resource_predicate_test.go b/deploy/operator/internal/controller/primary_resource_predicate_test.go index 3fa182e7bdb1..2f263a939f2c 100644 --- a/deploy/operator/internal/controller/primary_resource_predicate_test.go +++ b/deploy/operator/internal/controller/primary_resource_predicate_test.go @@ -20,14 +20,48 @@ package controller import ( "testing" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/event" ) +func TestDGDPrimaryPredicateObservesMetadataWithoutGenerationChanges(t *testing.T) { + t.Log("The primary predicate combines metadata and generation/deletion changes, not status") + old := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}} + filter := dgdPrimaryPredicate() + for _, test := range []struct { + name string + mutate func(*v1beta1.DynamoGraphDeployment) + want bool + }{ + {"annotation", func(o *v1beta1.DynamoGraphDeployment) { + o.Annotations = map[string]string{consts.KubeAnnotationDynamoDiscoveryBackend: "kubernetes"} + }, true}, + {"label", func(o *v1beta1.DynamoGraphDeployment) { o.Labels = map[string]string{"priorityClassName": "batch"} }, true}, + {"generation", func(o *v1beta1.DynamoGraphDeployment) { o.Generation++ }, true}, + {"deletion", func(o *v1beta1.DynamoGraphDeployment) { + now := metav1.Now() + o.DeletionTimestamp = &now + }, true}, + {"status noise", func(o *v1beta1.DynamoGraphDeployment) { + o.Status.State = v1beta1.DGDStateSuccessful + o.ResourceVersion = "2" + }, false}, + } { + t.Run(test.name, func(t *testing.T) { + changed := old.DeepCopy() + test.mutate(changed) + assert.Equal(t, test.want, filter.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: changed})) + }) + } +} + func TestGenerationOrDeletionChangedPredicate(t *testing.T) { - pred := generationOrDeletionChangedPredicate() + pred := commoncontroller.GenerationOrDeletionChangedPredicate() oldObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Generation: 1}} t.Run("preserves non-update event handling", func(t *testing.T) { diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index d2b53a854a6a..a8704ddbca64 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -9,6 +9,7 @@ import ( "fmt" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/modelendpoint" @@ -83,6 +84,9 @@ func SetupDynamoComponentDeployment(mgr ctrl.Manager, opts DynamoComponentDeploy } func SetupDynamoGraphDeployment(mgr ctrl.Manager, opts DynamoGraphDeploymentSetupOptions) error { + if err := lpxcontroller.Setup(mgr, opts.Config, opts.RuntimeConfig, opts.DockerSecretRetriever); err != nil { + return err + } if err := (&DynamoGraphDeploymentReconciler{ Client: mgr.GetClient(), Recorder: mgr.GetEventRecorder("dynamographdeployment"), diff --git a/deploy/operator/internal/controller/upgrade_test.go b/deploy/operator/internal/controller/upgrade_test.go index 90c3dea5aa00..f4e68f515fe5 100644 --- a/deploy/operator/internal/controller/upgrade_test.go +++ b/deploy/operator/internal/controller/upgrade_test.go @@ -231,6 +231,7 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch strategy: type: RollingUpdate rollingUpdate: @@ -426,6 +427,7 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch workerTemplate: metadata: labels: @@ -512,6 +514,7 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch size: 2 rolloutStrategy: type: "" @@ -684,6 +687,7 @@ spec: terminationGracePeriodSeconds: 60 securityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch replicas: 1 minAvailable: 1 - name: vllmdecodeworker @@ -785,6 +789,7 @@ spec: terminationGracePeriodSeconds: 60 securityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch replicas: 1 minAvailable: 1 cliqueStartupType: CliqueStartupTypeAnyOrder diff --git a/deploy/operator/internal/controller_common/resource.go b/deploy/operator/internal/controller_common/resource.go index 7f07d52cb1b2..31a4102367d4 100644 --- a/deploy/operator/internal/controller_common/resource.go +++ b/deploy/operator/internal/controller_common/resource.go @@ -100,10 +100,10 @@ func resolveSyncOptions(opts []SyncOption) syncOptions { return options } -// checkControllerOwnership verifies that existing is controlled by parentResource. +// CheckControllerOwnership verifies that existing is controlled by parentResource. // A namespaced name is not sufficient evidence of ownership: a resource with no // controller owner or one owned by a different parent is a collision. -func checkControllerOwnership(existing, parentResource client.Object, scheme *runtime.Scheme) error { +func CheckControllerOwnership(existing, parentResource client.Object, scheme *runtime.Scheme) error { if parentResource == nil { return nil } @@ -196,7 +196,7 @@ func SyncResource[T client.Object](ctx context.Context, r Reconciler, parentReso logs.Info(fmt.Sprintf("%s found.", resourceType)) if toDelete { if !options.sharedOwnership { - err = checkControllerOwnership(oldResource, parentResource, r.Scheme()) + err = CheckControllerOwnership(oldResource, parentResource, r.Scheme()) if err != nil { logs.Error(err, "Refusing to delete a resource with conflicting controller ownership") return @@ -264,7 +264,6 @@ func SyncObservedResource[T client.Object]( } updateAnnotations(desired, hash, 1) - recordResourceEvent(r, desired, corev1.EventTypeNormal, fmt.Sprintf("Create%s", resourceType), "Create", "Creating a new %s %s", resourceType, resourceNamespace) if err := r.Create(ctx, desired); err != nil { logs.Error(err, "Failed to create Resource.") recordResourceEvent(r, desired, corev1.EventTypeWarning, fmt.Sprintf("Create%s", resourceType), "Create", "Failed to create %s %s: %s", resourceType, resourceNamespace, err) @@ -277,7 +276,7 @@ func SyncObservedResource[T client.Object]( } if !resolveSyncOptions(opts).sharedOwnership { - if err := checkControllerOwnership(observed, parentResource, r.Scheme()); err != nil { + if err := CheckControllerOwnership(observed, parentResource, r.Scheme()); err != nil { logs.Error(err, "Refusing to reconcile a resource with conflicting controller ownership") var zero T return false, zero, err @@ -291,7 +290,6 @@ func SyncObservedResource[T client.Object]( } if !changeResult.NeedsUpdate { logs.Info(fmt.Sprintf("%s spec is the same. Skipping update.", resourceType)) - recordResourceEvent(r, observed, corev1.EventTypeNormal, fmt.Sprintf("Update%s", resourceType), "Update", "Skipping update %s %s", resourceType, resourceNamespace) return false, observed, nil } if changeResult.NewHash == nil { @@ -392,7 +390,7 @@ func nonEnvelopeFields(obj map[string]interface{}) map[string]interface{} { // backward-compatible hashing. For spec-less resources (ConfigMaps, Secrets, // Roles, etc.), it returns a map of all content fields. func getContentFields(u *unstructured.Unstructured) (any, bool) { - if spec, found, err := unstructured.NestedFieldCopy(u.Object, "spec"); err == nil && found { + if spec, found, err := unstructured.NestedFieldNoCopy(u.Object, "spec"); err == nil && found { return spec, true } @@ -416,7 +414,7 @@ func CopySpec(source, destination client.Object) error { } destUnstructured := &unstructured.Unstructured{Object: destMap} - if spec, found, err := unstructured.NestedFieldCopy(sourceUnstructured.Object, "spec"); err == nil && found { + if spec, found, err := unstructured.NestedFieldNoCopy(sourceUnstructured.Object, "spec"); err == nil && found { // Keep unstructured destinations opaque so unknown provider fields survive. if destinationUnstructured, ok := destination.(*unstructured.Unstructured); ok { return unstructured.SetNestedField(destinationUnstructured.Object, spec, "spec") @@ -471,14 +469,19 @@ type SpecChangeResult struct { // - SpecChangeResult with update information // - error if hash computation fails func GetSpecChangeResult(current client.Object, desired client.Object) (SpecChangeResult, error) { - desiredHash, err := GetSpecHash(desired) + desiredSpec, err := getSpec(desired) if err != nil { return SpecChangeResult{}, err } - currentMatchesDesired, err := specContentEqualPreserveListOrder(current, desired) + desiredHash, err := GetResourceHash(desiredSpec) if err != nil { return SpecChangeResult{}, err } + currentSpec, err := getSpec(current) + if err != nil { + return SpecChangeResult{}, err + } + currentMatchesDesired := equality.Semantic.DeepEqual(currentSpec, desiredSpec) lastAppliedHash := getAnnotation(current, NvidiaAnnotationHashKey) lastAppliedGenStr := getAnnotation(current, NvidiaAnnotationGenerationKey) @@ -539,7 +542,9 @@ func GetSpecChangeResult(current client.Object, desired client.Object) (SpecChan } // Detect manual changes: if current generation > last applied generation, - // someone else modified the resource after our last update + // someone else modified the resource after our last update. Some resources + // can advance generation for metadata-only writes, so confirm that the + // current content actually drifted before forcing a rewrite. if currentGen > 0 && currentGen > lastAppliedGen { if currentMatchesDesired { return annotationOnlyChange(), nil @@ -553,18 +558,6 @@ func GetSpecChangeResult(current client.Object, desired client.Object) (SpecChan }, nil } -func specContentEqualPreserveListOrder(current, desired client.Object) (bool, error) { - currentSpec, err := getSpec(current) - if err != nil { - return false, err - } - desiredSpec, err := getSpec(desired) - if err != nil { - return false, err - } - return equality.Semantic.DeepEqual(currentSpec, desiredSpec), nil -} - // getAnnotation safely retrieves an annotation value from an object func getAnnotation(obj client.Object, key string) string { annotations := obj.GetAnnotations() @@ -642,20 +635,17 @@ func GetResourceHash(obj any) (string, error) { return fmt.Sprintf("%x", hasher.Sum(nil)), nil } -// SortKeys recursively sorts the keys of a map to ensure consistent serialization +// SortKeys consumes a JSON-decoded tree, ordering slices for consistent serialization. func SortKeys(obj interface{}) interface{} { switch obj := obj.(type) { case map[string]interface{}: - sortedMap := make(map[string]interface{}) - keys := make([]string, 0, len(obj)) - for k := range obj { - keys = append(keys, k) + if obj == nil { + return map[string]interface{}{} } - sort.Strings(keys) - for _, k := range keys { - sortedMap[k] = SortKeys(obj[k]) + for k, v := range obj { + obj[k] = SortKeys(v) } - return sortedMap + return obj case []interface{}: // Check if the slice contains maps and sort them by the "name" field or the first available field if len(obj) > 0 { @@ -693,12 +683,14 @@ func SortKeys(obj interface{}) interface{} { // Helper function to get the first key of a map (alphabetically sorted) func firstKey(m map[string]interface{}) string { - keys := make([]string, 0, len(m)) + var first string + firstSet := false for k := range m { - keys = append(keys, k) + if !firstSet || k < first { + first, firstSet = k, true + } } - sort.Strings(keys) - return keys[0] + return first } // AppendUniqueImagePullSecrets appends secrets to existing, skipping any that already exist by name. diff --git a/deploy/operator/internal/controller_common/resource_test.go b/deploy/operator/internal/controller_common/resource_test.go index d57111542419..29ea799e55b2 100644 --- a/deploy/operator/internal/controller_common/resource_test.go +++ b/deploy/operator/internal/controller_common/resource_test.go @@ -29,10 +29,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/events" "github.com/bsm/gomega" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -522,8 +522,9 @@ func TestGetSpecChangeResult_GenerationTracking(t *testing.T) { name string currentGeneration int64 lastAppliedGeneration string // empty string means annotation not set - lastAppliedHash string // empty string means annotation not set, "match" means compute from current - desiredReplicas int64 // different from current (2) means hash will differ + lastAppliedHash string // empty string means annotation not set, "match" means compute from current, "desired" means compute from desired + currentReplicas int64 // zero means 2 + desiredReplicas int64 expectNeedsUpdate bool expectSpecNeedsUpdate bool expectManualChangeDetected bool @@ -547,6 +548,18 @@ func TestGetSpecChangeResult_GenerationTracking(t *testing.T) { expectManualChangeDetected: false, expectNewGeneration: 7, }, + { + name: "manual content change detected - generation increased", + currentGeneration: 7, + lastAppliedGeneration: "5", + lastAppliedHash: "desired", + currentReplicas: 3, + desiredReplicas: 2, + expectNeedsUpdate: true, + expectSpecNeedsUpdate: true, + expectManualChangeDetected: true, + expectNewGeneration: 8, + }, { // Upgrade scenario: hash matches but no generation annotation yet. name: "missing generation annotation - annotations only when spec is equivalent", @@ -621,6 +634,10 @@ func TestGetSpecChangeResult_GenerationTracking(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := gomega.NewGomegaWithT(t) + currentReplicas := tt.currentReplicas + if currentReplicas == 0 { + currentReplicas = 2 + } // Create current resource current := &unstructured.Unstructured{ @@ -634,7 +651,7 @@ func TestGetSpecChangeResult_GenerationTracking(t *testing.T) { "annotations": map[string]interface{}{}, }, "spec": map[string]interface{}{ - "replicas": int64(2), + "replicas": currentReplicas, }, }, } @@ -655,13 +672,18 @@ func TestGetSpecChangeResult_GenerationTracking(t *testing.T) { } // Set annotations based on test case - // "match" means the lastAppliedHash should match the CURRENT spec's hash - // (simulating that operator last applied what's currently in the cluster) + // "match" means the operator last applied the current content. + // "desired" means the operator last applied the desired content before + // the current object drifted. annotations := make(map[string]string) if tt.lastAppliedHash == "match" { hash, err := GetSpecHash(current) g.Expect(err).To(gomega.BeNil()) annotations[NvidiaAnnotationHashKey] = hash + } else if tt.lastAppliedHash == "desired" { + hash, err := GetSpecHash(desired) + g.Expect(err).To(gomega.BeNil()) + annotations[NvidiaAnnotationHashKey] = hash } else if tt.lastAppliedHash != "" { annotations[NvidiaAnnotationHashKey] = tt.lastAppliedHash } diff --git a/deploy/operator/internal/controller_common/workload_predicates.go b/deploy/operator/internal/controller_common/workload_predicates.go new file mode 100644 index 000000000000..c772d637ba91 --- /dev/null +++ b/deploy/operator/internal/controller_common/workload_predicates.go @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller_common + +import ( + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// GenerationOrDeletionChangedPredicate observes desired-state changes and finalizer entry. +func GenerationOrDeletionChangedPredicate() predicate.Predicate { + return predicate.Or( + predicate.GenerationChangedPredicate{}, + predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + DeleteFunc: func(event.DeleteEvent) bool { return false }, + GenericFunc: func(event.GenericEvent) bool { return false }, + UpdateFunc: func(update event.UpdateEvent) bool { + return update.ObjectOld.GetDeletionTimestamp().IsZero() && + !update.ObjectNew.GetDeletionTimestamp().IsZero() + }, + }, + ) +} + +// groveScheduledConditionChanged reports whether the scheduling conditions +// consumed by Grove readiness changed. +func groveScheduledConditionChanged(oldConditions, newConditions []metav1.Condition) bool { + for _, conditionType := range []string{ + groveconstants.ConditionTypePodCliqueScheduled, + groveconstants.ConditionTypeMinAvailableBreached, + } { + oldCondition := meta.FindStatusCondition(oldConditions, conditionType) + newCondition := meta.FindStatusCondition(newConditions, conditionType) + if (oldCondition == nil) != (newCondition == nil) { + return true + } + if oldCondition != nil && + newCondition != nil && + (oldCondition.Status != newCondition.Status || + oldCondition.Reason != newCondition.Reason) { + return true + } + } + return false +} + +// PodCliqueStatusChangeIsSignificant mirrors every PodClique field consumed by +// Grove readiness and capacity classification. +func PodCliqueStatusChangeIsSignificant( + oldPodClique *grovev1alpha1.PodClique, + newPodClique *grovev1alpha1.PodClique, +) bool { + return oldPodClique.Status.ReadyReplicas != newPodClique.Status.ReadyReplicas || + oldPodClique.Status.UpdatedReplicas != newPodClique.Status.UpdatedReplicas || + oldPodClique.Status.Replicas != newPodClique.Status.Replicas || + oldPodClique.Status.ScheduledReplicas != newPodClique.Status.ScheduledReplicas || + oldPodClique.Status.ScheduleGatedReplicas != newPodClique.Status.ScheduleGatedReplicas || + oldPodClique.Spec.Replicas != newPodClique.Spec.Replicas || + !ptr.Equal(oldPodClique.Status.ObservedGeneration, newPodClique.Status.ObservedGeneration) || + !ptr.Equal(oldPodClique.Status.CurrentPodCliqueSetGenerationHash, newPodClique.Status.CurrentPodCliqueSetGenerationHash) || + (oldPodClique.Status.UpdateProgress != nil && oldPodClique.Status.UpdateProgress.UpdateEndedAt != nil) != + (newPodClique.Status.UpdateProgress != nil && newPodClique.Status.UpdateProgress.UpdateEndedAt != nil) || + groveScheduledConditionChanged(oldPodClique.Status.Conditions, newPodClique.Status.Conditions) +} + +// PodCliqueScalingGroupStatusChangeIsSignificant mirrors every PodCliqueScalingGroup field +// consumed by Grove readiness and capacity classification. +func PodCliqueScalingGroupStatusChangeIsSignificant( + oldScalingGroup *grovev1alpha1.PodCliqueScalingGroup, + newScalingGroup *grovev1alpha1.PodCliqueScalingGroup, +) bool { + return oldScalingGroup.Status.AvailableReplicas != newScalingGroup.Status.AvailableReplicas || + oldScalingGroup.Status.UpdatedReplicas != newScalingGroup.Status.UpdatedReplicas || + oldScalingGroup.Status.Replicas != newScalingGroup.Status.Replicas || + oldScalingGroup.Status.ScheduledReplicas != newScalingGroup.Status.ScheduledReplicas || + oldScalingGroup.Spec.Replicas != newScalingGroup.Spec.Replicas || + !ptr.Equal(oldScalingGroup.Status.ObservedGeneration, newScalingGroup.Status.ObservedGeneration) || + !ptr.Equal(oldScalingGroup.Status.CurrentPodCliqueSetGenerationHash, newScalingGroup.Status.CurrentPodCliqueSetGenerationHash) || + (oldScalingGroup.Status.UpdateProgress != nil && oldScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) != + (newScalingGroup.Status.UpdateProgress != nil && newScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) || + groveScheduledConditionChanged(oldScalingGroup.Status.Conditions, newScalingGroup.Status.Conditions) +} diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index cdeb213a5fa8..1e919e6468e4 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -332,8 +332,8 @@ func gmsExtraClientContainersError( } func backendFrameworkForGeneratedDCDs(parentDGD *v1beta1.DynamoGraphDeployment) (string, error) { - if parentDGD.Spec.BackendFramework != "" { - return parentDGD.Spec.BackendFramework, nil + if backendFramework, err := ParseBackendFramework(parentDGD.Spec.BackendFramework); err == nil { + return string(backendFramework), nil } var detected BackendFramework @@ -1361,27 +1361,6 @@ func expandMultinodeGMSRoles(componentName string, numberOfNodes int32, totalEng return roles } -// LongestPodCliqueNameForDGDComponent returns the longest rendered PodClique -// name for a DGD component using the same role expansion as Grove rendering. -func LongestPodCliqueNameForDGDComponent( - componentName string, - component *v1beta1.DynamoComponentDeploymentSharedSpec, -) string { - lowerComponentName := strings.ToLower(componentName) - if component == nil || !component.UsesPCSG() { - return lowerComponentName - } - - longestName := lowerComponentName - for _, role := range expandRolesForComponent(componentName, component.Replicas, component.GetNumberOfNodes(), component) { - roleName := strings.ToLower(role.Name) - if len(roleName) > len(longestName) { - longestName = roleName - } - } - return longestName -} - // PCSNameForDGD computes the PodCliqueSet name for a DGD, auto-truncating if // the DGD name is too long to fit within Grove's combined resource name limit. // @@ -1391,17 +1370,10 @@ func LongestPodCliqueNameForDGDComponent( func PCSNameForDGD(dgdName string, components []v1beta1.DynamoComponentDeploymentSharedSpec) string { maxComponentBudget := 0 for i := range components { - component := &components[i] - componentName := component.ComponentName - lowerName := strings.ToLower(componentName) - var budget int - if component.UsesPCSG() { - // PCSG = lowerName, PCLQ = longest rendered role name. - budget = len(lowerName) + len(LongestPodCliqueNameForDGDComponent(componentName, component)) - } else { - // Single-node: PCLQ = lowerName (no PCSG). - budget = len(lowerName) + if components[i].ManagedByExternalController() { + continue } + budget := ComponentNameBudget(&components[i]) if budget > maxComponentBudget { maxComponentBudget = budget } @@ -1436,6 +1408,18 @@ const ( BackendFrameworkNoop BackendFramework = "noop" ) +// ParseBackendFramework converts a string to BackendFramework type. +// Returns an error if the framework string is not recognized. +func ParseBackendFramework(framework string) (BackendFramework, error) { + bf := BackendFramework(framework) + switch bf { + case BackendFrameworkVLLM, BackendFrameworkSGLang, BackendFrameworkTRTLLM, BackendFrameworkNoop: + return bf, nil + default: + return "", fmt.Errorf("unsupported backend framework: %s (valid values: vllm, sglang, trtllm, noop)", framework) + } +} + // ContainerGPUCount lazily resolves the main container's scalar or DRA-backed // GPU count. The same resolver can be shared across all roles of a component. type ContainerGPUCount func() (int64, error) @@ -1628,6 +1612,7 @@ func applyDefaultSecurityContext(podSpec *corev1.PodSpec) { // which maintains compatibility with both root and non-root images if podSpec.SecurityContext.FSGroup == nil { podSpec.SecurityContext.FSGroup = ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)) + podSpec.SecurityContext.FSGroupChangePolicy = ptr.To(corev1.FSGroupChangeOnRootMismatch) } } @@ -1635,6 +1620,7 @@ func applyDefaultSecurityContext(podSpec *corev1.PodSpec) { // Includes standard environment variables (DYNAMO_PORT, NATS_SERVER, ETCD_ENDPOINTS) // Deployment-specific environment merging should be handled by the caller // containerGPUs lazily resolves the main container's scalar or DRA-backed GPU count. +// component and operatorConfig must be non-nil. // //nolint:gocyclo func GenerateBasePodSpec( @@ -1651,20 +1637,63 @@ func GenerateBasePodSpec( deployerOverride MultinodeDeployer, // Optional: overrides factory-created deployer when non-nil containerGPUs ContainerGPUCount, ) (*corev1.PodSpec, error) { + return generateBasePodSpecWithDefaults( + component, + backendFramework, + secretsRetriever, + parentGraphDeploymentName, + namespace, + role, + numberOfNodes, + operatorConfig, + multinodeDeploymentType, + serviceName, + deployerOverride, + ComponentDefaultsFactory(string(component.ComponentType)), + containerGPUs, + ) +} + +// generateBasePodSpecWithDefaults requires non-nil component and operatorConfig. +// +//nolint:gocyclo +func generateBasePodSpecWithDefaults( + component *v1beta1.DynamoComponentDeploymentSharedSpec, + backendFramework BackendFramework, + secretsRetriever SecretsRetriever, + parentGraphDeploymentName string, + namespace string, + role Role, + numberOfNodes int32, + operatorConfig *configv1alpha1.OperatorConfiguration, + multinodeDeploymentType commonconsts.MultinodeDeploymentType, + serviceName string, + deployerOverride MultinodeDeployer, + componentDefaults ComponentDefaults, + containerGPUs ContainerGPUCount, +) (*corev1.PodSpec, error) { + // Resolve the one merge policy shared by every PodTemplate overlay. + podTemplateMergeStrategy, err := v1alpha1.ResolveExtraPodSpecMergeStrategy( + v1alpha1.ExtraPodSpecMergeStrategy(component.ExtraPodSpecMergeStrategy), + operatorConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy, + ) + if err != nil { + return nil, fmt.Errorf("failed to resolve podTemplate merge strategy: %w", err) + } + // Start with base container generated per component type annotations := GetPodTemplateAnnotations(component) componentContext, err := generateComponentContext(component, parentGraphDeploymentName, namespace, numberOfNodes, NewDiscoveryContext(operatorConfig.Discovery.Backend, annotations)) if err != nil { return nil, err } - componentDefaults := ComponentDefaultsFactory(string(component.ComponentType)) container, err := componentDefaults.GetBaseContainer(componentContext) if err != nil { return nil, fmt.Errorf("failed to get base container: %w", err) } if main := GetMainContainer(component); main != nil { - if err := mergeContainerByName(&container, main); err != nil { + if err := mergeContainerByName(&container, main, podTemplateMergeStrategy); err != nil { return nil, fmt.Errorf("failed to merge podTemplate main container: %w", err) } } @@ -1706,7 +1735,7 @@ func GenerateBasePodSpec( sidecars := make([]corev1.Container, 0) if component.PodTemplate != nil { - podSpecOverride := component.PodTemplate.Spec.DeepCopy() + podSpecOverride := *component.PodTemplate.Spec.DeepCopy() for _, userContainer := range podSpecOverride.Containers { if userContainer.Name != commonconsts.MainContainerName { sidecars = append(sidecars, userContainer) @@ -1714,7 +1743,7 @@ func GenerateBasePodSpec( } podSpecOverride.Containers = nil - if err := mergo.Merge(&podSpec, podSpecOverride, mergo.WithOverride); err != nil { + if err := mergePodSpecOverride(&podSpec, podSpecOverride, podTemplateMergeStrategy); err != nil { return nil, fmt.Errorf("failed to merge podTemplate spec: %w", err) } } @@ -1824,30 +1853,6 @@ func validateContainerVolumeMounts(volumeMounts []corev1.VolumeMount) error { return nil } -func mergeContainerByName(base *corev1.Container, override *corev1.Container) error { - if override == nil { - return nil - } - user := override.DeepCopy() - user.Name = commonconsts.MainContainerName - baseEnv := base.Env - if err := mergo.Merge(base, *user, mergo.WithOverride); err != nil { - return err - } - base.Env = MergeEnvs(baseEnv, user.Env) - if user.LivenessProbe != nil { - base.LivenessProbe = user.LivenessProbe.DeepCopy() - } - if user.ReadinessProbe != nil { - base.ReadinessProbe = user.ReadinessProbe.DeepCopy() - } - if user.StartupProbe != nil { - base.StartupProbe = user.StartupProbe - } - base.Name = commonconsts.MainContainerName - return nil -} - func applyCompilationCache(container *corev1.Container, component *v1beta1.DynamoComponentDeploymentSharedSpec, backendFramework BackendFramework) error { if component.CompilationCache == nil { return nil @@ -2369,8 +2374,6 @@ type cliqueParams struct { // buildCliqueForRole generates a single PodCliqueTemplateSpec for the given role, // injecting labels, annotations, checkpoint config, and scheduler settings. -// -//nolint:gocyclo func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, error) { podSpec, err := generatePodSpecForRole( p.r, p.component, p.backendFramework, p.secretsRetriever, @@ -2381,12 +2384,23 @@ func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, e return nil, fmt.Errorf("failed to generate podSpec for role %s: %w", p.r.Name, err) } + // Decorate the completed ordinary template through shared clique assembly. + return buildCliqueFromTemplate(p, corev1.PodTemplateSpec{ + ObjectMeta: generatePodMetadata(p.component, p.dynamoDeployment, getDGDAlphaComponent(p.dynamoDeployment, p.componentName), p.componentName, p.discoveryContext), + Spec: *podSpec, + }) +} + +// buildCliqueFromTemplate consumes a completed Pod template and its owned metadata. +// +//nolint:gocyclo +func buildCliqueFromTemplate(p cliqueParams, template corev1.PodTemplateSpec) (*grovev1alpha1.PodCliqueTemplateSpec, error) { // MinAvailable serves two purposes for Grove PCLQ: // 1. It defines the minimum number of pods that are guaranteed to be gang scheduled. // 2. It defines the minimum requirement of available pods in a PodClique. Violation of this threshold will result // in termination of the PodGang that it belongs to. minAvailable := int32(1) - // single-node standalone pclq set to component.MinAvailable if defined + // A component without a scaling group owns the availability threshold on its PCLQ. if !p.usesPCSG && p.component.MinAvailable != nil { minAvailable = *p.component.MinAvailable } @@ -2394,7 +2408,7 @@ func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, e // replica count. Plain multi-node needs every leader/worker rank ready for // collective operations. multi-node inter-pod GMS without failover creates // one replica per rank PCLQ, so this also evaluates to minAvailable=1. - if p.isMultinode && !p.isInterPodFailover { + if p.isMultinode && (!p.isInterPodFailover || !p.component.IsInterPodGMSEnabled()) { minAvailable = p.r.Replicas } replicas := p.r.Replicas @@ -2414,7 +2428,7 @@ func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, e RoleName: strings.ToLower(p.r.Name), Replicas: replicas, MinAvailable: ptr.To(minAvailable), - PodSpec: *podSpec, + PodSpec: template.Spec, }, } @@ -2425,15 +2439,11 @@ func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, e ) } - labels, err := generateLabels(p.component, p.dynamoDeployment, p.componentName, p.discoveryContext) - if err != nil { - return nil, fmt.Errorf("failed to generate labels: %w", err) - } - clique.Labels = labels + clique.Labels = template.Labels if p.isInterPodFailover && p.r.Role != RoleGMS { clique.Labels[commonconsts.KubeLabelDynamoFailoverEngineGroupMember] = commonconsts.KubeLabelValueTrue } - // Strip discovery labels from RoleGMS pods. generateLabels applies them + // Strip discovery labels from RoleGMS pods. generatePodMetadata applies them // unconditionally to every role for container-mode Pod reflector filtering // (see #8067), but GMS weight-server pods run gpu_memory_service.cli.server // — not the dynamo runtime — and never register a DynamoWorkerMetadata CR. @@ -2445,10 +2455,7 @@ func buildCliqueForRole(p cliqueParams) (*grovev1alpha1.PodCliqueTemplateSpec, e delete(clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) } - annotations, err := generateAnnotations(p.component, p.dynamoDeployment, p.componentName) - if err != nil { - return nil, fmt.Errorf("failed to generate annotations: %w", err) - } + annotations := template.Annotations for _, annotationKey := range commonconsts.KubeTopologySourceAnnotationKeys() { delete(annotations, annotationKey) } @@ -2555,6 +2562,7 @@ func resolveGroveSchedulerQueue( // GenerateGrovePodCliqueSet reads the provider inputs needed to construct the // desired PodCliqueSet. Resolved domain values stay local and are passed to // the leaf rendering helpers that consume them. +// The deployment must contain only components owned by this PCS. func GenerateGrovePodCliqueSet( ctx context.Context, dynamoDeployment *v1beta1.DynamoGraphDeployment, @@ -2566,50 +2574,12 @@ func GenerateGrovePodCliqueSet( existingRestartAnnotations map[string]string, checkpointInfoByComponent map[string]*checkpoint.CheckpointInfo, ) (*grovev1alpha1.PodCliqueSet, error) { - if dynamoDeployment == nil { - return nil, fmt.Errorf("cannot render Grove PodCliqueSet without a DynamoGraphDeployment") - } - if operatorConfig == nil { - return nil, fmt.Errorf("cannot render Grove PodCliqueSet without operator configuration") - } - if runtimeConfig == nil { - return nil, fmt.Errorf("cannot render Grove PodCliqueSet without runtime configuration") - } - - gangSet := &grovev1alpha1.PodCliqueSet{} - gangSet.Name = PCSNameForDGD(dynamoDeployment.Name, dynamoDeployment.Spec.Components) - gangSet.Namespace = dynamoDeployment.Namespace - gangSet.Labels = maps.Clone(dynamoDeployment.Spec.Labels) - if gangSet.Labels == nil { - gangSet.Labels = make(map[string]string) - } - gangSet.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name - gangSet.Annotations = maps.Clone(dynamoDeployment.Spec.Annotations) - // Volcano queue selection is consumed by Grove from the PodCliqueSet annotation. - // KAI-Scheduler is injected later on each clique via schedulerName and queue label. - injectVolcanoQueueAnnotation(gangSet, dynamoDeployment.Annotations, runtimeConfig) - gangSet.Spec.Replicas = 1 - updateStrategy, err := groveUpdateStrategyFromAnnotations(dynamoDeployment.Annotations) + // Construct the common PCS envelope before rendering ordinary components. + gangSet, err := newGrovePodCliqueSet(dynamoDeployment, operatorConfig, runtimeConfig) if err != nil { return nil, err } - if updateStrategy != nil { - gangSet.Spec.UpdateStrategy = &grovev1alpha1.PodCliqueSetUpdateStrategy{ - Type: *updateStrategy, - } - } - gangSet.Spec.Template.HeadlessServiceConfig = &grovev1alpha1.HeadlessServiceConfig{ - PublishNotReadyAddresses: true, - } - gangSet.Spec.Template.StartupType = ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder) - gangSet.Spec.Template.PriorityClassName = dynamoDeployment.Spec.PriorityClassName - if operatorConfig.Orchestrators.Grove.TerminationDelay.Duration > 0 { - gangSet.Spec.Template.TerminationDelay = &operatorConfig.Orchestrators.Grove.TerminationDelay - } - - // Inject deployment-level topology constraint (PCS template). - // specToGroveTopologyConstraint returns nil when input is nil, so this is a no-op without TAS. - gangSet.Spec.Template.TopologyConstraint = specToGroveTopologyConstraint(dynamoDeployment.Spec.TopologyConstraint) + gangSet.Name = PCSNameForDGD(dynamoDeployment.Name, dynamoDeployment.Spec.Components) validatedQueueName, err := resolveGroveSchedulerQueue(ctx, dynamoDeployment.Annotations, runtimeConfig) if err != nil { @@ -2628,27 +2598,27 @@ func GenerateGrovePodCliqueSet( } } - var scalingGroups []grovev1alpha1.PodCliqueScalingGroupConfig - var resourceClaimTemplates []grovev1alpha1.ResourceClaimTemplateConfig + //nolint:prealloc + var ( + scalingGroups []grovev1alpha1.PodCliqueScalingGroupConfig + resourceClaimTemplates []grovev1alpha1.ResourceClaimTemplateConfig + ) for i := range dynamoDeployment.Spec.Components { component := dynamoDeployment.Spec.Components[i].DeepCopy() componentName := component.ComponentName dynamoNamespace := GetDynamoNamespace(dynamoDeployment, component) - propagateDGDAnnotations(dynamoDeployment.GetAnnotations(), component) podTemplate := ensurePodTemplate(component) podTemplate.Labels[commonconsts.KubeLabelDynamoNamespace] = dynamoNamespace - // Determine backend framework using hybrid approach + if discoveryBackend != "" { + podTemplate.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend] = string(discoveryBackend) + } backendFramework, err := getBackendFrameworkFromComponent(component, dynamoDeployment) if err != nil { return nil, fmt.Errorf("failed to determine backend framework for component %s: %w", componentName, err) } - if discoveryBackend != "" { - podTemplate.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend] = string(discoveryBackend) - } - // Get checkpoint info for this component if available. var checkpointInfo *checkpoint.CheckpointInfo if checkpointInfoByComponent != nil { @@ -2663,8 +2633,8 @@ func GenerateGrovePodCliqueSet( isInterPodFailover := component.IsInterPodFailoverEnabled() usesPCSG := component.UsesPCSG() roles := expandRolesForComponent(componentName, component.Replicas, numberOfNodes, component) - var cliqueNames []string + var cliqueNames []string for _, r := range roles { clique, err := buildCliqueForRole(cliqueParams{ r: r, @@ -2718,12 +2688,53 @@ func GenerateGrovePodCliqueSet( )) } } - if len(scalingGroups) > 0 { - gangSet.Spec.Template.PodCliqueScalingGroupConfigs = scalingGroups + + gangSet.Spec.Template.PodCliqueScalingGroupConfigs = scalingGroups + gangSet.Spec.Template.ResourceClaimTemplates = resourceClaimTemplates + return gangSet, nil +} + +// newGrovePodCliqueSet constructs the shared Grove envelope. The caller assigns +// the workload identity and fills its cliques. Inputs must be non-nil and are not mutated. +func newGrovePodCliqueSet( + dynamoDeployment *v1beta1.DynamoGraphDeployment, + operatorConfig *configv1alpha1.OperatorConfiguration, + runtimeConfig *controller_common.RuntimeConfig, +) (*grovev1alpha1.PodCliqueSet, error) { + // Build the shared Grove object before rendering its component cliques. + gangSet := &grovev1alpha1.PodCliqueSet{} + gangSet.Namespace = dynamoDeployment.Namespace + gangSet.Labels = maps.Clone(dynamoDeployment.Spec.Labels) + if gangSet.Labels == nil { + gangSet.Labels = make(map[string]string) } - if len(resourceClaimTemplates) > 0 { - gangSet.Spec.Template.ResourceClaimTemplates = resourceClaimTemplates + gangSet.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name + gangSet.Annotations = maps.Clone(dynamoDeployment.Spec.Annotations) + if gangSet.Annotations == nil { + gangSet.Annotations = make(map[string]string) } + // Volcano queue selection is consumed by Grove from the PodCliqueSet annotation. + // KAI-Scheduler is injected later on each clique via schedulerName and queue label. + injectVolcanoQueueAnnotation(gangSet, dynamoDeployment.Annotations, runtimeConfig) + gangSet.Spec.Replicas = 1 + updateStrategy, err := groveUpdateStrategyFromAnnotations(dynamoDeployment.Annotations) + if err != nil { + return nil, err + } + if updateStrategy != nil { + gangSet.Spec.UpdateStrategy = &grovev1alpha1.PodCliqueSetUpdateStrategy{ + Type: *updateStrategy, + } + } + gangSet.Spec.Template.HeadlessServiceConfig = &grovev1alpha1.HeadlessServiceConfig{ + PublishNotReadyAddresses: true, + } + gangSet.Spec.Template.StartupType = ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder) + gangSet.Spec.Template.PriorityClassName = dynamoDeployment.Spec.PriorityClassName + if operatorConfig.Orchestrators.Grove.TerminationDelay.Duration > 0 { + gangSet.Spec.Template.TerminationDelay = &operatorConfig.Orchestrators.Grove.TerminationDelay + } + gangSet.Spec.Template.TopologyConstraint = specToGroveTopologyConstraint(dynamoDeployment.Spec.TopologyConstraint) return gangSet, nil } @@ -2846,46 +2857,30 @@ func generatePodSpecForRole( return podSpec, nil } -func generateLabels( +// generatePodMetadata builds owned label and annotation maps from nonnil component and DGD inputs. +// alphaComponent may be nil when no converted alpha component is available. +func generatePodMetadata( component *v1beta1.DynamoComponentDeploymentSharedSpec, dynamoDeployment *v1beta1.DynamoGraphDeployment, + alphaComponent *v1alpha1.DynamoComponentDeploymentSharedSpec, componentName string, discovery DiscoveryContext, -) (map[string]string, error) { +) metav1.ObjectMeta { labels := make(map[string]string) - labels[commonconsts.KubeLabelDynamoSelector] = GetDCDResourceName(dynamoDeployment, componentName, "") - labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name - labels[commonconsts.KubeLabelDynamoComponent] = componentName - if component.ComponentType != "" { - labels[commonconsts.KubeLabelDynamoComponentType] = string(component.ComponentType) - } - if dynamoDeployment.HasEPPComponent() && IsWorkerComponent(string(component.ComponentType)) { - labels[commonconsts.KubeLabelDynamoComponentClass] = commonconsts.ComponentClassWorker - } - if subComponentType := getDGDComponentAlphaSubComponentType(dynamoDeployment, componentName); subComponentType != "" { - labels[commonconsts.KubeLabelDynamoSubComponentType] = subComponentType - } - labels[commonconsts.KubeLabelDynamoNamespace] = GetDynamoNamespace(dynamoDeployment, component) + annotations := make(map[string]string) // Add base model label if modelRef is specified AddBaseModelLabel(labels, component.ModelRef) // Merge user-supplied labels first so they cannot overwrite checkpoint labels. setMetricsLabels(labels, dynamoDeployment) - if dynamoDeployment.Spec.Labels != nil { - if err := mergo.Merge(&labels, dynamoDeployment.Spec.Labels, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge labels: %w", err) - } + maps.Copy(labels, dynamoDeployment.Spec.Labels) + maps.Copy(annotations, dynamoDeployment.Spec.Annotations) + if alphaComponent != nil { + maps.Copy(labels, alphaComponent.Labels) + maps.Copy(annotations, alphaComponent.Annotations) } - if componentLabels := getDGDComponentAlphaLabels(dynamoDeployment, componentName); componentLabels != nil { - if err := mergo.Merge(&labels, componentLabels, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge preserved component labels: %w", err) - } - } - if podTemplateLabels := GetPodTemplateLabels(component); podTemplateLabels != nil { - if err := mergo.Merge(&labels, podTemplateLabels, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge podTemplate labels: %w", err) - } - } - // Re-apply system labels after user merge to prevent override + maps.Copy(labels, GetPodTemplateLabels(component)) + maps.Copy(annotations, GetPodTemplateAnnotations(component)) + // Apply system labels after user merge to prevent override labels[commonconsts.KubeLabelDynamoSelector] = GetDCDResourceName(dynamoDeployment, componentName, "") labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name labels[commonconsts.KubeLabelDynamoComponent] = componentName @@ -2895,8 +2890,8 @@ func generateLabels( if dynamoDeployment.HasEPPComponent() && IsWorkerComponent(string(component.ComponentType)) { labels[commonconsts.KubeLabelDynamoComponentClass] = commonconsts.ComponentClassWorker } - if subComponentType := getDGDComponentAlphaSubComponentType(dynamoDeployment, componentName); subComponentType != "" { - labels[commonconsts.KubeLabelDynamoSubComponentType] = subComponentType + if alphaComponent != nil && alphaComponent.SubComponentType != "" { + labels[commonconsts.KubeLabelDynamoSubComponentType] = alphaComponent.SubComponentType } labels[commonconsts.KubeLabelDynamoNamespace] = GetDynamoNamespace(dynamoDeployment, component) if workerHash := GetPodTemplateLabels(component)[commonconsts.KubeLabelDynamoWorkerHash]; workerHash != "" { @@ -2915,15 +2910,7 @@ func generateLabels( labels[commonconsts.KubeLabelDynamoDiscoveryBackend] = commonconsts.DiscoveryBackendKubernetes labels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue } - return labels, nil -} - -func getDGDComponentAlphaSubComponentType(dgd *v1beta1.DynamoGraphDeployment, componentName string) string { - component := getDGDAlphaComponent(dgd, componentName) - if component == nil { - return "" - } - return component.SubComponentType + return metav1.ObjectMeta{Labels: labels, Annotations: annotations} } func getDGDComponentAlphaLabels(dgd *v1beta1.DynamoGraphDeployment, componentName string) map[string]string { @@ -2944,18 +2931,19 @@ func getDGDComponentAlphaAnnotations(dgd *v1beta1.DynamoGraphDeployment, compone func getDGDAlphaComponent(dgd *v1beta1.DynamoGraphDeployment, componentName string) *v1alpha1.DynamoComponentDeploymentSharedSpec { alpha := getDGDAlpha(dgd) - if alpha == nil || alpha.Spec.Services == nil { + if alpha == nil { return nil } return alpha.Spec.Services[componentName] } +// getDGDAlpha returns a read-only projection that may share data with dgd. func getDGDAlpha(dgd *v1beta1.DynamoGraphDeployment) *v1alpha1.DynamoGraphDeployment { if dgd == nil { return nil } alpha := &v1alpha1.DynamoGraphDeployment{} - if err := alpha.ConvertFrom(dgd.DeepCopy()); err != nil { + if err := alpha.ConvertFrom(dgd); err != nil { return nil } return alpha @@ -2971,27 +2959,6 @@ func GetDGDPreservedAlphaPVCs(dgd *v1beta1.DynamoGraphDeployment) []v1alpha1.PVC return append([]v1alpha1.PVC(nil), alpha.Spec.PVCs...) } -func generateAnnotations(component *v1beta1.DynamoComponentDeploymentSharedSpec, dynamoDeployment *v1beta1.DynamoGraphDeployment, componentName string) (map[string]string, error) { - annotations := make(map[string]string) - if dynamoDeployment.Spec.Annotations != nil { - if err := mergo.Merge(&annotations, dynamoDeployment.Spec.Annotations, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge DGD annotations: %w", err) - } - } - if componentAnnotations := getDGDComponentAlphaAnnotations(dynamoDeployment, componentName); componentAnnotations != nil { - if err := mergo.Merge(&annotations, componentAnnotations, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to merge preserved component annotations: %w", err) - } - } - if podTemplateAnnotations := GetPodTemplateAnnotations(component); podTemplateAnnotations != nil { - err := mergo.Merge(&annotations, podTemplateAnnotations, mergo.WithOverride) - if err != nil { - return nil, fmt.Errorf("failed to merge annotations: %w", err) - } - } - return annotations, nil -} - // DetectBackendFrameworkFromArgs detects the backend framework from command/args. func DetectBackendFrameworkFromArgs(command []string, args []string) (BackendFramework, error) { // Combine command and args to search through all parts @@ -3050,11 +3017,7 @@ func determineBackendFramework( } } - // Get explicit framework - var explicitFramework BackendFramework - if explicitBackendFramework != "" { - explicitFramework = BackendFramework(explicitBackendFramework) - } + explicitFramework, _ := ParseBackendFramework(explicitBackendFramework) // Validate consistency if both detected and explicit exist if detectedFramework != "" && detectedFramework != BackendFrameworkNoop && explicitFramework != "" && detectedFramework != explicitFramework { diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go new file mode 100644 index 000000000000..d840be168a04 --- /dev/null +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -0,0 +1,507 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamo + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RenderLPXBasePodCliqueSet applies shared Dynamo defaults to the Grove envelope +// and independently owned LPX role templates. The LPX controller completes the +// compiled workload and its identity. All pointer inputs except secretsRetriever +// must be non-nil; preflight supplies the validated workload and child PCS name. +func RenderLPXBasePodCliqueSet( + ctx context.Context, + dynamoDeployment *v1beta1.DynamoGraphDeployment, + operatorConfig *configv1alpha1.OperatorConfiguration, + runtimeConfig *controller_common.RuntimeConfig, + kubeClient client.Reader, + secretsRetriever SecretsRetriever, + selectedWorkload *dynamolpx.SelectedWorkload, + pcsName string, +) (*grovev1alpha1.PodCliqueSet, *dynamolpx.RenderInput, error) { + // Reuse the Grove envelope, but keep LPX compilation out of normal rendering. + pcs, err := newGrovePodCliqueSet(dynamoDeployment, operatorConfig, runtimeConfig) + if err != nil { + return nil, nil, err + } + pcs.Name = pcsName + queue, err := resolveGroveSchedulerQueue(ctx, dynamoDeployment.Annotations, runtimeConfig) + if err != nil { + return nil, nil, err + } + + // Cyborg uses ordinary worker KV-transfer defaults, including the binding's domains. + var topologyDomains []v1beta1.TopologyDomain + if dynamoDeployment.Spec.Experimental != nil { + topologyDomains, err = resolveGroveClusterTopologyDomains(ctx, kubeClient, dynamoDeployment.Spec.Experimental.KvTransferPolicy) + if err != nil { + return nil, nil, err + } + } + + // Preserve the complete source graph as the independent role-default context. + component := dynamolpx.ServingComponent(dynamoDeployment) + + input, gpuCliques, err := renderLPXComponents(cliqueParams{ + component: component, componentName: component.ComponentName, + dynamoDeployment: dynamoDeployment, operatorConfig: operatorConfig, runtimeConfig: runtimeConfig, + secretsRetriever: secretsRetriever, + discoveryBackend: controller_common.GetDiscoveryBackend(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), + discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), + validatedQueueName: queue, + groveClusterTopologyDomains: topologyDomains, + }, selectedWorkload) + if err != nil { + return nil, nil, err + } + pcs.Spec.Template.Cliques = gpuCliques + + // Preserve scheduler-visible source metadata below the new owner boundary. + // Explicit PCS/spec metadata retains precedence over inherited DGD metadata. + labels, annotations := lpxSchedulingMetadata(dynamoDeployment.Labels), lpxSchedulingMetadata(dynamoDeployment.Annotations) + maps.Copy(labels, pcs.Labels) + maps.Copy(annotations, pcs.Annotations) + pcs.Labels, pcs.Annotations = labels, annotations + return pcs, input, nil +} + +const ( + LPXDeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" + LPXDeploymentGenerationAnnotation = "lpx.nvidia.com/deployment-generation" + LPXInputRevisionAnnotation = "lpx.nvidia.com/input-revision" + LPXPCSNameAnnotation = "lpx.nvidia.com/podcliqueset-name" + LPXRestartAnnotation = "lpx.nvidia.com/restart-id" + LPXServingLabel = "lpx.nvidia.com/serving" + lpxGPUExecutionRole = "gpu" +) + +// LPXRestartToken advances only from the DGD's persisted restart selection. +// Preserve the last delivered token while another component restarts. +func LPXRestartToken(source *v1beta1.DynamoGraphDeployment, previous string) string { + request, observed := source.Spec.Restart, source.Status.Restart + if request == nil || request.ID == "" || observed == nil || observed.ObservedID != request.ID { + return previous + } + state := DetermineRestartState(source, observed) + for index := range source.Spec.Components { + component := &source.Spec.Components[index] + if component.IsLPX() && state.ShouldAnnotateComponent(component.ComponentName) { + return state.Timestamp + } + } + return previous +} + +// ValidateLPXSource rejects foreign, stale, or fabricated child handoffs before +// the LPX controller acquires a build or publishes a workload. +// Both objects must be non-nil. +func ValidateLPXSource(deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { + owner := metav1.GetControllerOf(deployment) + if deployment.Namespace != source.Namespace || deployment.Name != source.Name || + owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" || + owner.Name != source.Name || owner.UID != source.UID { + return fmt.Errorf("LPXGraphDeployment requires its exact source DGD and controller owner") + } + if provider := source.Annotations[commonconsts.KubeAnnotationWorkloadProvider]; provider != "" && provider != commonconsts.WorkloadProviderGrove { + return fmt.Errorf("LPX requires the Grove workload provider") + } + if !source.DeletionTimestamp.IsZero() { + return fmt.Errorf("source DGD is being deleted") + } + restart := LPXRestartToken(source, deployment.Annotations[LPXRestartAnnotation]) + revision, err := LPXInputRevision(source, restart) + if err != nil { + return err + } + if deployment.Spec.InputRevision != revision || deployment.Annotations[LPXRestartAnnotation] != restart || + deployment.Annotations[LPXPCSNameAnnotation] != PCSNameForLPX(source) { + return fmt.Errorf("LPXGraphDeployment is waiting for the current source input revision") + } + return nil +} + +// lpxInputRevisionPayload is the normalized source intent hashed for inputRevision. +// Keep complete selection and rendering inputs so nested role templates and build +// settings cannot silently fall out of the revision. +type lpxInputRevisionPayload struct { + Components []v1beta1.DynamoComponentDeploymentSharedSpec + Scheduling *v1beta1.SchedulingSpec + Labels map[string]string + Annotations map[string]string + PropagatedAnnotations map[string]string + Environment []corev1.EnvVar `json:"Env"` + PriorityClass string + BackendFramework string + RestartToken string `json:"Restart"` + KVTransferPolicy *v1beta1.KvTransferPolicy + TopologyConstraint *v1beta1.SpecTopologyConstraint + ProviderOverride *v1beta1.ProviderOverride + SchedulingLabels map[string]string `json:",omitempty"` + EPPEnabled bool `json:",omitempty"` + AlphaLabels map[string]map[string]string `json:",omitempty"` + AlphaAnnotations map[string]map[string]string `json:",omitempty"` + AlphaSubComponentType map[string]string `json:",omitempty"` +} + +// LPXInputRevision hashes all LPX components and their shared render inputs. +// Source identity is checked separately by ValidateLPXSource. Ordinary component payloads, +// DGD bookkeeping and raw restart requests are excluded: restart is the effective +// token selected by persisted DGD restart state. Operator configuration and external +// dependencies are not source revisions; the LPX controller observes them separately. +func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (string, error) { + components := dynamolpx.Components(dgd) + if len(components) == 0 { + return "", fmt.Errorf("LPX component is required") + } + + // Canonicalize authored lists on shallow copies; nested templates remain read-only. + canonical := make([]v1beta1.DynamoComponentDeploymentSharedSpec, len(components)) + for index, component := range components { + canonical[index] = *component + canonical[index].Roles = slices.Clone(component.Roles) + slices.SortFunc(canonical[index].Roles, func(a, b v1beta1.ComponentRoleSpec) int { + return strings.Compare(a.Name, b.Name) + }) + } + slices.SortFunc(canonical, func(a, b v1beta1.DynamoComponentDeploymentSharedSpec) int { + return strings.Compare(a.ComponentName, b.ComponentName) + }) + + annotations := lpxSchedulingMetadata(dgd.Annotations) + for _, key := range append(slices.Clone(dgdPropagatedAnnotationKeys), commonconsts.KubeAnnotationLPXSchedulerBackend, commonconsts.KubeAnnotationWorkloadProvider, + commonconsts.KubeAnnotationGroveUpdateStrategy, commonconsts.KubeAnnotationKaiSchedulerQueue, commonconsts.KubeAnnotationVolcanoQueue) { + if value, exists := dgd.Annotations[key]; exists { + annotations[key] = value + } + } + var policy *v1beta1.KvTransferPolicy + if dgd.Spec.Experimental != nil { + policy = dgd.Spec.Experimental.KvTransferPolicy + } + input := lpxInputRevisionPayload{ + Components: canonical, + Scheduling: dgd.Spec.Scheduling, + Labels: dgd.Spec.Labels, + Annotations: dgd.Spec.Annotations, + PropagatedAnnotations: annotations, + Environment: dgd.Spec.Env, + PriorityClass: dgd.Spec.PriorityClassName, + BackendFramework: dgd.Spec.BackendFramework, + RestartToken: restart, + KVTransferPolicy: policy, + TopologyConstraint: dgd.Spec.TopologyConstraint, + ProviderOverride: dgd.Spec.ProviderOverride, + SchedulingLabels: lpxSchedulingMetadata(dgd.Labels), + EPPEnabled: dgd.HasEPPComponent(), + AlphaLabels: make(map[string]map[string]string), + AlphaAnnotations: make(map[string]map[string]string), + AlphaSubComponentType: make(map[string]string), + } + // The shared role renderer also reads preserved alpha component metadata. + // Use the same conversion reader, excluding ordinary component payloads and + // keeping conversion bookkeeping out of LPX's input contract. + if alpha := getDGDAlpha(dgd); alpha != nil { + for _, component := range components { + if source := alpha.Spec.Services[component.ComponentName]; source != nil { + if len(source.Labels) != 0 { + input.AlphaLabels[component.ComponentName] = source.Labels + } + if len(source.Annotations) != 0 { + input.AlphaAnnotations[component.ComponentName] = source.Annotations + } + if source.SubComponentType != "" { + input.AlphaSubComponentType[component.ComponentName] = source.SubComponentType + } + } + } + } + data, err := json.Marshal(input) + if err != nil { + return "", err + } + return fmt.Sprintf("sha256:%x", sha256.Sum256(data)), nil +} + +// lpxSchedulingMetadata preserves the scheduler inputs previously inherited +// through the DGD owner. With existing scheduler RBAC, ownership traversal now +// stops at the PCS; do not include unrelated controller bookkeeping in revisions. +func lpxSchedulingMetadata(metadata map[string]string) map[string]string { + selected := make(map[string]string) + for key, value := range metadata { + if strings.HasPrefix(key, "kai.scheduler/") || key == "priorityClassName" || key == "project" || key == "user" { + selected[key] = value + } + } + return selected +} + +// PCSNameForLPX hashes the source identity within the Grove name budget, +// independently of ordinary components. The LPX planner checks materialized names. +func PCSNameForLPX(dgd *v1beta1.DynamoGraphDeployment) string { + // The serving component owns all generated LPX names, including draft cliques. + groupBudget, cliqueBudget := 8, 0 + if component := dynamolpx.ServingComponent(dgd); component != nil { + groupBudget = max(groupBudget, len(component.ComponentName)) + cliqueBudget = max(cliqueBudget, longestLPXCliqueNameLength(component.ComponentName), len(component.ComponentName)+len("-engine-gpu")) + } + budget := commonconsts.MaxCombinedGroveResourceNameLength - groupBudget - cliqueBudget + budget = max(budget, 8) + name := strings.ReplaceAll(dgd.Name, ".", "-") + digest := sha256.Sum256([]byte(dgd.Namespace + "/" + dgd.Name + "/" + string(dgd.UID))) + if budget <= 13 { + return fmt.Sprintf("%x", digest[:4]) + } + return strings.TrimRight(name[:min(len(name), budget-13)], "-") + fmt.Sprintf("-%x-lpx", digest[:4]) +} + +func longestLPXCliqueNameLength(componentName string) int { + // At most eight drafts and one target use a single-digit model index. + return len(strings.ToLower(componentName)) + len(commonconsts.GroveRoleSuffixWorker) + 5 +} + +// LPXComponentNameBudget reserves the serving group's name and its longest clique. +// Shared draft components use the serving component's generated names. +func LPXComponentNameBudget(componentName string) int { + return len(componentName) + longestLPXCliqueNameLength(componentName) +} + +// renderLPXComponents merges ordinary Dynamo defaults independently into every +// authored role. The full source DGD supplies discovery and shared defaults; +// LPX component's roles are returned to the same PCS renderer. +// Preflight has validated the runtime shape and every Agent template. +func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { + // Pass naming and runtime inputs; deployment identity is stamped only on final resources. + input := &dynamolpx.RenderInput{ + DGDName: p.dynamoDeployment.Name, + MinAvailable: p.component.MinAvailable, + Stages: make(map[string]corev1.PodTemplateSpec), + SSHSecretName: p.operatorConfig.MPI.SSHSecretName, + } + + // Resolve preserved alpha metadata once for all independently rendered roles. + var alphaComponents map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec + if alpha := getDGDAlpha(p.dynamoDeployment); alpha != nil { + alphaComponents = alpha.Spec.Services + } + + gpuCliques := make([]*grovev1alpha1.PodCliqueTemplateSpec, 0, 1) + for _, component := range dynamolpx.Components(p.dynamoDeployment) { + alphaComponent := alphaComponents[component.ComponentName] + agent := component.ComponentRole(v1beta1.ComponentRoleWorker) + lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) + lpuDefaults := &imageEntrypointComponentDefaults{ComponentDefaults: &BaseComponentDefaults{}} + lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, + p.discoveryContext, workload, lpuDefaults, nil) + if err != nil { + return nil, nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) + } + lpuTemplate.Labels[dynamolpx.StageLabel] = component.ComponentName + lpuTemplate.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" + input.Stages[component.ComponentName] = *lpuTemplate + conductor := component.ComponentRole(v1beta1.ComponentRoleLeader) + if component != p.component { + continue + } + + // Non-hybrid conductors use their own template, or the Agent fallback in LPX lowering. + if workload.Pipeline() != dynamolpx.PipelineLPX { + if conductor != nil && conductor.PodTemplate != nil { + role := lpxRoleComponent(component, conductor.PodTemplate, p.dynamoDeployment, p.discoveryBackend) + input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, + p.discoveryContext, workload, lpuDefaults, nil) + if err != nil { + return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) + } + input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName + input.Conductor.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" + } + continue + } + + // Keep the existing hybrid execution path; only its authored template location changes. + template, replicas := agent.PodTemplate, int32(1) + if conductor != nil { + replicas = ptr.Deref(conductor.Replicas, 1) + if conductor.PodTemplate != nil { + template = conductor.PodTemplate + } + } + role := lpxRoleComponent(component, template, p.dynamoDeployment, p.discoveryBackend) + role.ComponentType = v1beta1.ComponentTypeDecode + role.Replicas = ptr.To(replicas) + role.MinAvailable = nil + defaults := ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)) + if workload.BuildFamily() == dynamolpx.BuildFamilyXT { + defaults = &selectedCyborgComponentDefaults{ + ComponentDefaults: defaults, workload: workload, dgdName: p.dynamoDeployment.Name, + replicas: *role.Replicas, lpxPodSpec: lpuTemplate.Spec, + } + } else { + defaults = &imageEntrypointComponentDefaults{ComponentDefaults: defaults} + } + gpu := p + gpu.component = role + gpu.componentName = component.ComponentName + gpu.r = ServiceRole{Name: workload.CyborgTemplateName(), Role: RoleMain, Replicas: *role.Replicas} + gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, + p.discoveryContext, workload, defaults, p.groveClusterTopologyDomains) + if err != nil { + return nil, nil, fmt.Errorf("rendering %s.conductor: failed to generate podSpec for role %s: %w", component.ComponentName, gpu.r.Name, err) + } + clique, err := buildCliqueFromTemplate(gpu, *gpuTemplate) + if err != nil { + return nil, nil, fmt.Errorf("rendering %s.conductor: %w", component.ComponentName, err) + } + // Every GPU worker is required by the same compiled engine replica. + clique.Spec.MinAvailable = ptr.To(clique.Spec.Replicas) + clique.Labels[commonconsts.KubeLabelDynamoComponentType] = string(v1beta1.ComponentTypeLPX) + clique.Labels[dynamolpx.StageLabel] = component.ComponentName + clique.Labels[dynamolpx.ExecutionRoleLabel] = lpxGPUExecutionRole + gpuCliques = append(gpuCliques, clique) + } + return input, gpuCliques, nil +} + +func lpxRoleComponent(source *v1beta1.DynamoComponentDeploymentSharedSpec, template *corev1.PodTemplateSpec, dgd *v1beta1.DynamoGraphDeployment, backend configv1alpha1.DiscoveryBackend) *v1beta1.DynamoComponentDeploymentSharedSpec { + // Copy shared defaults and this role, without unrelated compiler inputs. + seed := *source + seed.LPX = nil + seed.Roles = nil + seed.PodTemplate = template + role := seed.DeepCopy() + + // Bind source metadata on the independently owned role template. + propagateDGDAnnotations(dgd.Annotations, role) + role.PodTemplate.Labels[commonconsts.KubeLabelDynamoNamespace] = GetDynamoNamespace(dgd, source) + if backend != "" { + role.PodTemplate.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend] = string(backend) + } + return role +} + +// imageEntrypointComponentDefaults preserves all role-specific defaults while +// leaving the executable to the workload image unless the user supplied one. +type imageEntrypointComponentDefaults struct { + ComponentDefaults +} + +func (d *imageEntrypointComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { + container, err := d.ComponentDefaults.GetBaseContainer(context) + if err != nil { + return corev1.Container{}, err + } + container.Command = nil + return container, nil +} + +// selectedCyborgComponentDefaults installs LPX-owned Cyborg bindings before +// the user's PodTemplate is merged, preserving the normal merge strategy. +type selectedCyborgComponentDefaults struct { + ComponentDefaults + workload *dynamolpx.SelectedWorkload + dgdName string + replicas int32 + lpxPodSpec corev1.PodSpec +} + +func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { + container, err := d.ComponentDefaults.GetBaseContainer(context) + if err != nil { + return corev1.Container{}, err + } + container.Command = nil + if err := dynamolpx.ApplySelectedCyborgContainerDefaults( + &container, + d.workload, + d.dgdName, + d.replicas, + d.lpxPodSpec, + ); err != nil { + return corev1.Container{}, err + } + return container, nil +} + +func (d *selectedCyborgComponentDefaults) GetBasePodSpec(context ComponentContext) (corev1.PodSpec, error) { + podSpec, err := d.ComponentDefaults.GetBasePodSpec(context) + if err != nil { + return corev1.PodSpec{}, err + } + dynamolpx.ApplySelectedCyborgPodDefaults(&podSpec, d.dgdName) + return podSpec, nil +} + +// renderSelectedLPXRole consumes a private component copy; other inputs are read-only. +// alphaComponent may be nil; topologyDomains may be nil when no topology is consumed. +func renderSelectedLPXRole( + component *v1beta1.DynamoComponentDeploymentSharedSpec, + dgd *v1beta1.DynamoGraphDeployment, + alphaComponent *v1alpha1.DynamoComponentDeploymentSharedSpec, + operatorConfig *configv1alpha1.OperatorConfiguration, + secretsRetriever SecretsRetriever, + discoveryContext DiscoveryContext, + selectedWorkload *dynamolpx.SelectedWorkload, + defaults ComponentDefaults, + topologyDomains []v1beta1.TopologyDomain, +) (*corev1.PodTemplateSpec, error) { + componentName := component.ComponentName + // Capture authored precedence before PodSpec defaults fill the role's metadata. + metadata := generatePodMetadata(component, dgd, alphaComponent, componentName, discoveryContext) + applyDGDTemplateDefaults(component, dgd, topologyDomains) + basePodSpec, err := generateBasePodSpecWithDefaults( + component, + BackendFrameworkNoop, + secretsRetriever, + dgd.Name, + dgd.Namespace, + RoleMain, + 1, + operatorConfig, + commonconsts.MultinodeDeploymentTypeGrove, + componentName, + nil, + defaults, + func() (int64, error) { return 0, nil }, + ) + if err != nil { + return nil, err + } + if component.IsLPX() && selectedWorkload.BuildFamily() == dynamolpx.BuildFamilyXT { + explicitVolumes := component.PodTemplate.Spec.Volumes + runtimeVolumes := []string{"config"} + if selectedWorkload.Pipeline() != dynamolpx.PipelineLPX { + runtimeVolumes = append(runtimeVolumes, "tmp", "hugepages", "host-dev", "host-sys") + } + basePodSpec.Volumes = slices.DeleteFunc(basePodSpec.Volumes, func(volume corev1.Volume) bool { + return slices.Contains(runtimeVolumes, volume.Name) && !slices.ContainsFunc(explicitVolumes, + func(explicit corev1.Volume) bool { return explicit.Name == volume.Name }) + }) + } + for _, annotationKey := range commonconsts.KubeTopologySourceAnnotationKeys() { + delete(metadata.Annotations, annotationKey) + } + return &corev1.PodTemplateSpec{ + ObjectMeta: metadata, + Spec: *basePodSpec, + }, nil +} diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go new file mode 100644 index 000000000000..ad2e865622e0 --- /dev/null +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamo + +import ( + "os" + "strings" + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + "sigs.k8s.io/yaml" +) + +func TestBackendDetectionSkipsNonWorkers(t *testing.T) { + for _, componentType := range []string{commonconsts.ComponentTypeLPX, commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypePlanner} { + t.Run(componentType, func(t *testing.T) { + t.Log("Ignore ambiguous GPU commands and explicit backends for non-workers") + backend, err := determineBackendFramework(componentType, nil, + []string{"python -m dynamo.vllm; python -m dynamo.sglang"}, "trtllm") + require.NoError(t, err) + require.Equal(t, BackendFrameworkNoop, backend) + }) + } +} + +func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "chat-uid"}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "engine", ComponentType: v1beta1.ComponentTypeLPX, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLeader}, {Name: v1beta1.ComponentRoleWorker}}}, + }}, + } + name := PCSNameForLPX(source) + t.Log("An ordinary DGD named chat-lpx must not collide with the LPX PCS of chat") + ordinary := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat-lpx"}} + require.NotEqual(t, PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), name) + require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), name) + + for _, test := range []struct { + name string + mutate func(*v1beta1.DynamoGraphDeployment) + }{ + {"replacement source", func(d *v1beta1.DynamoGraphDeployment) { d.UID = "replacement-chat-uid" }}, + {"different namespace", func(d *v1beta1.DynamoGraphDeployment) { d.Namespace = "another-namespace" }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := source.DeepCopy() + test.mutate(changed) + require.NotEqual(t, name, PCSNameForLPX(changed)) + }) + } + t.Log("Replica, generation, and revision bookkeeping must not rename the workload") + source.Generation++ + source.ResourceVersion = "2" + source.Spec.Components[0].Replicas = ptr.To(int32(3)) + require.Equal(t, name, PCSNameForLPX(source)) + t.Log("A draft name cannot change the target-owned resource name budget") + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: strings.Repeat("d", 30), ComponentType: v1beta1.ComponentTypeLPX, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleWorker}}, + }) + require.Equal(t, name, PCSNameForLPX(source)) + + source.Name = "chat.example" + dotted := PCSNameForLPX(source) + source.Name = "chat-example" + require.NotEqual(t, dotted, PCSNameForLPX(source), "normalizing dots must not erase source identity") + + t.Log("Both readable and compact names fit the complete Grove name budget") + for componentLength := 1; componentLength <= 13; componentLength++ { + source.Spec.Components[0].ComponentName = strings.Repeat("e", componentLength) + for _, sourceName := range []string{"c", "chat.example", strings.Repeat("long.", 40) + "chat"} { + source.Name = sourceName + got := PCSNameForLPX(source) + require.Empty(t, validation.IsDNS1123Label(got), got) + require.LessOrEqual(t, len(got)+max(componentLength, 8)+componentLength+len("-engine-gpu"), commonconsts.MaxCombinedGroveResourceNameLength) + require.Equal(t, got, PCSNameForLPX(source)) + } + } +} + +func TestLPXInputRevision(t *testing.T) { + const updatedMetadata = "updated" + t.Log("Create a DGD with independent prefill and LPX engine capacity") + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "hybrid", Namespace: "test", UID: "source", Generation: 1}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, + {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeLPX, Replicas: ptr.To(int32(2)), LPX: &v1beta1.LPXConfig{BuildID: "hybrid-build"}, + Roles: []v1beta1.ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{}}, + {Name: v1beta1.ComponentRoleLeader, PodTemplate: &corev1.PodTemplateSpec{}}, + }}, + }}, + } + want, err := LPXInputRevision(source, "") + require.NoError(t, err) + require.Regexp(t, `^sha256:[a-f0-9]{64}$`, want) + pcsName := PCSNameForLPX(source) + require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), pcsName) + + t.Log("Prefill-only edits and DGD bookkeeping do not alter the LPX revision or PCS name") + source.Generation++ + source.ResourceVersion = "2" + source.Labels = map[string]string{"unrelated": "metadata"} + source.Annotations = map[string]string{"unrelated": "bookkeeping"} + source.Spec.Components[0].ComponentName = "a-long-independent-prefill" + source.Spec.Components[0].Replicas = ptr.To(int32(5)) + source.Spec.Restart = &v1beta1.Restart{ID: "not-yet-selected"} + got, err := LPXInputRevision(source, "") + require.NoError(t, err) + require.Equal(t, want, got) + require.Equal(t, pcsName, PCSNameForLPX(source)) + + t.Log("Each LPX component or shared-input change invalidates the revision; ordinary edits do not") + source.Spec.Scheduling = &v1beta1.SchedulingSpec{} + want, err = LPXInputRevision(source, "") + require.NoError(t, err) + for _, test := range []struct { + name string + wantChange bool + mutate func(*v1beta1.DynamoGraphDeployment) + }{ + {"component/name", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).ComponentName = "new-engine" }}, + {"component/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).Replicas = ptr.To(int32(3)) }}, + {"component/min-available", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).MinAvailable = ptr.To(int32(1)) }}, + {"component/namespace", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).GlobalDynamoNamespace = true }}, + {"component/runtime-version", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).RuntimeVersionOverride = "1.5.0" }}, + {"component/merge-strategy", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ExtraPodSpecMergeStrategy = v1beta1.ExtraPodSpecMergeStrategyOverride + }}, + {"component/shared-memory", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).SharedMemorySize = ptr.To(resource.MustParse("16Gi")) + }}, + {"component/model", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ModelRef = &v1beta1.ModelReference{Name: "model", Revision: "2"} + }}, + {"component/compilation-cache", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).CompilationCache = &v1beta1.CompilationCacheConfig{PVCName: "cache"} + }}, + {"component/build", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).LPX.BuildID = "next-build" }}, + {"component/settings", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":false}`)} + }}, + {"agent/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).Replicas = ptr.To(int32(4)) + }}, + {"conductor/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + }}, + {"agent/image", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "agent:next"}} + }}, + {"conductor/image", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "conductor:next"}} + }}, + {"agent/placement", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeSelector = map[string]string{"lpu": "new"} + }}, + {"conductor/placement", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec.NodeSelector = map[string]string{"gpu": "new"} + }}, + {"agent/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Labels = map[string]string{"role": "new"} + }}, + {"conductor/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Annotations = map[string]string{"role": "new"} + }}, + {"scheduling/deadline", true, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To(int64(60)) + }}, + {"shared/labels", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Labels = map[string]string{"workload": "new"} }}, + {"shared/annotations", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Annotations = map[string]string{"workload": "new"} }}, + {"shared/environment", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Env = []corev1.EnvVar{{Name: "SHARED", Value: "new"}} }}, + {"shared/priority", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.PriorityClassName = "inference" }}, + {"shared/backend", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.BackendFramework = "vllm" }}, + {"shared/kv-transfer", true, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Experimental = &v1beta1.DynamoGraphDeploymentExperimentalSpec{KvTransferPolicy: &v1beta1.KvTransferPolicy{LabelKey: "topology.kubernetes.io/zone", Domain: "zone"}} + }}, + {"shared/topology", true, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "cluster", PackDomain: "rack"} + }}, + {"shared/provider-override", true, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.ProviderOverride = &v1beta1.ProviderOverride{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueSet", Value: apiextensionsv1.JSON{Raw: []byte(`{"spec":{"template":{"topologyConstraint":{"packDomain":"rack"}}}}`)}} + }}, + {"restart/selected-lpx", true, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Restart = &v1beta1.Restart{ID: "restart-selected"} + d.Status.Restart = &v1beta1.RestartStatus{ObservedID: d.Spec.Restart.ID, Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{"decode"}} + }}, + {"ignored/ordinary-replicas", false, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Components[0].Replicas = ptr.To(int32(9)) }}, + {"ignored/ordinary-image", false, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Components[0].PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "prefill:next"}}}} + }}, + {"ignored/ordinary-restart", false, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Restart = &v1beta1.Restart{ID: "restart-ordinary"} + d.Status.Restart = &v1beta1.RestartStatus{ObservedID: d.Spec.Restart.ID, Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{d.Spec.Components[0].ComponentName}} + }}, + {"ignored/unselected-restart", false, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Restart = &v1beta1.Restart{ID: "not-delivered"} }}, + {"ignored/generation", false, func(d *v1beta1.DynamoGraphDeployment) { d.Generation++ }}, + {"ignored/resource-version", false, func(d *v1beta1.DynamoGraphDeployment) { d.ResourceVersion = "999" }}, + {"ignored/status", false, func(d *v1beta1.DynamoGraphDeployment) { d.Status.ObservedGeneration = d.Generation }}, + {"ignored/labels", false, func(d *v1beta1.DynamoGraphDeployment) { d.Labels["unrelated"] = updatedMetadata }}, + {"ignored/annotations", false, func(d *v1beta1.DynamoGraphDeployment) { d.Annotations["unrelated"] = updatedMetadata }}, + {"ignored/component-order", false, func(d *v1beta1.DynamoGraphDeployment) { + d.Spec.Components[0], d.Spec.Components[1] = d.Spec.Components[1], d.Spec.Components[0] + }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := source.DeepCopy() + test.mutate(changed) + got, err := LPXInputRevision(changed, LPXRestartToken(changed, "")) + require.NoError(t, err) + require.Equal(t, test.wantChange, got != want) + }) + } + + t.Log("Every consumed metadata key is an input, including both queue APIs and scheduler inheritance") + for _, key := range []string{ + commonconsts.KubeAnnotationEnableMetrics, commonconsts.KubeAnnotationDynamoDiscoveryBackend, + commonconsts.KubeAnnotationDynamoKubeDiscoveryMode, commonconsts.KubeAnnotationDynamoOperatorOriginVersion, + commonconsts.KubeAnnotationVLLMDistributedExecutorBackend, commonconsts.KubeAnnotationLPXSchedulerBackend, + commonconsts.KubeAnnotationWorkloadProvider, commonconsts.KubeAnnotationGroveUpdateStrategy, + commonconsts.KubeAnnotationKaiSchedulerQueue, commonconsts.KubeAnnotationVolcanoQueue, + "kai.scheduler/topology", "priorityClassName", "project", "user", + } { + t.Run("metadata/"+key, func(t *testing.T) { + changed := source.DeepCopy() + changed.Annotations[key] = updatedMetadata + got, err := LPXInputRevision(changed, "") + require.NoError(t, err) + require.NotEqual(t, want, got) + }) + } + for _, key := range []string{"kai.scheduler/preemptibility", "priorityClassName", "project", "user"} { + t.Run("scheduler-label/"+key, func(t *testing.T) { + changed := source.DeepCopy() + changed.Labels[key] = updatedMetadata + got, err := LPXInputRevision(changed, "") + require.NoError(t, err) + require.NotEqual(t, want, got) + }) + } + + t.Log("Both members independently contribute to the shared LPX revision") + pair := source.DeepCopy() + target := lpx.ServingComponent(pair) + target.Replicas = ptr.To(int32(1)) + draft := target.DeepCopy() + draft.ComponentName = "small-model" + draft.Roles = []v1beta1.ComponentRoleSpec{*draft.ComponentRole(v1beta1.ComponentRoleWorker)} + pair.Spec.Components = append(pair.Spec.Components, *draft) + pairRevision, err := LPXInputRevision(pair, "") + require.NoError(t, err) + for _, name := range []string{draft.ComponentName, target.ComponentName} { + changed := pair.DeepCopy() + changed.GetComponentByName(name).LPX.BuildID = "next-build" + after, err := LPXInputRevision(changed, "") + require.NoError(t, err) + require.NotEqual(t, pairRevision, after, name) + } + got, err = LPXInputRevision(source, "selected-restart") + require.NoError(t, err) + require.NotEqual(t, want, got) +} + +func TestLPXInputRevisionTracksIndirectRenderMetadata(t *testing.T) { + for _, change := range []string{"epp-presence", "alpha-label", "alpha-annotation", "alpha-subtype"} { + t.Run(change, func(t *testing.T) { + t.Log("Capture the GPU role metadata derived from the complete source graph") + payload, err := os.ReadFile("testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + component := lpx.ServingComponent(source) + role := lpxRoleComponent(component, component.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate, source, "") + role.ComponentType = v1beta1.ComponentTypeDecode + metadata := generatePodMetadata(role, source, getDGDAlphaComponent(source, component.ComponentName), component.ComponentName, DiscoveryContext{}) + before, err := LPXInputRevision(source, "") + require.NoError(t, err) + + t.Log("Change an indirect render input without changing the native LPX component") + changed := source.DeepCopy() + if change == "epp-presence" { + changed.Spec.Components = append(changed.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "epp", ComponentType: v1beta1.ComponentTypeEPP}) + } else { + alpha := &v1alpha1.DynamoGraphDeployment{} + require.NoError(t, alpha.ConvertFrom(source)) + legacy := alpha.Spec.Services[component.ComponentName] + switch change { + case "alpha-label": + legacy.Labels = map[string]string{"render-input": "new"} + case "alpha-annotation": + legacy.Annotations = map[string]string{"render-input": "new"} + case "alpha-subtype": + legacy.SubComponentType = "custom-lpx" + } + require.NoError(t, alpha.ConvertTo(changed)) + } + require.Equal(t, component, lpx.ServingComponent(changed)) + nextMetadata := generatePodMetadata(role, changed, getDGDAlphaComponent(changed, component.ComponentName), component.ComponentName, DiscoveryContext{}) + if change == "alpha-annotation" { + require.NotEqual(t, metadata.Annotations, nextMetadata.Annotations) + } else { + require.NotEqual(t, metadata.Labels, nextMetadata.Labels) + } + after, err := LPXInputRevision(changed, "") + require.NoError(t, err) + require.NotEqual(t, before, after, "different rendered metadata requires a different child revision") + }) + } +} + +func TestLPXRestartTokenPreservesDeliveredTokenOnCancellation(t *testing.T) { + t.Log("A cancelled sequential restart must not select an LPX component that never started") + source := &v1beta1.DynamoGraphDeployment{ + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend}, + {ComponentName: "lpx", ComponentType: v1beta1.ComponentTypeLPX}, + }}, + Status: v1beta1.DynamoGraphDeploymentStatus{Restart: &v1beta1.RestartStatus{ + ObservedID: "cancelled-restart", Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{"frontend"}, + }}, + } + for _, request := range []*v1beta1.Restart{nil, {}} { + source.Spec.Restart = request + for _, delivered := range []string{"", "previously-delivered"} { + require.Equal(t, delivered, LPXRestartToken(source, delivered)) + } + } +} + +func TestLPXInputRevisionIgnoresUnrelatedConvertedMetadata(t *testing.T) { + t.Log("Keep EPP discovery enabled while editing only its independent workload") + payload, err := os.ReadFile("testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "epp", ComponentType: v1beta1.ComponentTypeEPP, Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "epp:before"}}}}, + }) + before, err := LPXInputRevision(source, "") + require.NoError(t, err) + source.Spec.Components[1].Replicas = ptr.To(int32(3)) + source.Spec.Components[1].PodTemplate.Spec.Containers[0].Image = "epp:after" + after, err := LPXInputRevision(source, "") + require.NoError(t, err) + require.Equal(t, before, after) + + t.Log("Ignore conversion bookkeeping and metadata on the ordinary component") + alpha := &v1alpha1.DynamoGraphDeployment{} + require.NoError(t, alpha.ConvertFrom(source)) + alpha.Spec.Services["epp"].Labels = map[string]string{"ordinary-label": "changed"} + alpha.Spec.Services["epp"].Annotations = map[string]string{"ordinary-annotation": "changed"} + alpha.Spec.Services["epp"].SubComponentType = "ordinary-subtype" + converted := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, alpha.ConvertTo(converted)) + require.NotEqual(t, source.Annotations, converted.Annotations) + after, err = LPXInputRevision(converted, "") + require.NoError(t, err) + require.Equal(t, before, after) +} diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index f2213f4252aa..cb9288507c5e 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -844,7 +844,9 @@ func TestAppendMissingPVCVolumesForMountsAddsMissingPVCs(t *testing.T) { } mounts := []corev1.VolumeMount{ {Name: "cache", MountPath: "/cache"}, + {Name: "cache", MountPath: "/cache-copy"}, {Name: "model-cache", MountPath: "/models"}, + {Name: "model-cache", MountPath: "/models-copy"}, } got := appendMissingPVCVolumesForMounts(volumes, mounts) @@ -1244,7 +1246,7 @@ func TestGenerateGrovePodCliqueSet_ProjectsClusterTopologyDomainsToWorkerCliques assert.False(t, hasTopologyLabelVolume(cliques["frontend"].Spec.PodSpec.Volumes)) } -func TestGenerateLabelsAndAnnotations_UsePreservedAlphaDGDServiceMetadata(t *testing.T) { +func TestGeneratePodMetadata_UsePreservedAlphaDGDServiceMetadata(t *testing.T) { alpha := &v1alpha1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd", @@ -1274,16 +1276,13 @@ func TestGenerateLabelsAndAnnotations_UsePreservedAlphaDGDServiceMetadata(t *tes component := service ensurePodTemplate(component).Annotations["pod-template-annotation"] = "from-pod-template" - labels, err := generateLabels(component, beta, "worker", DiscoveryContext{}) - require.NoError(t, err) - assert.Equal(t, "kept", labels["legacy-label"]) - assert.Equal(t, "legacy-sub", labels[commonconsts.KubeLabelDynamoSubComponentType]) + metadata := generatePodMetadata(component, beta, getDGDAlphaComponent(beta, "worker"), "worker", DiscoveryContext{}) + assert.Equal(t, "kept", metadata.Labels["legacy-label"]) + assert.Equal(t, "legacy-sub", metadata.Labels[commonconsts.KubeLabelDynamoSubComponentType]) - annotations, err := generateAnnotations(component, beta, "worker") - require.NoError(t, err) - assert.Equal(t, "from-dgd", annotations["dgd-annotation"]) - assert.Equal(t, "kept", annotations["legacy-annotation"]) - assert.Equal(t, "from-pod-template", annotations["pod-template-annotation"]) + assert.Equal(t, "from-dgd", metadata.Annotations["dgd-annotation"]) + assert.Equal(t, "kept", metadata.Annotations["legacy-annotation"]) + assert.Equal(t, "from-pod-template", metadata.Annotations["pod-template-annotation"]) } // TestGenerateComponentContext tests the generateComponentContext function @@ -1996,6 +1995,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, + Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, @@ -2041,7 +2041,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, ImagePullSecrets: []corev1.LocalObjectReference{ { @@ -2236,7 +2237,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { ServiceAccountName: commonconsts.PlannerServiceAccountName, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, RestartPolicy: corev1.RestartPolicyAlways, @@ -2608,6 +2610,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, + Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, @@ -2655,7 +2658,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { RestartPolicy: corev1.RestartPolicyAlways, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -2869,7 +2873,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { RestartPolicy: corev1.RestartPolicyAlways, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -3055,7 +3060,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -3218,7 +3224,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), ServiceAccountName: commonconsts.PlannerServiceAccountName, SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, RestartPolicy: corev1.RestartPolicyAlways, Volumes: []corev1.Volume{ @@ -3631,6 +3638,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, + Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, @@ -3687,7 +3695,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -3877,7 +3886,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { PodSpec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -4064,7 +4074,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -4227,7 +4238,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), ServiceAccountName: commonconsts.PlannerServiceAccountName, SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -6292,6 +6304,40 @@ func TestGenerateBasePodSpec_InitContainerPullSecrets(t *testing.T) { } } +func TestGenerateBasePodSpec_EmptyMainContainerPortsOverrideWorkerDefaults(t *testing.T) { + podSpec, err := GenerateBasePodSpec( + &v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "worker", + ComponentType: v1beta1.ComponentTypeWorker, + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: commonconsts.MainContainerName, + Ports: []corev1.ContainerPort{}, + }, + }, + }, + }, + }, + BackendFrameworkVLLM, + &mockSecretsRetriever{}, + "test-deployment", + "default", + RoleMain, + 1, + &configv1alpha1.OperatorConfiguration{}, + commonconsts.MultinodeDeploymentTypeGrove, + "worker", + nil, + staticContainerGPUCount(0), + ) + + require.NoError(t, err) + require.NotEmpty(t, podSpec.Containers) + require.Empty(t, podSpec.Containers[0].Ports) +} + func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) { tests := []struct { name string @@ -6509,6 +6555,7 @@ func TestGenerateBasePodSpec_DiscoverBackend(t *testing.T) { nil, // Use default deployer staticContainerGPUCount(0), // No GPUs needed by this test ) + if !assert.NoError(t, err) { return } @@ -6649,7 +6696,8 @@ func TestGenerateBasePodSpec_Worker(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ // Only fsGroup is injected by default for volume permissions - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, Volumes: []corev1.Volume{ { @@ -6824,6 +6872,7 @@ func TestGenerateBasePodSpec_GPUMemoryServiceExtraClientContainers(t *testing.T) nil, staticContainerGPUCount(0), ) + require.NoError(t, err) t.Log("Verify every requested container is wired as a GMS client") @@ -7185,6 +7234,7 @@ func TestGenerateBasePodSpec_TRTLLMSSHMountUsesSecretVolume(t *testing.T) { nil, staticContainerGPUCount(0), ) + require.NoError(t, err) var sshVolumes []corev1.Volume @@ -7678,6 +7728,7 @@ func TestGenerateBasePodSpec_ConvertedCompilationCacheMountIsNotDuplicated(t *te nil, staticContainerGPUCount(0), ) + require.NoError(t, err) require.NotEmpty(t, podSpec.Containers) @@ -7718,6 +7769,7 @@ func TestGenerateBasePodSpec_ConvertedCompilationCacheUsesDefaultMount(t *testin nil, staticContainerGPUCount(0), ) + require.NoError(t, err) require.NotEmpty(t, podSpec.Containers) assert.Contains(t, podSpec.Containers[0].VolumeMounts, corev1.VolumeMount{ @@ -7958,7 +8010,8 @@ func TestGenerateBasePodSpec_SecurityContext(t *testing.T) { ComponentType: commonconsts.ComponentTypeFrontend, }, expectedSecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), }, description: "Operator should only inject fsGroup for volume permissions, not UID/GID (backward compatible)", }, @@ -8663,8 +8716,8 @@ func TestGenerateGrovePodCliqueSet_RestartAnnotations(t *testing.T) { } } -func TestGenerateLabels_ReassertsRestoreIdentityLabelsAfterMetadataMerge(t *testing.T) { - labels, err := generateLabels( +func TestGeneratePodMetadata_ReassertsRestoreIdentityLabelsAfterMetadataMerge(t *testing.T) { + labels := generatePodMetadata( betaComponent(t, &v1alpha1.DynamoComponentDeploymentSharedSpec{ ComponentType: commonconsts.ComponentTypeWorker, DynamoNamespace: ptr.To("default-test-dgd"), @@ -8690,10 +8743,10 @@ func TestGenerateLabels_ReassertsRestoreIdentityLabelsAfterMetadataMerge(t *test betaDGD(t, &v1alpha1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "test-dgd", Namespace: "default"}, }), + nil, "Worker", DiscoveryContext{Backend: configv1alpha1.DiscoveryBackendKubernetes}, - ) - require.NoError(t, err) + ).Labels assert.Equal(t, "test-dgd-worker", labels[commonconsts.KubeLabelDynamoSelector]) assert.Equal(t, "Worker", labels[commonconsts.KubeLabelDynamoComponent]) assert.Equal(t, "default-test-dgd", labels[commonconsts.KubeLabelDynamoNamespace]) @@ -9147,7 +9200,7 @@ func TestGenerateGrovePodCliqueSet_SingleNodeForceScalingGroup(t *testing.T) { } got, err := GenerateGrovePodCliqueSet( - context.Background(), + t.Context(), beta, &configv1alpha1.OperatorConfiguration{}, &controller_common.RuntimeConfig{}, @@ -10883,6 +10936,25 @@ func TestPCSNameForDGD(t *testing.T) { Multinode: &v1beta1.MultinodeSpec{NodeCount: 2}, } } + lpxComponent := func(name string) v1beta1.DynamoComponentDeploymentSharedSpec { + return v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: name, + ComponentType: v1beta1.ComponentTypeLPX, + LPX: &v1beta1.LPXConfig{BuildID: "build-id"}, + } + } + interPodGMSComponent := func(name string) v1beta1.DynamoComponentDeploymentSharedSpec { + return v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: name, + Experimental: &v1beta1.ExperimentalSpec{ + GPUMemoryService: &v1beta1.GPUMemoryServiceSpec{ + Mode: v1beta1.GMSModeInterPod, + }, + }, + } + } + multinodeGMSComponent := interPodGMSComponent("decode") + multinodeGMSComponent.Multinode = &v1beta1.MultinodeSpec{NodeCount: 11} tests := []struct { name string @@ -10892,13 +10964,15 @@ func TestPCSNameForDGD(t *testing.T) { wantLen int // 0 means check exact match via want; >0 means check length }{ { - name: "short name passes through unchanged", - dgdName: "trtllm-disagg", + name: "single-node inter-pod GMS budgets the GMS clique name", + dgdName: "deepseek-v32-fp4-trtllm-dgd1", components: []v1beta1.DynamoComponentDeploymentSharedSpec{ singleNodeComponent("prefill"), - singleNodeComponent("decode"), + interPodGMSComponent("decode"), }, - want: "trtllm-disagg", + // decode GMS: PCSG=6, PCLQ=len("decode-gms-0")=12 → budget=18, + // pcsBudget=45-18=27; dgdName is 28 chars → needs truncation to 27. + wantLen: 27, }, { name: "short name with multinode passes through unchanged", @@ -10911,18 +10985,27 @@ func TestPCSNameForDGD(t *testing.T) { }, { name: "long name gets truncated with hash", - dgdName: "deepseek-v32-fp4-trtllm-dgd", + dgdName: "deepseek-v32-fp4-trtllm-dgd1", components: []v1beta1.DynamoComponentDeploymentSharedSpec{ multinodeComponent("prefill"), - multinodeComponent("decode"), + multinodeGMSComponent, }, - // prefill multinode: PCSG=7, PCLQ=7+1+3=11 → budget=18, pcsBudget=45-18=27 - // dgdName is 28 chars → needs truncation to 27 - wantLen: 27, + // decode multinode GMS: PCSG=6, PCLQ=len("decode-wkr-10")=13 → budget=19, + // pcsBudget=45-19=26; dgdName is 28 chars → needs truncation to 26. + wantLen: 26, + }, + { + name: "normal PCS name does not budget LPX worker cliques", + dgdName: "abcdefghijklmnopqrstuvwxyzabcdef", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + lpxComponent("lpu"), + }, + // LPX has its own PCS name budget, so this name stays unchanged. + wantLen: 32, }, { name: "deterministic - same input always produces same output", - dgdName: "deepseek-v32-fp4-trtllm-dgd", + dgdName: "deepseek-v32-fp4-trtllm-dgd1", components: []v1beta1.DynamoComponentDeploymentSharedSpec{ multinodeComponent("prefill"), }, @@ -10939,6 +11022,14 @@ func TestPCSNameForDGD(t *testing.T) { // 7 < minPCSNameLength(8), so clamped to 8 wantLen: 8, }, + { + name: "normal PCS name is independent of LPX role expansion", + dgdName: "deepseek-v32-fp4-lpu-dgd", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + lpxComponent("lpu"), + }, + want: "deepseek-v32-fp4-lpu-dgd", + }, { name: "empty components - no truncation needed", dgdName: "my-dgd", @@ -10972,19 +11063,10 @@ func TestPCSNameForDGD(t *testing.T) { maxComponentBudget := 0 for i := range tt.components { component := &tt.components[i] - lowerName := strings.ToLower(component.ComponentName) - var budget int - if component.GetNumberOfNodes() > 1 || component.IsInterPodGMSEnabled() { - maxCliqueNameLen := 0 - for _, role := range expandRolesForComponent(component.ComponentName, component.Replicas, component.GetNumberOfNodes(), component) { - if cliqueNameLen := len(strings.ToLower(role.Name)); cliqueNameLen > maxCliqueNameLen { - maxCliqueNameLen = cliqueNameLen - } - } - budget = len(lowerName) + maxCliqueNameLen - } else { - budget = len(lowerName) + if component.IsLPX() { + continue } + budget := ComponentNameBudget(component) if budget > maxComponentBudget { maxComponentBudget = budget } diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index ed8bd424659a..99f0e700a54d 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -3,8 +3,10 @@ package dynamo import ( "context" "fmt" + "strconv" "strings" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/go-logr/logr" @@ -20,12 +22,15 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" ) -// legacyGroveConditionReasonInsufficientScheduledPCSGReplicas can remain on -// persisted PCSG status after upgrading from Grove versions that emitted it. -const legacyGroveConditionReasonInsufficientScheduledPCSGReplicas = "InsufficientScheduledPodCliqueScalingGroupReplicas" +const ( + legacyConditionReasonInsufficientScheduledPCSGReplicas = "InsufficientScheduledPodCliqueScalingGroupReplicas" + groveResourceNotFoundReason = "resource not found" + groveObservedGenerationNilReason = "observedGeneration is nil" +) type GroveMultinodeDeployer struct { MultinodeDeployer @@ -105,6 +110,7 @@ type GroveReadiness struct { // EvaluateGroveReadiness resolves one Grove readiness snapshot from the // supplied PCS observation. A nil PCS represents an observed missing PCS. +// The deployment must contain only components owned by this PCS. func EvaluateGroveReadiness( ctx context.Context, reader client.Reader, @@ -123,15 +129,122 @@ func EvaluateGroveReadiness( }, nil } -// evaluateGroveComponents is the single per-component evaluation loop behind -// all public Grove readiness views. +// ComponentNameBudget returns the maximum length of the given component once rendered +func ComponentNameBudget(component *v1beta1.DynamoComponentDeploymentSharedSpec) int { + componentName := component.ComponentName + if component.UsesPCSG() { + // Each role expansion's first and last names bound its generated clique names. + roles := expandRolesForComponent(componentName, component.Replicas, component.GetNumberOfNodes(), component) + return len(componentName) + max(len(roles[0].Name), len(roles[len(roles)-1].Name)) + } + + return len(componentName) +} + +func observeComponentPCSGReadiness( + ctx context.Context, + reader client.Reader, + pcs *grovev1alpha1.PodCliqueSet, + resourceName, + namespace, + component string, + dgdReplicas *int32, +) (groveComponentReadiness, error) { + logger := log.FromContext(ctx) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + if err := reader.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, pcsg); err != nil { + if errors.IsNotFound(err) { + status := v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{resourceName}, + } + return groveComponentReadiness{status: status}.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) + } + + // Reuse the native observation while retaining the component's requested capacity for cutover. + desiredReplicas := ptr.Deref(dgdReplicas, pcsg.Spec.Replicas) + componentReadiness := pcsgReadiness(pcsg, logger) + componentReadiness.revision.desiredReplicas = desiredReplicas + if !componentReadiness.revision.generationObserved { + return componentReadiness, nil + } + + if pcsg.Spec.Replicas != desiredReplicas { + return componentReadiness.withResult(false, fmt.Sprintf("desired replicas=%d, PodCliqueScalingGroup spec replicas=%d", desiredReplicas, pcsg.Spec.Replicas), v1beta1.DGDReadyReasonUpdating), nil + } + + acceptedPCSRevisionHash := getAcceptedPCSRevisionHash(pcs) + if acceptedPCSRevisionHash == nil { + return componentReadiness.withResult(false, "PodCliqueSet revision is not accepted", v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + if pcsg.Status.CurrentPodCliqueSetGenerationHash == nil || *pcsg.Status.CurrentPodCliqueSetGenerationHash != *acceptedPCSRevisionHash { + return componentReadiness.withResult(false, "PodCliqueScalingGroup has not applied the current PodCliqueSet generation", v1beta1.DGDReadyReasonUpdating), nil + } + + if desiredReplicas == 0 { + return componentReadiness.withResult(true, "", ""), nil + } + + podCliqueList := &grovev1alpha1.PodCliqueList{} + if err := reader.List( + ctx, + podCliqueList, + client.InNamespace(namespace), + client.MatchingLabels{grovecommon.LabelPodCliqueScalingGroup: resourceName, commonconsts.KubeLabelDynamoComponent: component}, + ); err != nil { + logger.V(1).Info("Failed to get PodCliques for PodCliqueScalingGroup", "error", err, "resourceName", resourceName) + return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliques for PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) + } + + if len(podCliqueList.Items) == 0 { + logger.V(2).Info("Failed to find PodCliques for PodCliqueScalingGroup", "resourceName", resourceName) + return componentReadiness.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + + podCliquesByReplica := make(map[int][]grovev1alpha1.PodClique) + + for _, clique := range podCliqueList.Items { + index, err := strconv.Atoi(clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex]) + if err != nil { + logger.V(1).Info("Failed to read PodClique metadata", "error", err, "resourceName", clique.Name) + return groveComponentReadiness{}, fmt.Errorf("failed to read PodClique metadata %s/%s: %w", namespace, clique.Name, err) + } + podCliquesByReplica[index] = append(podCliquesByReplica[index], clique) + } + + for replica := int32(0); replica < desiredReplicas; replica++ { + cliques := podCliquesByReplica[int(replica)] + if len(cliques) == 0 { + return componentReadiness.withResult(false, fmt.Sprintf("%s: %s", resourceName, v1beta1.DGDReadyReasonSomeResourcesNotReady), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + + for i := range cliques { + cliqueReadiness := podCliqueReadiness(&cliques[i], logger) + if !cliqueReadiness.ready { + return componentReadiness.withResult(false, fmt.Sprintf("%s: %s", resourceName, cliqueReadiness.reason), cliqueReadiness.classification), nil + } + } + } + + return componentReadiness, nil +} + +// evaluateGroveComponents is the single per-component evaluation loop shared +// by all public Grove readiness views. // -// Each Check*Ready call returns the DGD-level Ready reason its component would +// Each component evaluator returns the DGD-level Ready reason its component would // imply (a v1beta1.DGDReadyReason* value) when that component is not ready. The // reasons are aggregated in place: if every not-ready component implies the // same reason, that reason is used; if they disagree, the result is // MixedNotReadyReasons. -func evaluateGroveComponents(ctx context.Context, reader client.Reader, dgd *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet) (allReady bool, classificationReason string, message string, componentStatuses map[string]v1beta1.ComponentReplicaStatus, err error) { +func evaluateGroveComponents( + ctx context.Context, + reader client.Reader, + dgd *v1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, +) (allReady bool, classificationReason string, message string, componentStatuses map[string]v1beta1.ComponentReplicaStatus, err error) { logger := log.FromContext(ctx) var notReadyComponents []string aggregatedReason := "" @@ -140,20 +253,28 @@ func evaluateGroveComponents(ctx context.Context, reader client.Reader, dgd *v1b for i := range dgd.Spec.Components { component := &dgd.Spec.Components[i] componentName := component.ComponentName - resourceName := GroveComponentResourceName(dgd, componentName) var componentReadiness groveComponentReadiness var checkErr error + resourceName := GroveComponentResourceName(dgd, componentName) if component.UsesPCSG() { - componentReadiness, checkErr = observePCSGReadiness(ctx, reader, resourceName, dgd.Namespace, logger) + componentReadiness, checkErr = observeComponentPCSGReadiness( + ctx, + reader, + pcs, + resourceName, + dgd.Namespace, + componentName, + component.Replicas, + ) } else { - componentReadiness, checkErr = observePodCliqueReadiness(ctx, reader, resourceName, dgd.Namespace, logger) + componentReadiness, checkErr = observePodCliqueReadiness(ctx, reader, resourceName, dgd.Namespace, component.Replicas, logger) } // A non-NotFound read error is a transient failure to determine // readiness. Propagate it (rather than folding it into a not-ready // result) so the reconcile retries with backoff and does not advance - // ObservedGeneration on a blip. NotFound is handled inside observe* as a - // legitimate not-ready state and never surfaces here. + // ObservedGeneration on a blip. NotFound is handled inside the observers + // as a legitimate not-ready state and never surfaces here. if checkErr != nil { return false, "", "", nil, fmt.Errorf("component %q: %w", componentName, checkErr) } @@ -182,6 +303,7 @@ func evaluateGroveComponents(ctx context.Context, reader client.Reader, dgd *v1b componentReadiness := componentReadinesses[component.ComponentName] componentStatus := componentReadiness.status componentStatus.RuntimeNamespace = namespacePlan.runtimeNamespace(dgd, component) + componentStatus.Ready = componentReadiness.ready componentStatuses[component.ComponentName] = componentStatus } @@ -397,11 +519,12 @@ func grovePodCliqueSetCliqueForComponent(pcs *grovev1alpha1.PodCliqueSet, compon // are ready, or SomeResourcesNotReady when the cause cannot be determined. It // is empty when the component is ready. func CheckPodCliqueReady(ctx context.Context, reader client.Reader, resourceName, namespace string, logger logr.Logger) (bool, string, v1beta1.ComponentReplicaStatus, string, error) { - componentReadiness, err := observePodCliqueReadiness(ctx, reader, resourceName, namespace, logger) + componentReadiness, err := observePodCliqueReadiness(ctx, reader, resourceName, namespace, nil, logger) return componentReadiness.ready, componentReadiness.reason, componentReadiness.status, componentReadiness.classification, err } -func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resourceName, namespace string, logger logr.Logger) (groveComponentReadiness, error) { +// A nil expectedReplicas leaves the desired capacity under the PodClique's control. +func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resourceName, namespace string, expectedReplicas *int32, logger logr.Logger) (groveComponentReadiness, error) { podClique := &grovev1alpha1.PodClique{} err := reader.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, podClique) if err != nil { @@ -414,7 +537,7 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{resourceName}, } - return groveComponentReadiness{status: serviceStatus}.withResult(false, "resource not found", v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + return groveComponentReadiness{status: serviceStatus}.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil } // A non-NotFound error is a transient failure to determine readiness, // not a legitimate not-ready state. Return it so the reconcile retries @@ -422,7 +545,18 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour logger.V(1).Info("Failed to get PodClique", "error", err, "resourceName", resourceName) return groveComponentReadiness{}, fmt.Errorf("failed to get PodClique %s/%s: %w", namespace, resourceName, err) } + // The cache must observe the requested scale before readiness or namespace cutover. + componentReadiness := podCliqueReadiness(podClique, logger) + if expectedReplicas != nil && podClique.Spec.Replicas != *expectedReplicas { + componentReadiness.revision.desiredReplicas = *expectedReplicas + componentReadiness.revision.generationObserved = false + return componentReadiness.withResult(false, fmt.Sprintf("desired replicas=%d, PodClique spec replicas=%d", *expectedReplicas, podClique.Spec.Replicas), v1beta1.DGDReadyReasonUpdating), nil + } + return componentReadiness, nil +} +func podCliqueReadiness(podClique *grovev1alpha1.PodClique, logger logr.Logger) groveComponentReadiness { + resourceName := podClique.Name desiredReplicas := podClique.Spec.Replicas readyReplicas := podClique.Status.ReadyReplicas updatedReplicas := podClique.Status.UpdatedReplicas @@ -448,7 +582,7 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour status: v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{resourceName}, - Replicas: podClique.Status.Replicas, + Replicas: replicas, UpdatedReplicas: podClique.Status.UpdatedReplicas, ReadyReplicas: &readyReplicas, }, @@ -466,25 +600,25 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour if observedGeneration == nil { logger.V(1).Info("PodClique observedGeneration is nil", "resourceName", resourceName) - return componentReadiness.withResult(false, "observedGeneration is nil", v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + return componentReadiness.withResult(false, groveObservedGenerationNilReason, v1beta1.DGDReadyReasonSomeResourcesNotReady) } if *observedGeneration < generation { logger.V(1).Info("PodClique spec not yet processed", "resourceName", resourceName, "generation", generation, "observedGeneration", observedGeneration) - return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady) } componentReadiness.status.ScheduledReplicas = &scheduledReplicas if desiredReplicas == 0 { - return componentReadiness.withResult(true, "", ""), nil + return componentReadiness.withResult(true, "", "") } // Fully ready: replicas exist, are updated, and are ready. Checked first so // a healthy component is never mis-diagnosed as InsufficientCapacity when // Grove does not populate scheduledReplicas on a ready PodClique. if replicas == desiredReplicas && updatedReplicas == desiredReplicas && readyReplicas == desiredReplicas { - return componentReadiness.withResult(true, "", ""), nil + return componentReadiness.withResult(true, "", "") } // Not ready: classify capacity signals, in order of reliability: @@ -493,32 +627,32 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour // 3. 0 < scheduledReplicas < desired (genuine partial scheduling) if scheduleGatedReplicas > 0 { logger.V(1).Info("PodClique has schedule-gated replicas", "resourceName", resourceName, "scheduleGated", scheduleGatedReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("schedule-gated replicas: %d", scheduleGatedReplicas), v1beta1.DGDReadyReasonInsufficientCapacity), nil + return componentReadiness.withResult(false, fmt.Sprintf("schedule-gated replicas: %d", scheduleGatedReplicas), v1beta1.DGDReadyReasonInsufficientCapacity) } if cond := meta.FindStatusCondition(podClique.Status.Conditions, groveconstants.ConditionTypePodCliqueScheduled); cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == groveconstants.ConditionReasonInsufficientScheduledPods { logger.V(1).Info("PodClique scheduling condition reports insufficient capacity", "resourceName", resourceName, "reason", cond.Reason, "message", cond.Message) - return componentReadiness.withResult(false, fmt.Sprintf("scheduling condition %s: %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity), nil + return componentReadiness.withResult(false, fmt.Sprintf("scheduling condition %s: %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity) } if scheduledReplicas > 0 && scheduledReplicas < desiredReplicas { logger.V(1).Info("PodClique partially scheduled", "resourceName", resourceName, "desired", desiredReplicas, "scheduled", scheduledReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity), nil + return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity) } if desiredReplicas != updatedReplicas { logger.V(1).Info("PodClique not fully updated", "resourceName", resourceName, "desired", desiredReplicas, "updated", updatedReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating), nil + return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating) } if replicas != desiredReplicas { logger.V(1).Info("PodClique performing rolling update", "resourceName", resourceName, "desired", desiredReplicas, "replicas", replicas) - return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating), nil + return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating) } // Scheduled and rolled out, but not enough ready replicas. logger.V(1).Info("PodClique not ready", "resourceName", resourceName, "desired", desiredReplicas, "ready", readyReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("scheduled but ready=%d/%d", readyReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady), nil + return componentReadiness.withResult(false, fmt.Sprintf("scheduled but ready=%d/%d", readyReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady) } // CheckPCSGReady determines if a Grove PodCliqueScalingGroup is fully ready and available. @@ -551,7 +685,12 @@ func observePCSGReadiness(ctx context.Context, reader client.Reader, resourceNam logger.V(1).Info("Failed to get PodCliqueScalingGroup", "error", err, "resourceName", resourceName) return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) } + return pcsgReadiness(pcsg, logger), nil +} +// pcsgReadiness observes a nonnil native scaling group without mutating it. +func pcsgReadiness(pcsg *grovev1alpha1.PodCliqueScalingGroup, logger logr.Logger) groveComponentReadiness { + resourceName := pcsg.Name desiredReplicas := pcsg.Spec.Replicas availableReplicas := pcsg.Status.AvailableReplicas updatedReplicas := pcsg.Status.UpdatedReplicas @@ -593,26 +732,26 @@ func observePCSGReadiness(ctx context.Context, reader client.Reader, resourceNam if observedGeneration == nil { logger.V(1).Info("PodCliqueScalingGroup observedGeneration is nil", "resourceName", resourceName) - return componentReadiness.withResult(false, "observedGeneration is nil", v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + return componentReadiness.withResult(false, groveObservedGenerationNilReason, v1beta1.DGDReadyReasonSomeResourcesNotReady) } if *observedGeneration < generation { logger.V(1).Info("PodCliqueScalingGroup spec not yet processed", "resourceName", resourceName, "generation", generation, "observedGeneration", observedGeneration) - return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady) } componentReadiness.status.ScheduledReplicas = &scheduledReplicas if desiredReplicas == 0 { // No replicas desired, so it's ready. - return componentReadiness.withResult(true, "", ""), nil + return componentReadiness.withResult(true, "", "") } // Fully ready: replicas exist, are updated, and are available. Checked // first so a healthy PCSG is never mis-diagnosed as InsufficientCapacity // when Grove does not populate scheduledReplicas on a ready group. if replicas == desiredReplicas && updatedReplicas == desiredReplicas && availableReplicas == desiredReplicas { - return componentReadiness.withResult(true, "", ""), nil + return componentReadiness.withResult(true, "", "") } // Not ready: the explicit MinAvailableBreached scheduling condition, @@ -624,28 +763,29 @@ func observePCSGReadiness(ctx context.Context, reader client.Reader, resourceNam // *availability* reason (InsufficientAvailablePodCliqueScalingGroupReplicas). if cond := meta.FindStatusCondition(pcsg.Status.Conditions, groveconstants.ConditionTypeMinAvailableBreached); cond != nil && cond.Status == metav1.ConditionFalse && - cond.Reason == legacyGroveConditionReasonInsufficientScheduledPCSGReplicas { + (cond.Reason == groveconstants.ConditionReasonScheduledReplicasBelowMinAvailable || + cond.Reason == legacyConditionReasonInsufficientScheduledPCSGReplicas) { logger.V(1).Info("PodCliqueScalingGroup MinAvailableBreached reports insufficient capacity", "resourceName", resourceName, "reason", cond.Reason, "message", cond.Message) - return componentReadiness.withResult(false, fmt.Sprintf("min-available breached (%s): %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity), nil + return componentReadiness.withResult(false, fmt.Sprintf("min-available breached (%s): %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity) } if scheduledReplicas > 0 && scheduledReplicas < desiredReplicas { logger.V(1).Info("PodCliqueScalingGroup partially scheduled", "resourceName", resourceName, "desired", desiredReplicas, "scheduled", scheduledReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity), nil + return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity) } if desiredReplicas != updatedReplicas { logger.V(1).Info("PodCliqueScalingGroup not fully updated", "resourceName", resourceName, "desired", desiredReplicas, "updated", updatedReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating), nil + return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating) } if replicas != desiredReplicas { logger.V(1).Info("PodCliqueScalingGroup performing rolling update", "resourceName", resourceName, "desired", desiredReplicas, "replicas", replicas) - return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating), nil + return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating) } // Scheduled and rolled out, but not enough available replicas. logger.V(1).Info("PodCliqueScalingGroup not ready", "resourceName", resourceName, "desired", desiredReplicas, "available", availableReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("scheduled but available=%d/%d", availableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady), nil + return componentReadiness.withResult(false, fmt.Sprintf("scheduled but available=%d/%d", availableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady) } // specToGroveTopologyConstraint converts a deployment-level topology constraint diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go new file mode 100644 index 000000000000..41250e69311e --- /dev/null +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamo + +import ( + "context" + "fmt" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dra" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// ResolveLPXGPUShape counts one complete engine replica from a non-nil rendered +// LPX PCS. GPU-role main containers define engine width; all role Pods define +// replica cost. Outer scaling-group replicas never multiply either value. +func ResolveLPXGPUShape(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet) (GPUShape, error) { + enginePods := make([]dra.PodSpecMultiplicity, 0, len(pcs.Spec.Template.Cliques)) + replicaPods := make([]dra.PodSpecMultiplicity, 0, len(pcs.Spec.Template.Cliques)) + for _, clique := range pcs.Spec.Template.Cliques { + pod := &clique.Spec.PodSpec + replicaPods = append(replicaPods, dra.PodSpecMultiplicity{PodSpec: pod, Count: clique.Spec.Replicas}) + if clique.Labels[lpx.ExecutionRoleLabel] != lpxGPUExecutionRole { + continue + } + engine := &corev1.PodSpec{ResourceClaims: pod.ResourceClaims} + for i := range pod.Containers { + if pod.Containers[i].Name == consts.MainContainerName { + engine.Containers = pod.Containers[i : i+1] + break + } + } + if len(engine.Containers) == 0 { + return GPUShape{}, fmt.Errorf("LPX GPU role %q has no main container", clique.Name) + } + enginePods = append(enginePods, dra.PodSpecMultiplicity{PodSpec: engine, Count: clique.Spec.Replicas}) + } + engineGPUs, err := dra.ResolvePodSetGPUCount(ctx, reader, pcs.Namespace, enginePods) + if err != nil { + return GPUShape{}, fmt.Errorf("resolve LPX engine GPUs: %w", err) + } + replicaGPUs, err := dra.ResolvePodSetGPUCount(ctx, reader, pcs.Namespace, replicaPods) + if err != nil { + return GPUShape{}, fmt.Errorf("resolve LPX replica GPUs: %w", err) + } + return GPUShape{GPUsPerEngine: int64(engineGPUs), GPUsPerReplica: int64(replicaGPUs)}, nil +} + +// EvaluateLPXGroveReadiness observes one complete engine, including every role +// in every replica. Ordinary DGD components are deliberately not evaluated here. +func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet) (GroveReadiness, error) { + component := lpx.ServingComponent(source) + status := v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, RuntimeNamespace: source.GetDynamoNamespaceForComponent(component)} + // Draft instances are counted from their own complete Agent cliques. + statuses := make(map[string]v1beta1.ComponentReplicaStatus) + for _, member := range lpx.Components(source) { + if member.ComponentName != component.ComponentName { + statuses[member.ComponentName] = v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodClique, RuntimeNamespace: status.RuntimeNamespace, + } + } + } + verifiedAvailable := int32(0) + result := func(ready bool, classification, message string) GroveReadiness { + status.Ready = ready + if status.AvailableReplicas != nil { + status.AvailableReplicas = ptr.To(min(*status.AvailableReplicas, verifiedAvailable)) + } + statuses[component.ComponentName] = status + for name, member := range statuses { + member.Ready = ready + statuses[name] = member + } + return GroveReadiness{Ready: ready, Classification: classification, Message: message, ComponentStatuses: statuses} + } + pending := func(message string) GroveReadiness { + return result(false, v1beta1.DGDReadyReasonSomeResourcesNotReady, message) + } + if pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) || + pcs.Annotations[LPXInputRevisionAnnotation] != deployment.Spec.InputRevision || len(pcs.Spec.Template.PodCliqueScalingGroupConfigs) != 1 { + return pending("Waiting for the exact LPX PodCliqueSet"), nil + } + hash := getAcceptedPCSRevisionHash(pcs) + if hash == nil { + return pending("Waiting for Grove to accept the LPX PodCliqueSet revision"), nil + } + config := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] + groupName := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name) + status.ComponentNames = []string{groupName} + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: groupName}, group); err != nil { + if apierrors.IsNotFound(err) { + return pending("Waiting for the LPX scaling group"), nil + } + return GroveReadiness{}, err + } + status.Replicas, status.UpdatedReplicas = group.Status.Replicas, group.Status.UpdatedReplicas + status.AvailableReplicas = ptr.To(group.Status.AvailableReplicas) + if !metav1.IsControlledBy(group, pcs) || !group.DeletionTimestamp.IsZero() || group.Status.ObservedGeneration == nil || *group.Status.ObservedGeneration != group.Generation { + return pending("Waiting for the exact observed LPX scaling group"), nil + } + status.ScheduledReplicas = ptr.To(group.Status.ScheduledReplicas) + replicas := ptr.Deref(component.Replicas, 1) + if group.Spec.Replicas != replicas || group.Status.CurrentPodCliqueSetGenerationHash == nil || *group.Status.CurrentPodCliqueSetGenerationHash != *hash { + return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity"), nil + } + // Observe every member before returning so partial draft readiness remains visible. + unreadyClassification, unreadyMessage := "", "" + noteUnready := func(classification, message string) { + if unreadyMessage == "" { + unreadyClassification, unreadyMessage = classification, message + } + } + for replica := int32(0); replica < replicas; replica++ { + replicaReady := true + for _, template := range pcs.Spec.Template.Cliques { + name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: groupName, Replica: int(replica)}, template.Name) + memberName := template.Labels[lpx.StageLabel] + readiness, err := observeLPXRole(ctx, reader, deployment, group, name, template.Spec.Replicas) + if err != nil { + return GroveReadiness{}, err + } + // Sum complete model instances, never physical Agent Pod counts. + if draft, isDraft := statuses[memberName]; isDraft { + draft.ComponentNames = append(draft.ComponentNames, name) + if readiness.status.ReadyReplicas != nil { + draft.Replicas += readiness.status.Replicas + draft.UpdatedReplicas += readiness.status.UpdatedReplicas + draft.ScheduledReplicas = ptr.To(ptr.Deref(draft.ScheduledReplicas, 0) + ptr.Deref(readiness.status.ScheduledReplicas, 0)) + draft.ReadyReplicas = ptr.To(ptr.Deref(draft.ReadyReplicas, 0) + *readiness.status.ReadyReplicas) + } + statuses[memberName] = draft + } + if !readiness.ready { + noteUnready(readiness.classification, readiness.reason) + replicaReady = false + } + } + if replicaReady { + verifiedAvailable++ + } + } + if unreadyMessage != "" { + return result(false, unreadyClassification, unreadyMessage), nil + } + ready, message, classification := pcsgStatusReady(group, replicas) + if ready { + classification = v1beta1.DGDReadyReasonAllResourcesReady + } + return result(ready, classification, message), nil +} + +// observeLPXRole fences one clique by ownership and revision, then reports its +// counts in complete model instances. The group must already be observed at the +// accepted PCS revision; all pointer inputs are non-nil. +func observeLPXRole(ctx context.Context, reader client.Reader, deployment *v1alpha1.LPXGraphDeployment, group *grovev1alpha1.PodCliqueScalingGroup, name string, width int32) (groveComponentReadiness, error) { + role := groveComponentReadiness{} + clique := &grovev1alpha1.PodClique{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: group.Namespace, Name: name}, clique); err != nil { + if apierrors.IsNotFound(err) { + return role.withResult(false, fmt.Sprintf("Waiting for LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + return role, err + } + if !metav1.IsControlledBy(clique, group) || !clique.DeletionTimestamp.IsZero() || + clique.Annotations[LPXInputRevisionAnnotation] != deployment.Spec.InputRevision { + return role.withResult(false, fmt.Sprintf("Waiting for exact LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + if clique.Spec.Replicas != width || clique.Status.CurrentPodCliqueSetGenerationHash == nil || + *clique.Status.CurrentPodCliqueSetGenerationHash != *group.Status.CurrentPodCliqueSetGenerationHash { + return role.withResult(false, fmt.Sprintf("LPX role %s has not applied the desired revision and capacity", name), v1beta1.DGDReadyReasonUpdating), nil + } + + // A model instance is counted only after the complete build width is observed. + readiness := podCliqueReadiness(clique, log.FromContext(ctx)) + role = role.withResult(readiness.ready, readiness.reason, readiness.classification) + if clique.Status.ObservedGeneration == nil || *clique.Status.ObservedGeneration != clique.Generation { + return role, nil + } + if clique.Status.Replicas >= width { + role.status.Replicas = 1 + } + if clique.Status.UpdatedReplicas >= width { + role.status.UpdatedReplicas = 1 + } + scheduled, ready := int32(0), int32(0) + if clique.Status.ScheduledReplicas >= width || readiness.ready { + scheduled = 1 + } + if readiness.ready { + ready = 1 + } + role.status.ScheduledReplicas, role.status.ReadyReplicas = &scheduled, &ready + return role, nil +} + +func pcsgStatusReady(pcsg *grovev1alpha1.PodCliqueScalingGroup, desiredReplicas int32) (bool, string, string) { + if pcsg.Status.Replicas == desiredReplicas && + pcsg.Status.UpdatedReplicas == desiredReplicas && + pcsg.Status.AvailableReplicas == desiredReplicas { + return true, "", "" + } + + minAvailable := meta.FindStatusCondition(pcsg.Status.Conditions, groveconstants.ConditionTypeMinAvailableBreached) + if minAvailable != nil && minAvailable.Status == metav1.ConditionFalse && + (minAvailable.Reason == groveconstants.ConditionReasonScheduledReplicasBelowMinAvailable || + minAvailable.Reason == legacyConditionReasonInsufficientScheduledPCSGReplicas) { + return false, fmt.Sprintf("min-available breached (%s): %s", minAvailable.Reason, minAvailable.Message), v1beta1.DGDReadyReasonInsufficientCapacity + } + if scheduled := pcsg.Status.ScheduledReplicas; scheduled > 0 && scheduled < desiredReplicas { + return false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduled, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity + } + if pcsg.Status.UpdatedReplicas != desiredReplicas { + return false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, pcsg.Status.UpdatedReplicas), v1beta1.DGDReadyReasonUpdating + } + if pcsg.Status.Replicas != desiredReplicas { + return false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, pcsg.Status.Replicas), v1beta1.DGDReadyReasonUpdating + } + return false, fmt.Sprintf("scheduled but available=%d/%d", pcsg.Status.AvailableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady +} diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go new file mode 100644 index 000000000000..cac4eba15c0b --- /dev/null +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamo + +import ( + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestResolveLPXGPUShape(t *testing.T) { + for _, test := range []struct { + name string + want GPUShape + }{ + {"scalar-init-peak", GPUShape{GPUsPerEngine: 8, GPUsPerReplica: 21}}, + {"claim-template", GPUShape{GPUsPerEngine: 4, GPUsPerReplica: 7}}, + {"shared-claim", GPUShape{GPUsPerEngine: 2, GPUsPerReplica: 5}}, + {"lpu-only", GPUShape{}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Count two GPU workers and three auxiliary Pods in each complete engine replica") + nativeSidecar := gpuContainer("native-sidecar", "1") + nativeSidecar.RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + gpu := &grovev1alpha1.PodCliqueTemplateSpec{ + Name: "cyborg", Labels: map[string]string{lpx.ExecutionRoleLabel: lpxGPUExecutionRole}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 2, PodSpec: corev1.PodSpec{ + Containers: []corev1.Container{gpuContainer(consts.MainContainerName, "4"), gpuContainer("sidecar", "1")}, + InitContainers: []corev1.Container{nativeSidecar, gpuContainer("prepare", "8")}, + }}, + } + auxiliary := &grovev1alpha1.PodCliqueTemplateSpec{ + Name: "agents", Labels: map[string]string{lpx.ExecutionRoleLabel: "lpu"}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 3, PodSpec: corev1.PodSpec{ + Containers: []corev1.Container{gpuContainer("agent", "0"), gpuContainer("metrics", "1")}, + }}, + } + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, + Spec: grovev1alpha1.PodCliqueSetSpec{Template: grovev1alpha1.PodCliqueSetTemplateSpec{ + Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{gpu, auxiliary}, + PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{{Replicas: ptr.To(int32(3))}}, + }}, + } + claim := &resourcev1.ResourceClaim{ObjectMeta: metav1.ObjectMeta{Namespace: pcs.Namespace, Name: "gpu"}, + Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ + Name: "gpu", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: "gpu.nvidia.com", Count: 2, AllocationMode: resourcev1.DeviceAllocationModeExactCount}, + }}}}, + } + claimTemplate := &resourcev1.ResourceClaimTemplate{ObjectMeta: claim.ObjectMeta, + Spec: resourcev1.ResourceClaimTemplateSpec{Spec: claim.Spec}, + } + scheme := runtime.NewScheme() + require.NoError(t, resourcev1.AddToScheme(scheme)) + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(claim, claimTemplate, + &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu.nvidia.com"}}).Build() + switch test.name { + case "claim-template", "shared-claim": + pod := &gpu.Spec.PodSpec + pod.InitContainers = nil + for i := range pod.Containers { + pod.Containers[i].Resources = corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "gpu"}}} + } + pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(claimTemplate.Name)}} + if test.name == "shared-claim" { + pod.ResourceClaims[0].ResourceClaimTemplateName = nil + pod.ResourceClaims[0].ResourceClaimName = ptr.To(claim.Name) + } + case "lpu-only": + pcs.Spec.Template.Cliques = []*grovev1alpha1.PodCliqueTemplateSpec{auxiliary} + auxiliary.Spec.PodSpec.Containers = auxiliary.Spec.PodSpec.Containers[:1] + } + before := pcs.DeepCopy() + shape, err := ResolveLPXGPUShape(t.Context(), reader, pcs) + require.NoError(t, err) + require.Equal(t, test.want, shape) + require.Equal(t, before, pcs, "GPU accounting must not mutate shared Pod specs") + + if test.name == "claim-template" { + t.Log("Missing dependencies must not produce a misleading zero-GPU result") + require.NoError(t, reader.Delete(t.Context(), claimTemplate)) + _, err := ResolveLPXGPUShape(t.Context(), reader, pcs) + require.ErrorContains(t, err, "resolve LPX engine GPUs") + } + }) + } +} diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index dbcf4205d623..0533507bfa63 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -6,15 +6,18 @@ import ( "strings" "testing" + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -662,8 +665,7 @@ func TestCheckPodCliqueReady(t *testing.T) { WithStatusSubresource(objects...). Build() - logger := log.FromContext(ctx) - ready, reason, serviceStatus, classification, checkErr := CheckPodCliqueReady(ctx, fakeKubeClient, tt.resourceName, tt.namespace, logger) + ready, reason, serviceStatus, classification, checkErr := CheckPodCliqueReady(ctx, fakeKubeClient, tt.resourceName, tt.namespace, log.FromContext(ctx)) g.Expect(checkErr).NotTo(gomega.HaveOccurred()) g.Expect(ready).To(gomega.Equal(tt.wantReady)) @@ -678,297 +680,96 @@ func TestCheckPodCliqueReady(t *testing.T) { } } -func TestCheckPCSGReady(t *testing.T) { - ctx := context.Background() - +func TestEvaluateGroveReadinessUsesDeclaredLayout(t *testing.T) { tests := []struct { - name string - resourceName string - namespace string - existingPCSG *grovev1alpha1.PodCliqueScalingGroup - wantReady bool - wantReasonContains string - wantClassification string - wantServiceStatus v1beta1.ComponentReplicaStatus + name string + nodes int32 + gms bool + undeclared bool }{ - { - name: "PCSG not found", - resourceName: "missing-pcsg", - namespace: "default", - wantReady: false, - wantReasonContains: "resource not found", - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"missing-pcsg"}, - }, - }, - { - name: "PCSG fully ready", - resourceName: "ready-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ready-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 3, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 3, - AvailableReplicas: 3, - UpdatedReplicas: 3, - ScheduledReplicas: 3, - ObservedGeneration: ptr.To(int64(1)), - }, - }, - wantReady: true, - wantClassification: "", - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"ready-pcsg"}, - Replicas: 3, - UpdatedReplicas: 3, - AvailableReplicas: ptr.To(int32(3)), - ScheduledReplicas: ptr.To(int32(3)), - }, - }, - { - name: "PCSG with zero replicas desired", - resourceName: "zero-replicas-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "zero-replicas-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 0, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 0, - AvailableReplicas: 0, - UpdatedReplicas: 0, - ObservedGeneration: ptr.To(int64(1)), - }, - }, - wantReady: true, - wantClassification: "", - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"zero-replicas-pcsg"}, - Replicas: 0, - UpdatedReplicas: 0, - AvailableReplicas: ptr.To(int32(0)), - ScheduledReplicas: ptr.To(int32(0)), - }, - }, - { - name: "PCSG spec not yet processed - observedGeneration < generation", - resourceName: "stale-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "stale-pcsg", - Namespace: "default", - Generation: 3, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 2, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - AvailableReplicas: 2, - UpdatedReplicas: 2, - ObservedGeneration: ptr.To(int64(2)), - }, - }, - wantReady: false, - wantReasonContains: "spec not yet processed", - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"stale-pcsg"}, - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: ptr.To(int32(2)), - }, - }, - { - name: "PCSG not ready - available replicas less than desired", - resourceName: "not-ready-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "not-ready-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 3, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 3, - AvailableReplicas: 1, - UpdatedReplicas: 3, - ScheduledReplicas: 3, - ObservedGeneration: ptr.To(int64(1)), - }, - }, - wantReady: false, - wantReasonContains: "scheduled but available=1/3", - wantClassification: v1beta1.DGDReadyReasonPodsNotReady, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"not-ready-pcsg"}, - Replicas: 3, - UpdatedReplicas: 3, - AvailableReplicas: ptr.To(int32(1)), - ScheduledReplicas: ptr.To(int32(3)), - }, - }, - { - name: "PCSG not fully updated - updated replicas less than desired", - resourceName: "not-updated-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "not-updated-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 3, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 3, - AvailableReplicas: 3, - UpdatedReplicas: 2, - ScheduledReplicas: 3, - ObservedGeneration: ptr.To(int64(1)), - }, - }, - wantReady: false, - wantReasonContains: "desired=3, updated=2", - wantClassification: v1beta1.DGDReadyReasonUpdating, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"not-updated-pcsg"}, - Replicas: 3, - UpdatedReplicas: 2, - AvailableReplicas: ptr.To(int32(3)), - ScheduledReplicas: ptr.To(int32(3)), - }, - }, - { - name: "PCSG performing rolling update - replicas != desired", - resourceName: "rolling-update-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rolling-update-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 3, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 4, - AvailableReplicas: 3, - UpdatedReplicas: 3, - ScheduledReplicas: 4, - ObservedGeneration: ptr.To(int64(1)), - }, - }, - wantReady: false, - wantReasonContains: "performing rolling update", - wantClassification: v1beta1.DGDReadyReasonUpdating, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"rolling-update-pcsg"}, - Replicas: 4, - UpdatedReplicas: 3, - AvailableReplicas: ptr.To(int32(3)), - ScheduledReplicas: ptr.To(int32(4)), - }, - }, - { - name: "PCSG with nil observedGeneration", - resourceName: "nil-observed-gen-pcsg", - namespace: "default", - existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nil-observed-gen-pcsg", - Namespace: "default", - Generation: 1, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: 2, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - AvailableReplicas: 2, - UpdatedReplicas: 2, - ObservedGeneration: nil, - }, - }, - wantReady: false, - wantReasonContains: "observedGeneration is nil", - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - wantServiceStatus: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{"nil-observed-gen-pcsg"}, - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: ptr.To(int32(2)), - }, - }, + {name: "standalone", nodes: 1}, + {name: "multinode", nodes: 4}, + {name: "inter-pod GMS", nodes: 1, gms: true}, + {name: "multinode GMS", nodes: 4, gms: true}, + {name: "undeclared standalone group", nodes: 1, undeclared: true}, + {name: "undeclared multinode group", nodes: 4, undeclared: true}, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - g := gomega.NewGomegaWithT(t) - - s := scheme.Scheme - err := v1alpha1.AddToScheme(s) - g.Expect(err).NotTo(gomega.HaveOccurred()) - err = grovev1alpha1.AddToScheme(s) + g := gomega.NewWithT(t) + t.Log("Render the declared ordinary layout through the production renderer") + dgd := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "test-dgd", Namespace: "default"}, + Spec: v1beta1.DynamoGraphDeploymentSpec{BackendFramework: "vllm", Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "Worker", ComponentType: v1beta1.ComponentTypeWorker, Replicas: ptr.To(int32(2)), + Multinode: &v1beta1.MultinodeSpec{NodeCount: tt.nodes}, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")}}, + }}}}, + }}}, + } + if tt.gms { + dgd.Spec.Components[0].Experimental = &v1beta1.ExperimentalSpec{ + GPUMemoryService: &v1beta1.GPUMemoryServiceSpec{Mode: v1beta1.GMSModeInterPod}, + } + } + reader := newFakeGroveClient(g) + pcs, err := GenerateGrovePodCliqueSet(t.Context(), dgd, &configv1alpha1.OperatorConfiguration{}, + &controller_common.RuntimeConfig{Gate: features.Gates{DRA: true}}, reader, &mockSecretsRetriever{}, nil, nil, nil) g.Expect(err).NotTo(gomega.HaveOccurred()) - var objects []client.Object - if tt.existingPCSG != nil { - objects = append(objects, tt.existingPCSG) + t.Log("Prove emitted scaling groups use the component identity") + wantName, wantKind := pcs.Name+"-0-worker", v1beta1.ComponentKindPodClique + if dgd.Spec.Components[0].UsesPCSG() { + wantKind = v1beta1.ComponentKindPodCliqueScalingGroup + g.Expect(pcs.Spec.Template.PodCliqueScalingGroupConfigs).To(gomega.HaveLen(1)) + g.Expect(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Name).To(gomega.Equal("worker")) + } else { + g.Expect(pcs.Spec.Template.PodCliqueScalingGroupConfigs).To(gomega.BeEmpty()) } - fakeKubeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(objects...). - WithStatusSubresource(objects...). - Build() - - logger := log.FromContext(ctx) - ready, reason, serviceStatus, classification, checkErr := CheckPCSGReady(ctx, fakeKubeClient, tt.resourceName, tt.namespace, logger) - - g.Expect(checkErr).NotTo(gomega.HaveOccurred()) - g.Expect(ready).To(gomega.Equal(tt.wantReady)) - if tt.wantReasonContains != "" { - g.Expect(reason).To(gomega.ContainSubstring(tt.wantReasonContains)) - } else { - g.Expect(reason).To(gomega.Equal("")) + t.Log("An undeclared runtime group in an observed PCS must not redirect ordinary readiness") + if tt.undeclared { + var names []string + for _, clique := range pcs.Spec.Template.Cliques { + names = append(names, clique.Name) + } + pcs.Spec.Template.PodCliqueScalingGroupConfigs = []grovev1alpha1.PodCliqueScalingGroupConfig{{Name: "undeclared", CliqueNames: names}} } - g.Expect(classification).To(gomega.Equal(tt.wantClassification)) - g.Expect(serviceStatus).To(gomega.Equal(tt.wantServiceStatus)) + readiness, err := EvaluateGroveReadiness(t.Context(), reader, dgd, pcs) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(readiness.Ready).To(gomega.BeFalse()) + g.Expect(readiness.ComponentStatuses).To(gomega.HaveLen(1)) + g.Expect(readiness.ComponentStatuses["Worker"].ComponentKind).To(gomega.Equal(wantKind)) + g.Expect(readiness.ComponentStatuses["Worker"].ComponentNames).To(gomega.Equal([]string{wantName})) }) } } func TestEvaluateGroveReadiness(t *testing.T) { ctx := context.Background() - + newPCSGPodClique := func(name, pcsg, component string, groupReplica, replicas, readyReplicas int32) *grovev1alpha1.PodClique { + return &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{ + grovecommon.LabelPartOfKey: "test-dgd", + grovecommon.LabelPodCliqueScalingGroup: pcsg, + grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(groupReplica), + commonconsts.KubeLabelDynamoComponent: component, + }, + }, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: replicas}, + Status: grovev1alpha1.PodCliqueStatus{ + Replicas: replicas, + UpdatedReplicas: replicas, + ReadyReplicas: readyReplicas, + ScheduledReplicas: replicas, + ObservedGeneration: ptr.To(int64(1)), + }, + } + } tests := []struct { name string dgdSpec v1alpha1.DynamoGraphDeploymentSpec @@ -1045,6 +846,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ + newReadyPCS("test-dgd"), &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-decode", @@ -1054,11 +856,12 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 2, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 2, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, &grovev1alpha1.PodCliqueScalingGroup{ @@ -1070,13 +873,19 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 3, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 3, - UpdatedReplicas: 3, - AvailableReplicas: 3, - ScheduledReplicas: 3, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 3, + UpdatedReplicas: 3, + AvailableReplicas: 3, + ScheduledReplicas: 3, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, + newPCSGPodClique("test-dgd-0-decode-0", "test-dgd-0-decode", "decode", 0, 1, 1), + newPCSGPodClique("test-dgd-0-decode-1", "test-dgd-0-decode", "decode", 1, 1, 1), + newPCSGPodClique("test-dgd-0-prefill-0", "test-dgd-0-prefill", "prefill", 0, 1, 1), + newPCSGPodClique("test-dgd-0-prefill-1", "test-dgd-0-prefill", "prefill", 1, 1, 1), + newPCSGPodClique("test-dgd-0-prefill-2", "test-dgd-0-prefill", "prefill", 2, 1, 1), }, wantReady: true, wantReason: "", @@ -1084,6 +893,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { "decode": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-decode"}, + Ready: true, Replicas: 2, UpdatedReplicas: 2, AvailableReplicas: ptr.To(int32(2)), @@ -1092,6 +902,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, + Ready: true, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: ptr.To(int32(3)), @@ -1100,7 +911,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, { - name: "multinode service not ready - PCSG not ready", + name: "multinode service checks child PodClique readiness", dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { @@ -1115,6 +926,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ + newReadyPCS("test-dgd"), &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-worker", @@ -1124,23 +936,26 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 1, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 2, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, + newPCSGPodClique("test-dgd-0-worker-0", "test-dgd-0-worker", "worker", 0, 1, 1), + newPCSGPodClique("test-dgd-0-worker-1", "test-dgd-0-worker", "worker", 1, 1, 0), }, wantReady: false, - wantReason: "worker: scheduled but available=1/2", + wantReason: "worker: test-dgd-0-worker: scheduled but ready=0/1", wantServiceStatuses: map[string]v1beta1.ComponentReplicaStatus{ "worker": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-worker"}, Replicas: 2, UpdatedReplicas: 2, - AvailableReplicas: ptr.To(int32(1)), + AvailableReplicas: ptr.To(int32(2)), ScheduledReplicas: ptr.To(int32(2)), }, }, @@ -1176,6 +991,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ + newReadyPCS("test-dgd"), &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-frontend", @@ -1201,11 +1017,12 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 1, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 1, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, &grovev1alpha1.PodCliqueScalingGroup{ @@ -1217,13 +1034,18 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 2, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 2, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), + CurrentPodCliqueSetGenerationHash: ptr.To("current"), }, }, + newPCSGPodClique("test-dgd-0-decode-0", "test-dgd-0-decode", "decode", 0, 1, 1), + newPCSGPodClique("test-dgd-0-decode-1", "test-dgd-0-decode", "decode", 1, 1, 1), + newPCSGPodClique("test-dgd-0-prefill-0", "test-dgd-0-prefill", "prefill", 0, 1, 1), + newPCSGPodClique("test-dgd-0-prefill-1", "test-dgd-0-prefill", "prefill", 1, 1, 1), }, wantReady: false, wantReason: "decode: scheduled but available=1/2", @@ -1231,6 +1053,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, + Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -1247,6 +1070,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, + Ready: true, Replicas: 2, UpdatedReplicas: 2, AvailableReplicas: ptr.To(int32(2)), @@ -1300,6 +1124,33 @@ func TestEvaluateGroveReadiness(t *testing.T) { var objects []client.Object objects = append(objects, betaDGD) objects = append(objects, tt.existingGroveResources...) + var observedPCS *grovev1alpha1.PodCliqueSet + for _, object := range tt.existingGroveResources { + if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + observedPCS = pcs.DeepCopy() + break + } + } + if observedPCS != nil { + if betaDGD.Status.Components == nil { + betaDGD.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) + } + for i := range betaDGD.Spec.Components { + component := &betaDGD.Spec.Components[i] + if !IsWorkerComponent(string(component.ComponentType)) { + continue + } + observedPCS.Spec.Template.Cliques = append( + observedPCS.Spec.Template.Cliques, + &grovev1alpha1.PodCliqueTemplateSpec{Labels: map[string]string{ + commonconsts.KubeLabelDynamoComponent: component.ComponentName, + }}, + ) + status := betaDGD.Status.Components[component.ComponentName] + status.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) + betaDGD.Status.Components[component.ComponentName] = status + } + } fakeKubeClient := fake.NewClientBuilder(). WithScheme(s). @@ -1307,7 +1158,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { WithStatusSubresource(objects...). Build() - readiness, err := EvaluateGroveReadiness(ctx, fakeKubeClient, betaDGD, nil) + readiness, err := EvaluateGroveReadiness(ctx, fakeKubeClient, betaDGD, observedPCS) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(readiness.Ready).To(gomega.Equal(tt.wantReady)) @@ -1320,9 +1171,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { if component == nil { continue } - if !IsWorkerComponent(string(component.ComponentType)) { - wantStatus.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) - } + wantStatus.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) wantServiceStatuses[componentName] = wantStatus } g.Expect(readiness.ComponentStatuses).To(gomega.Equal(wantServiceStatuses)) @@ -1350,6 +1199,8 @@ func TestEvaluateGroveReadinessPublishesWorkerRuntimeNamespaceAfterCutover(t *te legacyPCS bool pcsObservedGenerationAhead bool childGenerationStale bool + desiredReplicas *int32 + childReplicas *int32 wantReady bool wantNamespace string }{ @@ -1362,6 +1213,13 @@ func TestEvaluateGroveReadinessPublishesWorkerRuntimeNamespaceAfterCutover(t *te {name: "accepted PCS revision publishes its rendered hash instead of the desired DGD hash", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, updateEnded: true, childReady: true, wantReady: true, wantNamespace: acceptedHash}, {name: "PCS observation ahead of its generation preserves the active worker namespace", activeNamespace: true, pcsAccepted: true, pcsObservedGenerationAhead: true, childRevision: targetRevision, updateEnded: true, childReady: true, wantReady: true, wantNamespace: "active"}, {name: "accepted PCS with a stale child generation preserves the active worker namespace", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, updateEnded: true, childReady: true, childGenerationStale: true, wantNamespace: "active"}, + {name: "cached pre-scale-up spec cannot publish readiness or the new namespace", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(2)), wantNamespace: "active"}, + {name: "cached pre-scale-down spec cannot publish readiness or the new namespace", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(1)), childReplicas: ptr.To(int32(2)), wantNamespace: "active"}, + {name: "cached zero replicas cannot complete scale-up", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(1)), childReplicas: ptr.To(int32(0)), wantNamespace: "active"}, + {name: "scale-to-zero waits for the cached spec", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(0)), wantNamespace: "active"}, + {name: "observed scale-up publishes readiness and the new namespace", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(2)), childReplicas: ptr.To(int32(2)), wantReady: true, wantNamespace: acceptedHash}, + {name: "observed scale-to-zero publishes readiness and the new namespace", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, desiredReplicas: ptr.To(int32(0)), childReplicas: ptr.To(int32(0)), wantReady: true, wantNamespace: acceptedHash}, + {name: "omitted replicas preserve externally managed capacity", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, childReady: true, childReplicas: ptr.To(int32(2)), wantReady: true, wantNamespace: acceptedHash}, {name: "accepted completed PCS revision remains published after worker health loss", activeNamespace: true, pcsAccepted: true, childRevision: targetRevision, updateEnded: true, wantNamespace: acceptedHash}, {name: "accepted legacy PCS with an unfinished child preserves the active worker namespace", activeNamespace: true, pcsAccepted: true, legacyPCS: true, childRevision: targetRevision, updateInProgress: true, childReady: true, wantReady: true, wantNamespace: "active"}, {name: "accepted legacy PCS publishes the base namespace", pcsAccepted: true, legacyPCS: true, childRevision: targetRevision, updateEnded: true, childReady: true, wantReady: true, wantNamespace: "base"}, @@ -1377,6 +1235,7 @@ func TestEvaluateGroveReadinessPublishesWorkerRuntimeNamespaceAfterCutover(t *te Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: componentName, ComponentType: v1beta1.ComponentTypePrefill, + Replicas: tt.desiredReplicas, }}, }, } @@ -1424,14 +1283,15 @@ func TestEvaluateGroveReadinessPublishesWorkerRuntimeNamespaceAfterCutover(t *te if tt.childGenerationStale { childGeneration = 2 } + replicas := ptr.Deref(tt.childReplicas, int32(1)) podClique := &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{Name: GroveComponentResourceName(dgd, componentName), Namespace: dgd.Namespace, Generation: childGeneration}, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: replicas}, Status: grovev1alpha1.PodCliqueStatus{ - Replicas: 1, - ReadyReplicas: 1, - UpdatedReplicas: 1, - ScheduledReplicas: 1, + Replicas: replicas, + ReadyReplicas: replicas, + UpdatedReplicas: replicas, + ScheduledReplicas: replicas, ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(tt.childRevision), }, @@ -1931,7 +1791,7 @@ func TestGroveComponentRevisionStateHasCompletedAcceptedPCSRevision(t *testing.T // Ready-reason classification tests (merged from classification_test.go). // These exercise the DGD-level Ready reason returned as the 4th value of -// CheckPodCliqueReady / CheckPCSGReady, focusing on the capacity-before- +// the PodClique and PCSG-backed component evaluators, focusing on the capacity-before- // readiness branches (schedule-gated, scheduling condition, partial scheduled // count) that the readiness/serviceStatus tables above do not isolate. // --------------------------------------------------------------------------- @@ -1939,9 +1799,8 @@ func TestGroveComponentRevisionStateHasCompletedAcceptedPCSRevision(t *testing.T // The Check*Ready classification tests below exercise the full Grove-status // reading path (client.Get + field/condition inspection) and assert the // DGD Ready reason string returned as the 4th value. They complement the -// existing TestCheckPodCliqueReady / TestCheckPCSGReady tables (which assert -// ready/reason/serviceStatus) by focusing on capacity-before-readiness -// classification ordering. +// readiness tables by focusing on capacity-before-readiness classification +// ordering. func TestCheckPodCliqueReadyClassification(t *testing.T) { ctx := context.Background() @@ -2086,12 +1945,13 @@ func TestCheckPodCliqueReadyClassification(t *testing.T) { } } -func TestCheckPCSGReadyClassification(t *testing.T) { +func TestPCSGBackedComponentReadyClassification(t *testing.T) { ctx := context.Background() tests := []struct { name string pcsg *grovev1alpha1.PodCliqueScalingGroup + dgdReplicas *int32 wantReady bool wantClassification string wantReasonContains string @@ -2102,15 +1962,35 @@ func TestCheckPCSGReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), + dgdReplicas: ptr.To(int32(2)), wantReady: true, wantClassification: "", }, + { + name: "omitted DGD replicas defer to the externally scaled PCSG", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, + ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }), + wantReady: true, + }, + { + name: "pinned DGD replicas take precedence over PCSG replicas", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, + ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }), + dgdReplicas: ptr.To(int32(3)), + wantClassification: v1beta1.DGDReadyReasonUpdating, + wantReasonContains: "desired replicas=3", + }, { name: "available below desired, fully scheduled -> PodsNotReady", pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ Replicas: 2, AvailableReplicas: 0, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonPodsNotReady, wantReasonContains: "available=0/2", @@ -2121,6 +2001,7 @@ func TestCheckPCSGReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 0, UpdatedReplicas: 2, ScheduledReplicas: 1, ObservedGeneration: ptr.To(int64(1)), }), + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonInsufficientCapacity, wantReasonContains: "scheduled=1/2", @@ -2135,10 +2016,11 @@ func TestCheckPCSGReadyClassification(t *testing.T) { // Status=False (Status=True is the availability reason). Type: groveconstants.ConditionTypeMinAvailableBreached, Status: metav1.ConditionFalse, - Reason: "InsufficientScheduledPodCliqueScalingGroupReplicas", + Reason: legacyConditionReasonInsufficientScheduledPCSGReplicas, LastTransitionTime: metav1.Now(), }}, }), + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonInsufficientCapacity, wantReasonContains: "min-available breached", @@ -2149,6 +2031,7 @@ func TestCheckPCSGReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 1, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonUpdating, wantReasonContains: "updated=1", @@ -2159,6 +2042,7 @@ func TestCheckPCSGReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 1, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonPodsNotReady, wantReasonContains: "available=1/2", @@ -2166,6 +2050,7 @@ func TestCheckPCSGReadyClassification(t *testing.T) { { name: "not found -> Unclassified", pcsg: nil, + dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, wantReasonContains: "resource not found", @@ -2176,18 +2061,44 @@ func TestCheckPCSGReadyClassification(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := gomega.NewGomegaWithT(t) - var objs []client.Object + pcs := newReadyPCS("pcs") + objs := []client.Object{pcs} if tt.pcsg != nil { + tt.pcsg.Status.CurrentPodCliqueSetGenerationHash = ptr.To("current") objs = append(objs, tt.pcsg) + for replica := range testPCSGReplicas { + objs = append(objs, &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("pcsg-%d-component", replica), + Namespace: "default", + Generation: 1, + Labels: map[string]string{ + grovecommon.LabelPodCliqueScalingGroup: testPCSGName, + grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(replica), + commonconsts.KubeLabelDynamoComponent: "component", + }, + }, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + ScheduledReplicas: 1, + ObservedGeneration: ptr.To(int64(1)), + }, + }) + } } c := newFakeGroveClient(g, objs...) - ready, reason, _, classification, checkErr := CheckPCSGReady(ctx, c, testPCSGName, "default", log.FromContext(ctx)) + readiness, checkErr := observeComponentPCSGReadiness( + ctx, c, pcs, testPCSGName, "default", "component", tt.dgdReplicas, + ) g.Expect(checkErr).NotTo(gomega.HaveOccurred()) - g.Expect(ready).To(gomega.Equal(tt.wantReady)) - g.Expect(classification).To(gomega.Equal(tt.wantClassification)) + g.Expect(readiness.ready).To(gomega.Equal(tt.wantReady)) + g.Expect(readiness.classification).To(gomega.Equal(tt.wantClassification)) if tt.wantReasonContains != "" { - g.Expect(reason).To(gomega.ContainSubstring(tt.wantReasonContains)) + g.Expect(readiness.reason).To(gomega.ContainSubstring(tt.wantReasonContains)) } }) } @@ -2196,8 +2107,8 @@ func TestCheckPCSGReadyClassification(t *testing.T) { // --- test helpers --- // Fixed resource identity used by the classification tests. The helpers below -// hardcode these so the name passed to CheckPodCliqueReady / CheckPCSGReady -// stays in sync with the object created in the fake client, and so the spec +// hardcode these so the names passed to the readiness evaluators stay in sync +// with the object created in the fake client, and so the spec // replica count (which every case shares) is defined in one place. Only the // status varies per test case. const ( @@ -2223,6 +2134,16 @@ func newPCSG(status grovev1alpha1.PodCliqueScalingGroupStatus) *grovev1alpha1.Po } } +func newReadyPCS(name string) *grovev1alpha1.PodCliqueSet { + return &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + Status: grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(int64(1)), + CurrentGenerationHash: ptr.To("current"), + }, + } +} + func newFakeGroveClient(g *gomega.WithT, objects ...client.Object) client.Client { s := scheme.Scheme g.Expect(grovev1alpha1.AddToScheme(s)).To(gomega.Succeed()) @@ -2236,11 +2157,9 @@ func newFakeGroveClient(g *gomega.WithT, objects ...client.Object) client.Client // TestGroveReadinessTransientErrorsPropagate verifies that a non-NotFound Get // error from a Grove child is returned as an error (so the reconcile retries and // does not advance ObservedGeneration), rather than folded into a normal -// not-ready result. NotFound remains a legitimate not-ready state (covered by -// the "not found" cases in TestCheckPodCliqueReady / TestCheckPCSGReady). +// not-ready result. NotFound remains a legitimate not-ready state. func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { ctx := context.Background() - logger := log.FromContext(ctx) transientErr := fmt.Errorf("transient API error") newClient := func(g *gomega.WithT) client.Client { @@ -2265,7 +2184,7 @@ func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { t.Run("CheckPodCliqueReady returns error on non-NotFound get failure", func(t *testing.T) { g := gomega.NewGomegaWithT(t) c := newClient(g) - ready, _, _, classification, err := CheckPodCliqueReady(ctx, c, "test-pc", "default", logger) + ready, _, _, classification, err := CheckPodCliqueReady(ctx, c, "test-pc", "default", log.FromContext(ctx)) g.Expect(err).To(gomega.HaveOccurred()) g.Expect(err.Error()).To(gomega.ContainSubstring("transient API error")) g.Expect(ready).To(gomega.BeFalse()) @@ -2273,14 +2192,16 @@ func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { g.Expect(classification).To(gomega.BeEmpty()) }) - t.Run("CheckPCSGReady returns error on non-NotFound get failure", func(t *testing.T) { + t.Run("PCSG-backed component returns error on non-NotFound get failure", func(t *testing.T) { g := gomega.NewGomegaWithT(t) c := newClient(g) - ready, _, _, classification, err := CheckPCSGReady(ctx, c, "test-pcsg", "default", logger) + readiness, err := observeComponentPCSGReadiness( + ctx, c, newReadyPCS("pcs"), "test-pcsg", "default", "component", ptr.To(int32(1)), + ) g.Expect(err).To(gomega.HaveOccurred()) g.Expect(err.Error()).To(gomega.ContainSubstring("transient API error")) - g.Expect(ready).To(gomega.BeFalse()) - g.Expect(classification).To(gomega.BeEmpty()) + g.Expect(readiness.ready).To(gomega.BeFalse()) + g.Expect(readiness.classification).To(gomega.BeEmpty()) }) t.Run("EvaluateGroveReadiness propagates a child read error", func(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/build.go b/deploy/operator/internal/dynamo/lpx/build.go new file mode 100644 index 000000000000..442628073d31 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/build.go @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "path/filepath" + "slices" + "strings" +) + +const lpuChipsPerNode = 8 + +// BuildCompilationMode records the compiler-authored execution mode in a normalized build. +type BuildCompilationMode string + +// BuildFamily identifies the physical LPU target family of a normalized build. +type BuildFamily string + +const ( + // BuildCompilationModeUnknown represents a build whose compilation mode is not known. + BuildCompilationModeUnknown BuildCompilationMode = "" + // BuildCompilationModeLPUOnly represents a build executed entirely on LPUs. + BuildCompilationModeLPUOnly BuildCompilationMode = "lpuOnly" + // BuildCompilationModeHybrid retains the model manifest's historical "lpx" value. + BuildCompilationModeHybrid BuildCompilationMode = "lpx" + // BuildFamilyXT identifies the XT8888 LPU target family. + BuildFamilyXT BuildFamily = "xt8888" + // BuildFamilyHX identifies the HX16x8x2x3 LPU target family. + BuildFamilyHX BuildFamily = "hx16x8x2x3" +) + +// Build is the registry's source-independent view of an LPU build. +// +// The version 2 Cap'n Proto manifest populates this shape before deployment code +// derives container args, replica counts, prop-sync settings, and tokenizer defaults. +type Build struct { + // Path is the absolute file or GCS reference of the build payload. + Path string + // Family is the physical LPU target family. + Family BuildFamily + // CompilationMode selects the compiler-authored LPU-only or hybrid artifact mode. + CompilationMode BuildCompilationMode + // Partitions contains the normalized physical compiler partitions. + Partitions []BuildPartition + // BatchSize is the normalized batch size for the build. + BatchSize int + // SelectedPropSyncChains contains source partition IDs grouped into selected prop-sync chains. + SelectedPropSyncChains [][]int + // StandaloneTokenEmbeddings reports whether token embeddings occupy a standalone partition. + StandaloneTokenEmbeddings bool + // SupportsCPUEmbeddings reports whether standalone token embeddings may run on the CPU. + SupportsCPUEmbeddings bool + // RuntimeTokenizerPath is the tokenizer path relative to the build payload. + RuntimeTokenizerPath string + // RuntimeTokenEmbeddingsPath is the token-embeddings path relative to the build payload. + RuntimeTokenEmbeddingsPath string + // IOFPGACount is the number of I/O FPGA endpoints described by the build. + IOFPGACount int32 + // IOFanoutFactor is the number of clients assigned to each I/O FPGA transaction. + IOFanoutFactor int32 + // runtimeSettings holds immutable normalized defaults on a snapshot and + // configured runtime settings shared by immutable component projections. + runtimeSettings map[string]any +} + +// BuildPartition describes one normalized physical compiler partition. +type BuildPartition struct { + // SourcePartitionID is the compiler partition id used in artifacts and + // selected prop-sync chains. It is not the slice index after sorting/filtering. + SourcePartitionID int + // PartPath is the nonempty relative gas-dir fragment under the build payload. + PartPath string + // Topology is the scheduler-facing node shape for this partition. + Topology Topology + // HXExtent is the scheduler-facing four-dimensional HX allocation. + HXExtent []int64 + + // runtimeNodeCount overrides the node count derived from Topology after + // selected prop-sync partitions are collapsed for the LPU runtime. Sub-host + // partitions still occupy one scheduler endpoint each, so their combined + // chip count alone cannot recover the number of scheduled Agent pods. + runtimeNodeCount int +} + +// effectiveNodeCount returns the number of Agent endpoints assigned to the +// partition. Source partitions derive it from topology; collapsed runtime +// partitions preserve the sum of their physical scheduler endpoints. +func (p BuildPartition) effectiveNodeCount() int { + if p.runtimeNodeCount > 0 { + return p.runtimeNodeCount + } + return p.Topology.Replicas() +} + +// omitStandaloneEmbeddingPartition removes host-only embedding work when the +// build manifest explicitly identifies it as safe to run outside the LPU. +func (b *Build) omitStandaloneEmbeddingPartition() { + if !b.SupportsCPUEmbeddings || !b.StandaloneTokenEmbeddings { + return + } + + // Normalized XT partitions are sorted by source ID; keep the retained interval independently owned. + if len(b.Partitions) > 1 && b.Partitions[0].SourcePartitionID == 0 { + b.Partitions = slices.Clone(b.Partitions[1:]) + } +} + +func buildRuntimePath(buildPath string, modelStoragePath string) (string, error) { + buildURL, err := parseBuildRef(buildPath) + if err != nil { + return "", fmt.Errorf("parse build path %q: %w", buildPath, err) + } + + switch buildURL.Scheme { + case BuildSchemeFile: + return buildURL.Path, nil + case BuildSchemeGCS: + if modelStoragePath == "" { + return "", fmt.Errorf("model storage path is empty") + } + + objectPath := strings.TrimPrefix(buildURL.Path, "/") + if objectPath == "" { + return "", fmt.Errorf("invalid GCS build path %q: missing object path", buildPath) + } + + // Validate every object path segment before joining the accepted path once. + for component := range strings.SplitSeq(objectPath, "/") { + if component == "" || component == "." || component == ".." { + return "", fmt.Errorf("invalid GCS build path %q: bad path segment %q", buildPath, component) + } + } + + return filepath.Join(modelStoragePath, "gcs", buildURL.Host, objectPath), nil + default: + return "", fmt.Errorf("unsupported build path scheme %q", buildURL.Scheme) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot.go b/deploy/operator/internal/dynamo/lpx/build_snapshot.go new file mode 100644 index 000000000000..6a4b5b5b2533 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot.go @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "syscall" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + // ErrBuildSnapshotInconsistent identifies a build locator whose contents + // cannot be treated as one immutable build snapshot. + ErrBuildSnapshotInconsistent = errors.New("immutable build snapshot is inconsistent") +) + +const ( + // Preserve the versioned digest domain so terminology changes do not alter + // immutable build content identities. + buildSnapshotIdentityVersion = "dynamo-lpx-compiler-snapshot/v3" + // The existing Model Express transport admits messages up to this size. + // Reuse that compatibility ceiling as one total retained control-plane + // metadata budget instead of permitting an unbounded sum of streamed + // chunks or local files. + maxBuildSnapshotMetadataBytes = modelExpressMaxMessageSize +) + +// BuildSnapshot is a provider-fenced copy of the exact manifest consumed by +// the LPX workload projection. +type BuildSnapshot struct { + ref string + contentID string + manifestBytes []byte +} + +// AcquireBuildSnapshot fences the required manifest-v2 compiler metadata with inventories +// and duplicate reads. The receiver must be non-nil and is not mutated. +func (r *ModelRegistry) AcquireBuildSnapshot(ctx context.Context, id string) (*BuildSnapshot, error) { + refURL, err := r.BuildURL(id) + if err != nil { + return nil, err + } + + pre, err := r.listBuildFiles(ctx, refURL) + if err != nil { + return nil, err + } + + // Require the supported binary compiler manifest. + if _, present := slices.BinarySearch(pre, gbuildManifestV2CapnpFile); !present { + return nil, fmt.Errorf( + "%w: immutable build %q is missing %s", + ErrBuildSnapshotInconsistent, + refURL.String(), + gbuildManifestV2CapnpFile, + ) + } + + // Capture the compiler manifest within the snapshot metadata budget. + manifestPath := gbuildManifestV2CapnpFile + manifestData, readErr := r.readBuildFileBounded(ctx, refURL, manifestPath, maxBuildSnapshotMetadataBytes) + if readErr != nil { + if errors.Is(readErr, errBuildFileTooLarge) { + return nil, fmt.Errorf( + "%w: compiler metadata %s in %q exceeds its bounded acquisition budget: %w", + ErrBuildSnapshotInconsistent, + manifestPath, + refURL.String(), + readErr, + ) + } + if isDefinitiveBuildMemberAbsence(readErr) { + return nil, fmt.Errorf( + "%w: compiler metadata %s disappeared from %q after inventory: %w", + ErrBuildSnapshotInconsistent, + manifestPath, + refURL.String(), + readErr, + ) + } + return nil, readErr + } + + // Verify the build inventory did not change after reading compiler metadata. + post, err := r.listBuildFiles(ctx, refURL) + if err != nil { + if isDefinitiveBuildMemberAbsence(err) { + return nil, fmt.Errorf( + "%w: build inventory disappeared from %q while acquiring it: %w", + ErrBuildSnapshotInconsistent, + refURL.String(), + err, + ) + } + return nil, err + } + if !slices.Equal(pre, post) { + return nil, fmt.Errorf( + "%w: build inventory changed while acquiring %q", + ErrBuildSnapshotInconsistent, + refURL.String(), + ) + } + + // Re-read the compiler manifest after verifying the build inventory. + verifiedData, readErr := r.readBuildFileBounded( + ctx, + refURL, + manifestPath, + maxBuildSnapshotMetadataBytes, + ) + if readErr != nil { + if errors.Is(readErr, errBuildFileTooLarge) { + return nil, fmt.Errorf( + "%w: compiler metadata %s grew beyond its bounded acquisition budget in %q: %w", + ErrBuildSnapshotInconsistent, + manifestPath, + refURL.String(), + readErr, + ) + } + if isDefinitiveBuildMemberAbsence(readErr) { + return nil, fmt.Errorf( + "%w: compiler metadata %s disappeared from %q after inventory verification: %w", + ErrBuildSnapshotInconsistent, + manifestPath, + refURL.String(), + readErr, + ) + } + return nil, readErr + } + if !bytes.Equal(manifestData, verifiedData) { + return nil, fmt.Errorf( + "%w: compiler metadata %s changed while acquiring %q", + ErrBuildSnapshotInconsistent, + manifestPath, + refURL.String(), + ) + } + + // Bind the stable inventory and manifest bytes into the snapshot identity. + hash := sha256.New() + writeBuildSnapshotField(hash, []byte(buildSnapshotIdentityVersion)) + writeBuildSnapshotField(hash, []byte("inventory")) + writeBuildSnapshotField(hash, []byte(strconv.Itoa(len(pre)))) + for _, path := range pre { + writeBuildSnapshotField(hash, []byte("inventory-path")) + writeBuildSnapshotField(hash, []byte(path)) + } + writeBuildSnapshotField(hash, []byte("compiler-metadata")) + writeBuildSnapshotField(hash, []byte("1")) + writeBuildSnapshotField(hash, []byte("metadata-path")) + writeBuildSnapshotField(hash, []byte(manifestPath)) + writeBuildSnapshotField(hash, []byte("metadata-bytes")) + writeBuildSnapshotField(hash, manifestData) + return &BuildSnapshot{ + ref: refURL.String(), + contentID: fmt.Sprintf("sha256:%x", hash.Sum(nil)), + manifestBytes: manifestData, + }, nil +} + +func isDefinitiveBuildMemberAbsence(err error) bool { + return errors.Is(err, os.ErrNotExist) || + errors.Is(err, syscall.ENOTDIR) || + status.Code(err) == codes.NotFound +} + +func writeBuildSnapshotField(hash interface{ Write([]byte) (int, error) }, value []byte) { + _, _ = fmt.Fprintf(hash, "%d:", len(value)) + _, _ = hash.Write(value) +} + +// NormalizedBuildSnapshot is the validated, source-independent projection input. +type NormalizedBuildSnapshot struct { + contentID string + build *Build +} + +// normalizeBuildSnapshot validates and lowers a successfully acquired, non-nil +// snapshot into its normalized build representation. The snapshot is not mutated. +func normalizeBuildSnapshot(snapshot *BuildSnapshot) (NormalizedBuildSnapshot, error) { + // Decode and lower the required manifest-v2 compiler metadata. + manifest, err := decodeGbuildManifestV2(snapshot.manifestBytes) + if err != nil { + return NormalizedBuildSnapshot{}, err + } + build, err := buildFromGbuildManifestV2(snapshot.ref, manifest) + if err != nil { + return NormalizedBuildSnapshot{}, err + } + return NormalizedBuildSnapshot{contentID: snapshot.contentID, build: build}, nil +} + +func normalizeModelPath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + path = filepath.ToSlash(filepath.Clean(path)) + return strings.TrimPrefix(path, "./") +} + +// normalizeBuildFilePaths consumes a nonnil owned path slice and canonicalizes it in place. +func normalizeBuildFilePaths(paths []string) ([]string, error) { + // Normalize and retain compiler evidence while validating every listed path. + for index, rawPath := range paths { + path := normalizeModelPath(rawPath) + if path == "" || path == "." || filepath.IsAbs(path) || path == ".." || strings.HasPrefix(path, "../") { + return nil, fmt.Errorf("build inventory contains invalid relative path %q", rawPath) + } + paths[index] = path + } + + // Canonical ordering makes duplicate paths adjacent without a second index. + sort.Strings(paths) + for i := 1; i < len(paths); i++ { + if paths[i] == paths[i-1] { + return nil, fmt.Errorf("build inventory repeats relative path %q", paths[i]) + } + } + return paths, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go new file mode 100644 index 000000000000..0256ee3eecf8 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "errors" + "testing" + + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func manifestV2Payload(t *testing.T) []byte { + t.Helper() + payload, err := newManifestV2ContractFixture(t).Message().Marshal() + require.NoError(t, err) + return payload +} + +func TestAcquireBuildSnapshotFencesCompilerMetadata(t *testing.T) { + t.Parallel() + + t.Log("Create two different revision-2 compiler metadata reads for one build") + const buildID = "model/build" + client := &fakeModelServiceClient{ + list: &modelpb.ModelFileList{Files: []*modelpb.ModelFileInfo{ + {RelativePath: gbuildManifestV2CapnpFile}, + }}, + fileStreams: []*fakeModelFileStream{ + {chunks: modelFileChunks(gbuildManifestV2CapnpFile, "first")}, + {chunks: modelFileChunks(gbuildManifestV2CapnpFile, "second")}, + }, + } + registry, err := NewModelRegistry("gs://bucket/registry", client) + require.NoError(t, err) + + t.Log("Reject compiler metadata that changes during acquisition") + _, err = registry.AcquireBuildSnapshot(t.Context(), buildID) + require.ErrorIs(t, err, ErrBuildSnapshotInconsistent) + require.ErrorContains(t, err, "compiler metadata manifest.v2.capnp.bin changed while acquiring") +} + +func TestAcquireBuildSnapshotClassifiesSelectedMemberDisappearance(t *testing.T) { + t.Parallel() + + t.Log("Define definitive and transient failures for both manifest reads") + payload := manifestV2Payload(t) + for _, testCase := range []struct { + name string + failingRead int + readErr error + }{ + {name: "gRPC NotFound on first read", failingRead: 1, readErr: status.Error(codes.NotFound, "object disappeared")}, + {name: "gRPC NotFound on duplicate read", failingRead: 2, readErr: status.Error(codes.NotFound, "object disappeared")}, + {name: "transient gRPC failure on first read", failingRead: 1, readErr: status.Error(codes.Unavailable, "temporary object-store failure")}, + {name: "transient gRPC failure on duplicate read", failingRead: 2, readErr: status.Error(codes.Unavailable, "temporary object-store failure")}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Log("Construct the selected Model Express read sequence") + streams := []*fakeModelFileStream{{err: testCase.readErr}} + if testCase.failingRead == 2 { + streams = append([]*fakeModelFileStream{ + {chunks: modelFileChunks(gbuildManifestV2CapnpFile, string(payload))}, + }, streams...) + } + client := &fakeModelServiceClient{ + list: &modelpb.ModelFileList{Files: []*modelpb.ModelFileInfo{{RelativePath: gbuildManifestV2CapnpFile}}}, + fileStreams: streams, + } + registry, err := NewModelRegistry("gs://bucket/registry", client) + require.NoError(t, err) + + t.Log("Preserve the transport error while classifying definitive absence as inconsistency") + _, err = registry.AcquireBuildSnapshot(t.Context(), "model/build") + require.Error(t, err) + require.Equal(t, status.Code(testCase.readErr) == codes.NotFound, errors.Is(err, ErrBuildSnapshotInconsistent)) + require.ErrorIs(t, err, testCase.readErr) + }) + } +} + +func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { + t.Parallel() + + t.Log("Serve a valid revision-2 compiler manifest through Model Express") + payload := manifestV2Payload(t) + for _, ref := range []string{"model/build", "gs://bucket/registry/model/build"} { + t.Run(ref, func(t *testing.T) { + t.Log("Create an independent client and registry for this reference form") + client := &fakeModelServiceClient{ + fileStreams: []*fakeModelFileStream{ + {chunks: modelFileChunks(gbuildManifestV2CapnpFile, string(payload))}, + {chunks: modelFileChunks(gbuildManifestV2CapnpFile, string(payload))}, + }, + list: &modelpb.ModelFileList{Files: []*modelpb.ModelFileInfo{{RelativePath: gbuildManifestV2CapnpFile}}}, + } + registry, err := NewModelRegistry("gs://bucket/registry", client) + require.NoError(t, err) + + t.Log("Normalize the manifest and retain the current registry locator") + build, err := normalizeRegistryFixtureBuild(t.Context(), registry, ref) + require.NoError(t, err) + require.Equal(t, "gs://bucket/registry/model/build", build.Path) + require.Equal(t, 8, build.BatchSize) + require.Len(t, build.Partitions, 1) + require.Equal(t, "part-0", build.Partitions[0].PartPath) + require.Len(t, client.listRequests, 2) + require.Len(t, client.filesRequests, 2) + for _, request := range client.filesRequests { + require.Equal(t, []string{gbuildManifestV2CapnpFile}, request.GetFileSelector().GetPaths()) + } + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go new file mode 100644 index 000000000000..9ca75a7f724d --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeManifestV2Payload(t *testing.T, buildDir string, payload []byte) { + t.Helper() + require.NoError(t, os.MkdirAll(buildDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, gbuildManifestV2CapnpFile), payload, 0o600)) +} + +func TestAcquireBuildSnapshotTracksLocalContent(t *testing.T) { + t.Parallel() + + t.Log("Write revision-2 compiler metadata beside invalid publication JSON and unrelated payloads") + registryDir := t.TempDir() + buildDir := filepath.Join(registryDir, "build-id") + payload := manifestV2Payload(t) + writeManifestV2Payload(t, buildDir, payload) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, gbuildManifestJSONFile), []byte(`not valid json`), 0o600)) + for _, path := range []string{"extra-metadata.json", "part-0/chip-0.gas", "weights.bin"} { + fullPath := filepath.Join(buildDir, filepath.FromSlash(path)) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o700)) + require.NoError(t, os.WriteFile(fullPath, []byte(`{}`), 0o600)) + } + + t.Log("Inventory every file beneath the build root, including metadata, artifacts and weights") + paths, err := localBuildFilePaths(buildDir) + require.NoError(t, err) + require.ElementsMatch(t, []string{ + "extra-metadata.json", + gbuildManifestJSONFile, + gbuildManifestV2CapnpFile, + "part-0/chip-0.gas", + "weights.bin", + }, paths) + + t.Log("Acquire the absolute file URL without a configured registry prefix and retain the exact binary manifest") + registry, err := NewModelRegistry("", nil) + require.NoError(t, err) + ref := (&url.URL{Scheme: BuildSchemeFile, Path: buildDir}).String() + first, err := registry.AcquireBuildSnapshot(t.Context(), ref) + require.NoError(t, err) + require.Equal(t, payload, first.manifestBytes) + + t.Log("Normalize exclusively from the binary revision-2 contract") + normalized, err := normalizeBuildSnapshot(first) + require.NoError(t, err) + build := normalized.build + require.Equal(t, 8, build.BatchSize) + require.Equal(t, int64(8), build.runtimeSettings["batch_size"]) + require.EqualValues(t, 4, build.IOFPGACount) + require.EqualValues(t, 2, build.IOFanoutFactor) + + t.Log("Resolve and normalize the relative build ID through a configured local registry") + relativeRegistry, err := NewModelRegistry(registryDir, nil) + require.NoError(t, err) + relativeBuild, err := normalizeRegistryFixtureBuild(t.Context(), relativeRegistry, "build-id") + require.NoError(t, err) + require.Equal(t, 8, relativeBuild.BatchSize) + + t.Log("Changing only a payload path changes snapshot identity") + require.NoError(t, os.Rename(filepath.Join(buildDir, "part-0/chip-0.gas"), filepath.Join(buildDir, "part-0/chip-1.gas"))) + second, err := registry.AcquireBuildSnapshot(t.Context(), ref) + require.NoError(t, err) + require.NotEqual(t, first.contentID, second.contentID) + + t.Log("Changing only the manifest bytes changes identity again without changing the inventory") + writeManifestV2Payload(t, buildDir, append(payload, 0)) + third, err := registry.AcquireBuildSnapshot(t.Context(), ref) + require.NoError(t, err) + require.NotEqual(t, second.contentID, third.contentID) + + t.Log("Acquire a stable build whose manifest bytes are malformed") + writeManifestV2Payload(t, buildDir, []byte("invalid")) + snapshot, err := registry.AcquireBuildSnapshot(t.Context(), ref) + require.NoError(t, err) + + t.Log("Reject malformed compiler input after successful acquisition") + _, err = normalizeBuildSnapshot(snapshot) + require.ErrorContains(t, err, "parsing manifest.v2.capnp.bin") +} + +func TestAcquireBuildSnapshotRejectsLegacyCompilerMetadata(t *testing.T) { + t.Parallel() + + t.Log("Write a build containing only unsupported revision-1 and JSON compiler metadata") + buildDir := t.TempDir() + for _, path := range []string{ + "manifest.capnp.bin", + "allocation_metadata.json", + "compile_summary.json", + "gas_program_layout.json", + gbuildManifestJSONFile, + } { + require.NoError(t, os.WriteFile(filepath.Join(buildDir, path), []byte(`{}`), 0o600)) + } + registry, err := NewModelRegistry(buildDir, nil) + require.NoError(t, err) + + t.Log("Require revision-2 binary compiler metadata without fallback") + _, err = registry.AcquireBuildSnapshot(t.Context(), buildDir) + require.ErrorIs(t, err, ErrBuildSnapshotInconsistent) + require.ErrorContains(t, err, "is missing manifest.v2.capnp.bin") +} + +func TestNormalizeBuildFilePathsCanonicalizesAndRejectsAmbiguity(t *testing.T) { + t.Parallel() + + t.Log("Canonicalize and sort an unambiguous build inventory") + inventory, err := normalizeBuildFilePaths([]string{"z/file.gas", gbuildManifestJSONFile, "./a/file.gas"}) + require.NoError(t, err) + require.Equal(t, []string{"a/file.gas", gbuildManifestJSONFile, "z/file.gas"}, inventory) + + for name, paths := range map[string][]string{ + "normalized duplicate": {"a/file.gas", "./a/file.gas"}, + "parent traversal": {"../outside.gas"}, + "absolute path": {filepath.Join(string(filepath.Separator), "outside.gas")}, + "empty path": {" "}, + } { + t.Run(name, func(t *testing.T) { + t.Log("Reject the selected ambiguous or unsafe inventory path") + _, err := normalizeBuildFilePaths(paths) + require.Error(t, err) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/build_test.go b/deploy/operator/internal/dynamo/lpx/build_test.go new file mode 100644 index 000000000000..2b7308f120ca --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/build_test.go @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildRuntimePath(t *testing.T) { + t.Parallel() + + t.Log("Define local and GCS runtime-path resolution contracts") + tests := []struct { + name string + buildPath string + modelStoragePath string + want string + wantErr string + }{ + {name: "absolute path", buildPath: "/snapshot/../build", want: "/build"}, + {name: "file URL", buildPath: "file:///snapshot/../build", want: "/build"}, + {name: "GCS URL", buildPath: "gs://bucket/model/build", modelStoragePath: "/models", want: "/models/gcs/bucket/model/build"}, + {name: "relative path", buildPath: "relative/build", wantErr: "ref \"relative/build\" must be an absolute path or URL"}, + {name: "GCS without storage", buildPath: "gs://bucket/model", wantErr: "model storage path is empty"}, + {name: "GCS without host", buildPath: "gs:///model", modelStoragePath: "/models", wantErr: "invalid ref \"gs:///model\": missing host"}, + {name: "GCS without object", buildPath: "gs://bucket", modelStoragePath: "/models", wantErr: "invalid GCS build path \"gs://bucket\": missing object path"}, + {name: "GCS empty segment", buildPath: "gs://bucket/a//b", modelStoragePath: "/models", wantErr: "bad path segment \"\""}, + {name: "GCS dot segment", buildPath: "gs://bucket/a/./b", modelStoragePath: "/models", wantErr: "bad path segment \".\""}, + {name: "GCS parent segment", buildPath: "gs://bucket/a/../b", modelStoragePath: "/models", wantErr: "bad path segment \"..\""}, + {name: "unsupported scheme", buildPath: "https://bucket/model", wantErr: "unsupported build path scheme \"https\""}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Resolve the build reference into its runtime filesystem path") + got, err := buildRuntimePath(test.buildPath, test.modelStoragePath) + + t.Log("Verify the resolved path or exact error contract") + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh new file mode 100644 index 000000000000..b70c758c54bc --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +AGENT_BINARY="${AGENT_BINARY:-agent-hybrid-v2-roce}" +HOME="${HOME:-/root}" +MPI_SSH_PORT="${MPI_SSH_PORT:-@@LPX_SSH_PORT@@}" +READINESS_PORT="${READINESS_PORT:-@@LPX_READINESS_PORT@@}" +RDMA_PORT="${RDMA_PORT:-@@LPX_RDMA_PORT@@}" + +readarray -t SOURCE_PARTITIONS < "${LPU_CONFIG_DIR}/partition_ids" +readarray -t PARTITION_PATHS < "${LPU_CONFIG_DIR}/partition_paths" +readarray -t TOPOLOGIES < "${LPU_CONFIG_DIR}/topologies" + +SOURCE_PARTITION="${SOURCE_PARTITIONS[$CONFIG_PARTITION_INDEX]}" +PARTITION_PATH="${PARTITION_PATHS[$CONFIG_PARTITION_INDEX]}" +TOPOLOGY="${TOPOLOGIES[$CONFIG_PARTITION_INDEX]}" + +agent_part_from_partition_path() { + local part_path="$1" + local part_name="${part_path##*/}" + if [[ "${part_name}" =~ ^part-?([0-9]+)$ ]]; then + printf "%s\n" "${BASH_REMATCH[1]}" + return + fi + + printf "%s\n" "${SOURCE_PARTITION}" +} + +agent_assemble_dir_from_partition_path() { + local part_path="$1" + local agent_part="$2" + local part_name="${part_path##*/}" + local parent="${part_path%/*}" + if [[ "${part_name}" != "part-${agent_part}" ]]; then + local assemble_dir="/tmp/dynamo-agent-assemble/row-${CONFIG_PARTITION_INDEX}" + mkdir -p "${assemble_dir}" + ln -sfn "${GAS_DIR}/${part_path}" "${assemble_dir}/part-${agent_part}" + printf "%s\n" "${assemble_dir}" + return + fi + if [[ "${parent}" == "${part_path}" ]]; then + printf "%s\n" "${GAS_DIR}" + return + fi + printf "%s/%s\n" "${GAS_DIR}" "${parent}" +} + +AGENT_PART="$(agent_part_from_partition_path "${PARTITION_PATH}")" +AGENT_ASSEMBLE_DIR="$(agent_assemble_dir_from_partition_path "${PARTITION_PATH}" "${AGENT_PART}")" + +setup_ssh() { + mkdir -p "${HOME}/.ssh" "${HOME}/.ssh/host_keys" + cp /ssh-pk/private.key "${HOME}/.ssh/id_rsa" + cp /ssh-pk/private.key.pub "${HOME}/.ssh/id_rsa.pub" + cp /ssh-pk/private.key.pub "${HOME}/.ssh/authorized_keys" + chmod 600 "${HOME}/.ssh/id_rsa" "${HOME}/.ssh/authorized_keys" + chmod 644 "${HOME}/.ssh/id_rsa.pub" + printf "Host *\nIdentityFile %s/.ssh/id_rsa\nStrictHostKeyChecking no\nPort %s\n" "${HOME}" "${MPI_SSH_PORT}" > "${HOME}/.ssh/config" + + ssh-keygen -t rsa -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" -N "" -q + ssh-keygen -t ecdsa -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" -N "" -q + ssh-keygen -t ed25519 -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" -N "" -q +} + +run_sshd() { + setup_ssh + echo "Starting sshd on port ${MPI_SSH_PORT} for partition ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}" + exec /bin/sshd -D -e \ + -o "PidFile none" \ + -o "Port ${MPI_SSH_PORT}" \ + -o "AuthorizedKeysFile ${HOME}/.ssh/authorized_keys" \ + -o "StrictModes no" \ + -o "PermitRootLogin yes" \ + -o "PubkeyAuthentication yes" \ + -o "SetEnv PARTITION_ID=\"${PARTITION_ID}\" RANK_IN_PARTITION=\"${RANK_IN_PARTITION}\"" \ + -h "${HOME}/.ssh/host_keys/ssh_host_rsa_key" \ + -h "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" \ + -h "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" +} + +partition_hostnames() { + for ((rank = 0; rank < NODE_COUNT; rank++)); do + local node_index=$((NODE_OFFSET + rank)) + printf "%s-%d.%s\n" "${GROVE_PCLQ_NAME}" "${node_index}" "${GROVE_HEADLESS_SERVICE}" + done +} + +wait_for_peer_ssh() { + local host="$1" + for i in $(seq 1 120); do + if ssh -o ConnectTimeout=2 "root@${host}" echo ok >/dev/null 2>&1; then + echo "${host}:${MPI_SSH_PORT} reachable" + return + fi + if [[ "${i}" == "120" ]]; then + echo "${host}:${MPI_SSH_PORT} unreachable after 120 attempts" >&2 + exit 1 + fi + sleep 1 + done +} + +run_lpu_system_init +command -v "${AGENT_BINARY}" + +if (( NODE_COUNT == 1 )); then + echo "Starting ${AGENT_BINARY} for logical partition ${LOGICAL_PARTITION_INDEX}, source partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}" + exec "${AGENT_BINARY}" \ + --assemble-dir "${AGENT_ASSEMBLE_DIR}" \ + --part "${AGENT_PART}" \ + --topology "${TOPOLOGY}" \ + --port "${RDMA_PORT}" \ + --readiness-port "${READINESS_PORT}" \ + --stage-name continue +fi + +if [[ "${PARTITION_RANK}" != "0" ]]; then + run_sshd +fi + +setup_ssh +PARTITION_HOSTS="$(partition_hostnames)" +readarray -t NODE_HOSTS <<< "${PARTITION_HOSTS}" + +MPI_HOSTS=() +MPI_HOSTS+=("${POD_IP}:1") +for host in "${NODE_HOSTS[@]:1}"; do + MPI_HOSTS+=("${host}:1") +done +MPI_HOST_LIST=$(IFS=,; echo "${MPI_HOSTS[*]}") + +for host in "${NODE_HOSTS[@]:1}"; do + wait_for_peer_ssh "${host}" +done + +MPI_ARGS=( + --allow-run-as-root + --host "${MPI_HOST_LIST}" + -np "${NODE_COUNT}" + --map-by ppr:1:node:OVERSUBSCRIBE + --mca pml ob1 + --mca btl tcp,self + --mca mtl ^ofi + --mca plm_rsh_args "-p ${MPI_SSH_PORT}" + --bind-to none + -x GROQ_DRACO_NETWORK_UP_CHECK_TIMEOUT + -x DRACO_FORCE_RECONFIGURE + -x DRACO_MAXFILE_OVERRIDE + -x LPU_VSAP_DIB_CREDIT_AUTO_RESTORE_US + --output TAG +) +if [[ -n "${MPI_BTL_TCP_IF_INCLUDE:-}" ]]; then + MPI_ARGS+=(--mca btl_tcp_if_include "${MPI_BTL_TCP_IF_INCLUDE}") +fi + +echo "Starting mpirun for logical partition ${LOGICAL_PARTITION_INDEX}, source partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}" +exec numactl --cpunodebind=0 --membind=0 mpirun "${MPI_ARGS[@]}" -- \ + "${AGENT_BINARY}" \ + --assemble-dir "${AGENT_ASSEMBLE_DIR}" \ + --part "${AGENT_PART}" \ + --topology "${TOPOLOGY}" \ + --multinode \ + --port "${RDMA_PORT}" \ + --readiness-port "${READINESS_PORT}" \ + --stage-name continue diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh new file mode 100644 index 000000000000..fe21d019c884 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Preserve legacy Agent names from Dynamo's runtime partition mapping. +export PARTITION_ID="${LOGICAL_PARTITION_INDEX}" +export RANK_IN_PARTITION="${PARTITION_RANK}" diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh new file mode 100644 index 000000000000..3e0e0d51c8a7 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +LPU_CONFIG_DIR="${LPU_CONFIG_DIR:-@@LPX_CONFIG_MOUNT_PATH@@}" +CLIQUE_POD_INDEX="${GROVE_PCLQ_POD_INDEX:-}" +if ! [[ "${CLIQUE_POD_INDEX}" =~ ^[0-9]+$ ]]; then + echo "GROVE_PCLQ_POD_INDEX must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}" >&2 + exit 1 +fi + +readarray -t PARTITION_NODE_COUNTS < "${LPU_CONFIG_DIR}/nodes_per_partition" +readarray -t PARTITION_NODE_OFFSETS < "${LPU_CONFIG_DIR}/partition_node_offsets" +if (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} != ${#PARTITION_NODE_OFFSETS[@]} )); then + echo "nodes_per_partition and partition_node_offsets must contain the same non-zero number of rows" >&2 + exit 1 +fi + +if [[ -f "${LPU_CONFIG_DIR}/partition_models" ]]; then + readarray -t PARTITION_MODELS < "${LPU_CONFIG_DIR}/partition_models" + readarray -t PARTITION_INDICES < "${LPU_CONFIG_DIR}/partition_indices" + if [[ -z "${LPU_MODEL_NAME:-}" ]]; then + echo "LPU_MODEL_NAME is required when partition_models is configured" >&2 + exit 1 + fi + if (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); then + echo "partition_models and partition_indices must match the partition config row count" >&2 + exit 1 + fi +else + PARTITION_MODELS=() + PARTITION_INDICES=() +fi + +CONFIG_PARTITION_INDEX="" +LOGICAL_PARTITION_INDEX="" +PARTITION_RANK="" +NODE_COUNT="" +NODE_OFFSET="" +PARTITION_MATCH_COUNT=0 +for i in "${!PARTITION_NODE_COUNTS[@]}"; do + row_node_count="${PARTITION_NODE_COUNTS[$i]}" + row_node_offset="${PARTITION_NODE_OFFSETS[$i]}" + if ! [[ "${row_node_count}" =~ ^[1-9][0-9]*$ && "${row_node_offset}" =~ ^[0-9]+$ ]]; then + echo "Invalid partition row ${i}: node count ${row_node_count}, node offset ${row_node_offset}" >&2 + exit 1 + fi + if (( ${#PARTITION_MODELS[@]} > 0 )) && [[ "${PARTITION_MODELS[$i]}" != "${LPU_MODEL_NAME}" ]]; then + continue + fi + if (( CLIQUE_POD_INDEX < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count )); then + continue + fi + + CONFIG_PARTITION_INDEX="${i}" + LOGICAL_PARTITION_INDEX="${PARTITION_INDICES[$i]:-${i}}" + PARTITION_RANK="$((CLIQUE_POD_INDEX - row_node_offset))" + NODE_COUNT="${row_node_count}" + NODE_OFFSET="${row_node_offset}" + PARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + 1)) +done + +if (( PARTITION_MATCH_COUNT != 1 )); then + echo "Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition rows for model ${LPU_MODEL_NAME:-}" >&2 + exit 1 +fi +if ! [[ "${LOGICAL_PARTITION_INDEX}" =~ ^[0-9]+$ ]]; then + echo "Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}" >&2 + exit 1 +fi diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh new file mode 100644 index 000000000000..8acd4b88879d --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +run_lpu_system_init() { + if [[ ! -e /dev/fd ]]; then + ln -svf /proc/self/fd /dev/fd || true + fi + mkdir -p /tmp + mkdir -p /var/run/dpdk/rte || true + chmod 777 /var/run/dpdk/rte || true + ulimit -l unlimited || true + + if command -v lpu-system-init >/dev/null 2>&1; then + local init_ok="unknown" + local init_status=0 + for attempt in $(seq 1 2); do + echo "Running lpu-system-init (attempt ${attempt})..." + local init_output + local -a init_args=(--ci-precommand) + if (( attempt == 2 )); then + init_args+=(--ci-remediation-commands) + fi + set +e + init_output="$(lpu-system-init "${init_args[@]}" 2>&1)" + init_status=$? + set -e + echo "${init_output}" + init_ok="$(echo "${init_output}" | grep -o '"ok": *[a-z]*' | tail -1 | grep -o 'true\|false' || echo "unknown")" + if (( init_status == 0 )) && [[ "${init_ok}" != "false" ]]; then + break + fi + if (( attempt == 1 )); then + if (( init_status != 0 )); then + echo "lpu-system-init exited with status ${init_status}; retrying with remediation" >&2 + else + echo "lpu-system-init reported ok=false; retrying with remediation" >&2 + fi + fi + done + if (( init_status != 0 )) || [[ "${init_ok}" == "false" ]]; then + echo "lpu-system-init failed after 2 attempts; aborting" >&2 + exit 1 + fi + else + echo "lpu-system-init not found; continuing without pre-init" >&2 + fi +} diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh new file mode 100644 index 000000000000..74da21d455b2 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +HOME="${HOME:-/root}" + +if [[ "${LPU_RUN_SYSTEM_INIT:-true}" == "true" ]]; then + run_lpu_system_init +fi + +ulimit -l unlimited || true + +%s + +if command -v setcap >/dev/null 2>&1; then + for binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do + if [[ -e "$binary" ]]; then + echo "Lifting capabilities of $binary" + setcap cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep "$(realpath "$binary")" || true + fi + done +fi + +# Pass down information from the k8s downward API into SSH sessions. +SETENV_LINE="" +SETENV_LINE="${SETENV_LINE} K8S_POD_NAME=\"$POD_NAME\"" +SETENV_LINE="${SETENV_LINE} K8S_POD_NAMESPACE=\"$POD_NAMESPACE\"" +SETENV_LINE="${SETENV_LINE} K8S_POD_UID=\"$POD_UID\"" +%sif [[ -n "${LPU_MODEL_NAME:-}" ]]; then + SETENV_LINE="${SETENV_LINE} LPU_MODEL_NAME=\"$LPU_MODEL_NAME\"" +fi +exec /bin/sshd -D -e \ + -o "PidFile none" \ + -o "Port @@LPX_SSH_PORT@@" \ + -o "AuthorizedKeysFile $HOME/.ssh/authorized_keys" \ + -o "StrictModes no" \ + -o PermitRootLogin=yes \ + -o PasswordAuthentication=no \ + -o PubkeyAuthentication=yes \ + -o "SetEnv $SETENV_LINE" \ + -h "$HOME/.ssh/host_keys/ssh_host_rsa_key" \ + -h "$HOME/.ssh/host_keys/ssh_host_ecdsa_key" \ + -h "$HOME/.ssh/host_keys/ssh_host_ed25519_key" diff --git a/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh b/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh new file mode 100644 index 000000000000..cd9b82bd546c --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +mkdir -p $HOME/.ssh $HOME/.ssh/host_keys +ls -la /ssh-pk/ +cp /ssh-pk/private.key $HOME/.ssh/id_rsa +cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub +cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys +chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys +chmod 644 $HOME/.ssh/id_rsa.pub +printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking no\nPort @@LPX_SSH_PORT@@\n' > $HOME/.ssh/config + +ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N '' +ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N '' +ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N '' diff --git a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go new file mode 100644 index 000000000000..89296217e722 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "capnproto.org/go/capnp/v3" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + otherFixtureValue = "other" + v2TestBuildName = "v2-test-build" + v3HXTopologyFamily = "16x8x2x3" + v3OpaqueTopology = "opaque-v3-topology" +) + +type unreachableBuildSnapshotSource struct{} + +func (unreachableBuildSnapshotSource) AcquireBuildSnapshot(context.Context, string) (*BuildSnapshot, error) { + return nil, fmt.Errorf("unexpected build snapshot acquisition") +} + +type staticBuildSnapshotSource map[string]*BuildSnapshot + +func (source staticBuildSnapshotSource) AcquireBuildSnapshot(_ context.Context, buildID string) (*BuildSnapshot, error) { + snapshot := source[buildID] + if snapshot == nil { + return nil, fmt.Errorf("unknown test build %q", buildID) + } + return snapshot, nil +} + +func newSelectedTestDGD(t *testing.T, name string, components ...v1beta1.DynamoComponentDeploymentSharedSpec) *v1beta1.DynamoGraphDeployment { + t.Helper() + + t.Log("Construct one fresh selected-LPX projection fixture around the caller-owned configuration") + return &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "test", UID: "uid", Generation: 1, + Annotations: map[string]string{ + commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend, + }, + }, + Spec: v1beta1.DynamoGraphDeploymentSpec{ + Components: components, + }, + } +} + +// testLPXComponent constructs an independent component with the caller's role membership. +func testLPXComponent(name, buildID string, roles ...v1beta1.ComponentRoleSpec) v1beta1.DynamoComponentDeploymentSharedSpec { + return v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: name, + ComponentType: v1beta1.ComponentTypeLPX, + LPX: &v1beta1.LPXConfig{BuildID: buildID}, + Roles: roles, + } +} + +// testLPXPodTemplate constructs an independent role template for each caller. +func testLPXPodTemplate(image string) *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: image}}}, + } +} + +func acquireTestSnapshot(t *testing.T, buildDir string) *BuildSnapshot { + t.Helper() + registry, err := NewModelRegistry(buildDir, nil) + require.NoError(t, err) + snapshot, err := registry.AcquireBuildSnapshot(t.Context(), buildDir) + require.NoError(t, err) + return snapshot +} + +func normalizeTestSnapshot(t *testing.T, snapshot *BuildSnapshot) NormalizedBuildSnapshot { + t.Helper() + normalized, err := normalizeBuildSnapshot(snapshot) + require.NoError(t, err) + return normalized +} + +func projectTestBuild(t *testing.T, snapshot NormalizedBuildSnapshot, + pipeline Pipeline, settings string) *ModelProjection { + t.Helper() + + t.Log("Project the caller-owned normalized build with the requested model settings") + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: pipeline, Models: []string{"default"}, BuildSnapshot: snapshot, ModelSettings: json.RawMessage(settings), + }) + require.NoError(t, err) + return projectionBatch[0] +} + +func writeV2CompilerFixture(t *testing.T) string { + t.Helper() + return writeCompilerFixture(t, newV2CompilerFixture()) +} + +func newV2CompilerFixture() testV3CapnpFixture { + topology := "URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA" + fixture := newV3CompilerFixture() + fixture.buildDirectoryName = v2TestBuildName + fixture.numLPUNodes = 4 + fixture.selectedPropSyncChains = [][]uint32{{7, 8}} + fixture.partitions = []testV3CapnpPartition{ + {id: 7, deviceType: manifestcapnp.DeviceType_lpu, topology: topology, numChips: 16, devicesPerNode: 8}, + {id: 8, deviceType: manifestcapnp.DeviceType_lpu, topology: topology, numChips: 16, devicesPerNode: 8}, + } + return fixture +} + +func writeV3CompilerFixture(t *testing.T) string { + t.Helper() + return writeCompilerFixture(t, newV3CompilerFixture()) +} + +type testV3CapnpFixture struct { + buildDirectoryName string + contractRevision uint32 + pipelineName string + compilationMode manifestcapnp.CompilationMode + numLPUNodes uint32 + selectedPropSyncChains [][]uint32 + partSelect bool + partitions []testV3CapnpPartition +} + +type testV3CapnpPartition struct { + id uint32 + deviceType manifestcapnp.DeviceType + topology string + numChips uint32 + devicesPerNode uint32 + topologyFamily string + partitionShape []uint32 +} + +func newV3CompilerFixture() testV3CapnpFixture { + return testV3CapnpFixture{ + contractRevision: manifestcapnp.CurrentContractRevision, + pipelineName: "default", + compilationMode: manifestcapnp.CompilationMode_lpuOnly, + numLPUNodes: 2, + partitions: []testV3CapnpPartition{{ + id: 1, + deviceType: manifestcapnp.DeviceType_lpu, + topology: v3OpaqueTopology, + numChips: 16, + devicesPerNode: 16, + }}, + } +} + +func writeCompilerFixture(t *testing.T, fixture testV3CapnpFixture) string { + t.Helper() + + t.Log("Create the selected compiler fixture directory") + buildDir := t.TempDir() + if fixture.buildDirectoryName != "" { + buildDir = filepath.Join(buildDir, fixture.buildDirectoryName) + require.NoError(t, os.Mkdir(buildDir, 0o700)) + } + + t.Log("Write the revision-2 compiler manifest fixture") + writeTestV3CapnpManifest(t, buildDir, fixture) + return buildDir +} + +func writeTestV3CapnpManifest(t *testing.T, buildDir string, fixture testV3CapnpFixture) { + t.Helper() + + t.Log("Encode the revision-2 manifest header and model metadata") + message, segment := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnp.NewRootManifest(segment) + require.NoError(t, err) + manifest.SetContractRevision(fixture.contractRevision) + model, err := manifest.NewModel() + require.NoError(t, err) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stopTokens, err := tokenizer.NewStopTokens(1) + require.NoError(t, err) + stopTokens.Set(0, 1) + + t.Log("Encode the deployment program, runtime I/O, and selected prop-sync chains") + deployment, err := manifest.NewDeployment() + require.NoError(t, err) + deployment.SetCompilationMode(fixture.compilationMode) + deployment.SetNumLpuNodes(fixture.numLPUNodes) + program, err := deployment.NewProgram() + require.NoError(t, err) + require.NoError(t, program.SetProgramName(fixture.pipelineName)) + program.SetBatchSize(1) + program.SetSequenceLength(8192) + program.SetInputSize(1) + program.SetOutputSize(1) + program.SetNumKvCaches(1) + runtimeIO, err := deployment.NewRuntimeIo() + require.NoError(t, err) + runtimeIO.SetProtocol(runtimeIOProtocolHost) + runtimeIO.SetReserved1(0) + runtimeIO.SetIoFpgaCount(1) + runtimeIO.SetFanoutFactor(1) + if fixture.selectedPropSyncChains != nil { + chains, err := deployment.NewSelectedPropSyncChains(int32(len(fixture.selectedPropSyncChains))) + require.NoError(t, err) + for chainIndex, partitionIDs := range fixture.selectedPropSyncChains { + chain := chains.At(chainIndex) + ids, err := chain.NewPartitionIds(int32(len(partitionIDs))) + require.NoError(t, err) + for idIndex, partitionID := range partitionIDs { + ids.Set(idIndex, partitionID) + } + } + } + + t.Log("Encode the flat revision-2 artifact inventory") + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + if fixture.partSelect { + partSelect, err := artifacts.NewPartSelect() + require.NoError(t, err) + selected, err := partSelect.NewPartitions(1) + require.NoError(t, err) + selected.At(0).SetDeviceType(manifestcapnp.DeviceType_lpu) + selected.At(0).SetPartitionId(fixture.partitions[0].id) + } + partitions, err := artifacts.NewPartitions(int32(len(fixture.partitions))) + require.NoError(t, err) + for index, fixturePartition := range fixture.partitions { + partition := partitions.At(index) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(fixturePartition.deviceType) + ref.SetPartitionId(fixturePartition.id) + switch fixturePartition.deviceType { + case manifestcapnp.DeviceType_lpu: + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", fixturePartition.id))) + require.NoError(t, detail.SetTopology(fixturePartition.topology)) + detail.SetNumChips(fixturePartition.numChips) + detail.SetDevicesPerNode(fixturePartition.devicesPerNode) + if fixturePartition.topologyFamily != "" || fixturePartition.partitionShape != nil { + metadata, err := detail.NewTopologyMetadata() + require.NoError(t, err) + require.NoError(t, metadata.SetTopologyFamily(fixturePartition.topologyFamily)) + shape, err := metadata.NewPartitionShape(int32(len(fixturePartition.partitionShape))) + require.NoError(t, err) + for dimension, extent := range fixturePartition.partitionShape { + shape.Set(dimension, extent) + } + } + case manifestcapnp.DeviceType_cuda: + _, err := partition.Detail().NewCuda() + require.NoError(t, err) + case manifestcapnp.DeviceType_cpu: + _, err := partition.Detail().NewCpu() + require.NoError(t, err) + default: + t.Fatalf("unsupported test device type %q", fixturePartition.deviceType) + } + } + + t.Log("Publish the revision-2 compiler manifest") + data, err := message.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, gbuildManifestV2CapnpFile), data, 0o600)) +} diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go new file mode 100644 index 000000000000..632b3de98d6b --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +const ( + lpuDatacenterConfigFilepath = lpuConfigMountPath + "/datacenter.toml" + lpuModelConfigFilepath = lpuConfigMountPath + "/model_config.toml" + lpuExpandedDatacenterConfigFilepath = runtimeTemporaryStorageMountPath + "/datacenter.toml" + lpuNovaEntrypointScript = `set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" ` + lpuDatacenterConfigFilepath + ` > ` + lpuExpandedDatacenterConfigFilepath + `; exec /bin/nova "$@"` +) + +func updateLPUConductorContainer( + container *corev1.Container, + allocation string, +) { + // Wrap image-only workloads so the role always starts Nova after config expansion. + container.Name = "conductor" + datacenterConfigFilepath := lpuDatacenterConfigFilepath + if len(container.Command) == 0 { + args := container.Args + container.Command = []string{"/bin/bash"} + container.Args = append([]string{"-c", lpuNovaEntrypointScript, "--"}, args...) + datacenterConfigFilepath = lpuExpandedDatacenterConfigFilepath + } + + // Nova and OpenMPI require root in the supported direct-DGD runtime. + if container.SecurityContext == nil { + container.SecurityContext = &corev1.SecurityContext{} + } + container.SecurityContext.RunAsUser = ptr.To(int64(0)) + container.SecurityContext.RunAsGroup = ptr.To(int64(0)) + container.SecurityContext.RunAsNonRoot = ptr.To(false) + + // Append the complete operator-owned Nova launch contract after user arguments. + container.Args = append(container.Args, + "--agent-path", + "/bin/agent", + "--agent-wd", + runtimeTemporaryStorageMountPath, + "--conductor-bind-host", + "0.0.0.0", + "--conductor-hostname", + "$(POD_IP)", + "--control-server-addr", + "$(POD_IP):51045", + + // configmap + "--datacenter-config-filepath", + datacenterConfigFilepath, + "--model-config-filepath", + lpuModelConfigFilepath, + + // logs + "--json-logs", + "--disable-tracing", + + // mpirun + "--mpirun-path", + "/bin/mpirun", + "--mpi-ssh-cmd", + fmt.Sprintf( + "ssh -p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s", + LPUSSHPort, + conductorSSHPrivateKeyPath, + ), + "--mpi-use-tcp", + + // alloc + "--allocation", allocation, + + // conductor/agent lifecycle + "--agent-disable-auto-restart", + "--agent-server-max-retries", + "60", + "--agent-server-retry-sleep-dur", + "2s", + "--start-port", + "12345", + "--graceful-shutdown-timeout", + "0s", + + // misc + "--response-timeout", + "120s", + + // dyn + "--frontend-type", + "dynamo", + "--endpoint-id", + "dyn://$(DYN_NAMESPACE).backend.generate", + "--request-plane", + "tcp", + "--store-backend", + "kubernetes", + ) + + // Allow root OpenMPI to reach the SSH workers with the copied private key. + container.Env = append(container.Env, + corev1.EnvVar{Name: "OMPI_ALLOW_RUN_AS_ROOT", Value: "1"}, + corev1.EnvVar{Name: "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM", Value: "1"}, + corev1.EnvVar{ + Name: "OMPI_MCA_plm_rsh_args", + Value: fmt.Sprintf( + "-p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=%s", + LPUSSHPort, + conductorSSHPrivateKeyPath, + ), + }, + ) + container.VolumeMounts = setVolumeMount(container.VolumeMounts, conductorSSHKeyVolumeMount()) + + // Default readiness to Nova's health endpoint while preserving an explicit probe. + if container.ReadinessProbe == nil { + container.ReadinessProbe = &corev1.Probe{ + FailureThreshold: 3, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + SuccessThreshold: 1, + TimeoutSeconds: 1, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(51045), + Scheme: corev1.URISchemeHTTP, + }, + }, + } + } +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go new file mode 100644 index 000000000000..465a28d45464 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "strconv" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +// configureHybridCyborg consumes a fresh hybrid clique from an admitted source +// whose conductor resources were validated during workload selection. +func configureHybridCyborg( + cyborg *grovev1alpha1.PodCliqueTemplateSpec, + projection *ModelProjection, + workloadDigest string, + modelStorage lpuModelStorage, + agentTemplateNames []string, + plan *MaterializationPlan, + cyborgConfigMap *corev1.ConfigMap, +) error { + container := findMainContainer(cyborg.Spec.PodSpec.Containers) + + // Apply manifest-aware runtime bindings to the selected Cyborg container. + if err := withLPUModelStorage(&cyborg.Spec.PodSpec, container, modelStorage); err != nil { + return err + } + cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, cyborg.Spec.Replicas) + if err != nil { + return err + } + cyborgBatchSize, err = configuredCyborgBatchSize(container, cyborgBatchSize) + if err != nil { + return err + } + if projection.configuredBuild.Family == BuildFamilyXT { + setContainerEnv(container, false, corev1.EnvVar{Name: "RDMA_PORT", Value: strconv.Itoa(lpuRDMAPort)}) + } + applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) + applyCyborgSWACacheIDs(container, cyborgBatchSize) + if err := applyCyborgManifestPath(container, projection, modelStorage.mount.MountPath); err != nil { + return err + } + if err := wrapCyborgDecodeForSwaBatch(container, cyborgBatchSize); err != nil { + return err + } + + cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" + delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) + cyborg.Annotations = roleAnnotations( + cyborg.Annotations, + lpxv1alpha1.PodRoleCyborgWorker, + workloadDigest, + ) + if cyborgConfigMap != nil { + cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) + } + if projection.configuredBuild.Family == BuildFamilyXT { + cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) + } + if plan.ConductorTemplate != "" { + cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, plan.ConductorTemplate) + } + if projection.configuredBuild.Family != BuildFamilyXT { + cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) + } + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go new file mode 100644 index 000000000000..92e97fa57f6c --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { + t.Parallel() + + t.Log("Project a split-I/O selected workload") + fixture := newV3CompilerFixture() + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + fixture.numLPUNodes = 1 + fixture.partitions[0].topology = "URSA_V2_1__Q8__8C__G_106__KP_FEC__GHZ_1_0__NO_FPGA" + fixture.partitions[0].numChips = 8 + fixture.partitions[0].devicesPerNode = 8 + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + build := normalized.build + build.BatchSize = 4 + build.CompilationMode = BuildCompilationModeHybrid + build.IOFPGACount = 2 + build.IOFanoutFactor = 2 + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, Models: []string{"default"}, + BuildSnapshot: normalized, + RuntimeBuildRef: "model-build", ModelSettings: json.RawMessage(`{"prop_sync":false}`), + }) + require.NoError(t, err) + projection := projectionBatch[0] + + t.Log("Render the Cyborg runtime contract") + pcs := renderTestPCS(true) + decode := namedClique(t, pcs, "lpu-engine-gpu") + decode.Spec.Replicas = 4 + decode.Spec.MinAvailable = ptr.To[int32](4) + decode.Spec.PodSpec.ResourceClaims = nil + decode.Spec.PodSpec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + } + decode.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} + decode.Spec.PodSpec.Containers[0].Env = append( + decode.Spec.PodSpec.Containers[0].Env, + corev1.EnvVar{Name: "RDMA_PORT", Value: "12345"}, + corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, + ) + input := RenderInput{ + DGDName: "dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + } + rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, input) + require.NoError(t, err) + cyborg := namedClique(t, rendered, "lpu-engine-gpu") + + t.Log("Verify the rendered Cyborg runtime I/O contract") + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "CYBORG_FPGA_GPI_IO_FPGA_COUNT", Value: "2", + }) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "CYBORG_BATCH_SIZE", Value: "3", + }) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "CYBORG_FPGA_GPI_REPLICA_INDEX", + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels['grove.io/podclique-pod-index']", + }}, + }) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "GBUILD_MANIFEST_PATH", Value: "/models/model-build/manifest.v2.capnp.bin", + }) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "RDMA_PORT", Value: "19877", + }) + require.Contains(t, cyborg.Spec.PodSpec.Volumes, renderTestPodSpec().Volumes[0]) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, renderTestPodSpec().Containers[0].VolumeMounts[0]) + require.Equal(t, []string{"/bin/sh", "-ec"}, cyborg.Spec.PodSpec.Containers[0].Command) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) + require.Equal(t, []string{"--", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args[1:]) + + t.Log("Preserve an image-owned entrypoint") + imageEntrypointPCS := renderTestPCS(true) + imageEntrypointCyborg := namedClique(t, imageEntrypointPCS, "lpu-engine-gpu") + imageEntrypointCyborg.Spec.Replicas = 4 + imageEntrypointCyborg.Spec.MinAvailable = ptr.To[int32](4) + imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args = []string{"serve"} + imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env = append( + imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env, + corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, + ) + + t.Log("Render and verify rejection of an unwrappable image-owned entrypoint") + input.Stages = map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}} + _, err = renderSelectedForTest(imageEntrypointPCS, []*ModelProjection{projection}, input) + require.ErrorContains(t, err, "cyborg command is empty; cannot wrap image entrypoint for SWA batch ID generation") + + t.Log("Reject incomplete endpoint and fanout coverage of the same split-I/O runtime") + + for _, test := range []struct { + name string + replicas int32 + wantError string + }{ + {"incomplete endpoints", 1, "Cyborg replicas 1 must be divisible by ioFpgaCount 2"}, + {"incomplete fanout", 2, "Cyborg replicas 2 must provide fanoutFactor 2 clients"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Reject incomplete Cyborg coverage during runtime rendering") + pcs := renderTestPCS(true) + cyborg := namedClique(t, pcs, "lpu-engine-gpu") + cyborg.Spec.Replicas = test.replicas + cyborg.Spec.MinAvailable = ptr.To(test.replicas) + cyborg.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} + _, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ + DGDName: "dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + }) + require.ErrorContains(t, err, test.wantError) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_resources.go b/deploy/operator/internal/dynamo/lpx/cyborg_resources.go new file mode 100644 index 000000000000..4fc6565b124e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_resources.go @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" +) + +// EffectiveCyborgGPUCount returns the effective exact nvidia.com/gpu count +// consumed by hybrid Cyborg inventory. Limits-only resources are valid because +// Kubernetes defaults their request; an explicit request must equal its limit. +// resources is not mutated. +func EffectiveCyborgGPUCount(resources corev1.ResourceRequirements) (int, error) { + // Select only the classic scalar projected by hybrid Cyborg inventory. + gpuName := corev1.ResourceName(commonconsts.KubeResourceGPUNvidia) + limit, hasLimit := resources.Limits[gpuName] + request, hasRequest := resources.Requests[gpuName] + + // Leave DRA claims and unrelated scalar resources to their owning paths. + if !hasLimit && !hasRequest { + return 0, nil + } + + // Validate an explicit limit before treating it as the effective request. + limitCount := 0 + if hasLimit { + limitValue := limit.Value() + if limit.CmpInt64(limitValue) != 0 { + err := fmt.Errorf("GPU resource %q quantity %q must be a whole number", gpuName, limit.String()) + return 0, fmt.Errorf("Cyborg GPU limit: %w", err) + } + limitCount = int(limitValue) + if limitCount <= 0 { + return 0, fmt.Errorf("Cyborg GPU limit %q must be positive, got %q", gpuName, limit.String()) + } + } + + // Validate an explicit request and require Kubernetes extended-resource equality. + if hasRequest { + requestValue := request.Value() + if request.CmpInt64(requestValue) != 0 { + err := fmt.Errorf("GPU resource %q quantity %q must be a whole number", gpuName, request.String()) + return 0, fmt.Errorf("Cyborg GPU request: %w", err) + } + requestCount := int(requestValue) + if requestCount <= 0 { + return 0, fmt.Errorf("Cyborg GPU request %q must be positive, got %q", gpuName, request.String()) + } + if !hasLimit { + return 0, fmt.Errorf("Cyborg GPU request %q requires an equal limit", gpuName) + } + if request.Cmp(limit) != 0 { + return 0, fmt.Errorf( + "Cyborg GPU request %q must equal its limit, got request %q and limit %q", + gpuName, + request.String(), + limit.String(), + ) + } + } + + // Kubernetes defaults a missing request from the validated limit. + return limitCount, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_resources_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_resources_test.go new file mode 100644 index 000000000000..a0187d909f2d --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_resources_test.go @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestEffectiveCyborgGPUCount(t *testing.T) { + t.Log("Define the exact LPX Cyborg scalar resource contract") + tests := []struct { + name, request, limit, wantErr string + resourceName corev1.ResourceName + want int + }{ + {name: "absent"}, + {name: "MIG does not satisfy classic inventory", resourceName: "nvidia.com/mig-3g.20gb", limit: "1"}, + {name: "limits only", limit: "2", want: 2}, + {name: "equal request and limit", request: "2", limit: "2", want: 2}, + {name: "request without limit", request: "1", wantErr: "requires an equal limit"}, + {name: "request does not equal limit", request: "1", limit: "2", wantErr: "must equal its limit"}, + {name: "fractional limit", limit: "500m", wantErr: "Cyborg GPU limit: GPU resource"}, + {name: "fractional request", request: "500m", limit: "1", wantErr: "Cyborg GPU request: GPU resource"}, + {name: "zero limit", limit: "0", wantErr: "Cyborg GPU limit"}, + {name: "negative request", request: "-1", limit: "1", wantErr: "Cyborg GPU request"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Resolve the effective exact nvidia.com/gpu count") + got, err := EffectiveCyborgGPUCount(testCyborgGPUResources(test.resourceName, test.request, test.limit)) + if test.wantErr != "" { + t.Log("Reject the invalid scalar resource relationship") + require.ErrorContains(t, err, test.wantErr) + return + } + + t.Log("Return the Kubernetes-effective limit count") + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} + +func testCyborgGPUResources(name corev1.ResourceName, request, limit string) corev1.ResourceRequirements { + // Default the resource name and materialize only the maps selected by the table case. + name = corev1.ResourceName(defaultTestString(string(name), commonconsts.KubeResourceGPUNvidia)) + resources := corev1.ResourceRequirements{} + if request != "" { + resources.Requests = corev1.ResourceList{name: resource.MustParse(request)} + } + if limit != "" { + resources.Limits = corev1.ResourceList{name: resource.MustParse(limit)} + } + + return resources +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go new file mode 100644 index 000000000000..a6c4a23eb025 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +const ( + // CyborgBatchSizeEnv configures the batch handled by each Cyborg replica. + CyborgBatchSizeEnv = "CYBORG_BATCH_SIZE" + cyborgFpgaGpiIOFPGACountEnv = "CYBORG_FPGA_GPI_IO_FPGA_COUNT" + cyborgFpgaGpiReplicaIndexEnv = "CYBORG_FPGA_GPI_REPLICA_INDEX" + cyborgSwaCacheIDsEnv = "CYBORG_SWA_CACHE_IDS" + gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" + grovePodCliquePodIndexPath = "metadata.labels['grove.io/podclique-pod-index']" +) + +// applyCyborgManifestPath projects an authoritative manifest location into one Cyborg container. +func applyCyborgManifestPath(container *corev1.Container, projection *ModelProjection, modelStoragePath string) error { + buildRoot, err := buildRuntimePath(lpuRuntimeBuildRef(projection, modelStoragePath), modelStoragePath) + if err != nil { + return fmt.Errorf("resolve GBuild manifest path: %w", err) + } + setContainerEnv(container, false, corev1.EnvVar{ + Name: gbuildManifestPathEnv, + Value: filepath.Join(buildRoot, gbuildManifestV2CapnpFile), + }) + return nil +} + +// applyCyborgRuntimeIO projects the resolved split-I/O fanout into one Cyborg container. +func applyCyborgRuntimeIO(container *corev1.Container, cyborgBatchSize int, ioFPGACount int32) { + setContainerEnv(container, false, + corev1.EnvVar{Name: cyborgFpgaGpiIOFPGACountEnv, Value: strconv.FormatInt(int64(ioFPGACount), 10)}, + corev1.EnvVar{ + Name: cyborgFpgaGpiReplicaIndexEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: grovePodCliquePodIndexPath, + }}, + }, + corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: strconv.Itoa(cyborgBatchSize)}, + ) +} + +// applyCyborgSWACacheIDs binds the replica index for an unbatched Cyborg container. +func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { + if cyborgBatchSize == 1 { + setContainerEnv(container, false, corev1.EnvVar{ + Name: cyborgSwaCacheIDsEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: grovePodCliquePodIndexPath, + }}, + }) + } +} + +const cyborgSwaBatchIDsEntrypointScript = `if [ -z "${CYBORG_SWA_CACHE_IDS+x}" ]; then + : "${CYBORG_FPGA_GPI_REPLICA_INDEX:?CYBORG_FPGA_GPI_REPLICA_INDEX is required}" + : "${CYBORG_BATCH_SIZE:?CYBORG_BATCH_SIZE is required}" + base=$((CYBORG_FPGA_GPI_REPLICA_INDEX * CYBORG_BATCH_SIZE)) + ids="${base}" + i=1 + while [ "${i}" -lt "${CYBORG_BATCH_SIZE}" ]; do + ids="${ids},$((base + i))" + i=$((i + 1)) + done + export CYBORG_SWA_CACHE_IDS="${ids}" +fi +exec "$@" +` + +// wrapCyborgDecodeForSwaBatch derives each batched replica's contiguous SWA cache IDs. +func wrapCyborgDecodeForSwaBatch(container *corev1.Container, cyborgBatchSize int) error { + if cyborgBatchSize == 1 { + return nil + } + if len(container.Command) == 0 { + return fmt.Errorf("cyborg command is empty; cannot wrap image entrypoint for SWA batch ID generation") + } + + // Preserve the complete image command while installing the batch-ID wrapper. + args := slices.Grow( + []string{cyborgSwaBatchIDsEntrypointScript, "--"}, + len(container.Command)+len(container.Args), + ) + args = append(args, container.Command...) + args = append(args, container.Args...) + container.Command = []string{"/bin/sh", "-ec"} + container.Args = args + return nil +} + +// cyborgRuntimeIO requires a nonnil normalized build and validates its Cyborg replica domain. +func cyborgRuntimeIO(build *Build, replicas int32) (int, int32, error) { + // Require every physical endpoint and client-owned transaction in this replica domain. + ioFPGACount := build.IOFPGACount + ioFanoutFactor := build.IOFanoutFactor + if replicas%ioFPGACount != 0 { + return 0, 0, fmt.Errorf("decode service Cyborg replicas %d must be divisible by ioFpgaCount %d", replicas, ioFPGACount) + } + if replicas/ioFPGACount%ioFanoutFactor != 0 { + return 0, 0, fmt.Errorf( + "decode service Cyborg replicas %d must provide fanoutFactor %d clients for each of %d I/O FPGA endpoints", + replicas, + ioFanoutFactor, + ioFPGACount, + ) + } + + // Derive the normalized batch carried by one fanout transaction on one endpoint. + return build.BatchSize / int(ioFPGACount) / int(ioFanoutFactor), ioFPGACount, nil +} + +func configuredCyborgBatchSize(container *corev1.Container, derived int) (int, error) { + for _, variable := range container.Env { + if variable.Name != CyborgBatchSizeEnv { + continue + } + if variable.ValueFrom != nil { + return 0, fmt.Errorf("Cyborg %s must use a literal value", CyborgBatchSizeEnv) + } + configured, err := strconv.Atoi(strings.TrimSpace(variable.Value)) + if err != nil || configured < 1 { + return 0, fmt.Errorf("Cyborg %s must be an integer greater than zero, got %q", CyborgBatchSizeEnv, variable.Value) + } + return configured, nil + } + return derived, nil +} + +// setContainerEnv updates variables in order while optionally preserving the first existing value. +func setContainerEnv(container *corev1.Container, preserveExisting bool, variables ...corev1.EnvVar) { +variables: + for _, variable := range variables { + for index := range container.Env { + if container.Env[index].Name == variable.Name { + if !preserveExisting { + container.Env[index] = variable + } + continue variables + } + } + container.Env = append(container.Env, variable) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go new file mode 100644 index 000000000000..3ab0074c016c --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "os/exec" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" +) + +func TestWrapCyborgDecodeForSwaBatchSkipsBatchOne(t *testing.T) { + t.Parallel() + + t.Log("Construct a single-item Cyborg container entrypoint") + container := &corev1.Container{ + Command: []string{"/usr/local/bin/dynamo_main"}, + Args: []string{"--model", "test"}, + } + original := container.DeepCopy() + + t.Log("Apply the batch-one runtime contract") + require.NoError(t, wrapCyborgDecodeForSwaBatch(container, 1)) + + t.Log("Verify batch one leaves the container unchanged") + require.Equal(t, original, container) +} + +func TestWrapCyborgDecodeForSwaBatch(t *testing.T) { + t.Parallel() + + t.Log("Define computed and explicitly overridden SWA cache ID scenarios") + tests := []struct { + name string + env []string + want string + }{ + { + name: "computes replica-local range with global offset", + env: []string{ + "CYBORG_FPGA_GPI_REPLICA_INDEX=3", + "CYBORG_BATCH_SIZE=2", + }, + want: "6,7", + }, + { + name: "keeps explicit override", + env: []string{ + "CYBORG_FPGA_GPI_REPLICA_INDEX=3", + "CYBORG_BATCH_SIZE=2", + "CYBORG_SWA_CACHE_IDS=100,101", + }, + want: "100,101", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Wrap the actual command and preserve literal argument boundaries") + command := []string{"/bin/sh", "-c"} + args := []string{`printf '%s\n' "${CYBORG_SWA_CACHE_IDS}" "$@"`, "--", "argument with spaces", "literal '$HOME'", ""} + container := &corev1.Container{Command: command, Args: args} + require.NoError(t, wrapCyborgDecodeForSwaBatch(container, 2)) + + t.Log("Changing caller-owned slices must not change the wrapped invocation") + command[0] = "mutated-command" + args[2] = "mutated-argument" + cmd := exec.Command(container.Command[0], append(container.Command[1:], container.Args...)...) + cmd.Env = tt.env + + t.Log("Execute the wrapper and verify cache IDs and forwarded argument bytes") + out, err := cmd.Output() + require.NoError(t, err) + require.Equal(t, tt.want+"\nargument with spaces\nliteral '$HOME'\n\n", string(out)) + }) + } +} + +func TestConfiguredCyborgBatchSizeRejectsInvalidEnvironment(t *testing.T) { + t.Log("Define invalid literal and field-sourced Cyborg batch-size variables") + tests := []corev1.EnvVar{ + {Name: CyborgBatchSizeEnv, Value: "0"}, + {Name: CyborgBatchSizeEnv, Value: "invalid"}, + {Name: CyborgBatchSizeEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + } + for _, variable := range tests { + t.Logf("Reject invalid Cyborg batch-size environment %+v", variable) + _, err := configuredCyborgBatchSize(&corev1.Container{Env: []corev1.EnvVar{variable}}, 1) + require.Error(t, err) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/doc.go b/deploy/operator/internal/dynamo/lpx/doc.go new file mode 100644 index 000000000000..5f21d862a6c6 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/doc.go @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// Package lpx implements LPU build normalization, workload projection, +// runtime configuration, and node-local LPX materialization. +package lpx diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go new file mode 100644 index 000000000000..358a841a48c3 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + selectedCyborgConfigSuffix = "-decode" + selectedCyborgInfiniBandName = "infiniband" + selectedCyborgInfiniBandPath = "/dev/infiniband" + selectedCyborgServerHostsFileEnv = "SERVER_HOSTS_FILE" + selectedCyborgTokenizerDirEnv = "TOKENIZER_DIR" + selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" +) + +func selectedCyborgConfigMapName(dgdName string) string { + return dgdName + selectedCyborgConfigSuffix +} + +// ApplySelectedCyborgContainerDefaults installs the operator-owned V2 Cyborg +// bindings before the user's main-container override is merged. container and +// workload must be non-nil, and workload must contain a projection. The +// function mutates container and reads workload and lpxPodSpec without mutating them. +func ApplySelectedCyborgContainerDefaults( + container *corev1.Container, + workload *SelectedWorkload, + dgdName string, + totalReplicas int32, + lpxPodSpec corev1.PodSpec, +) error { + projection := workload.modelProjections[0] + + configMapName := selectedCyborgConfigMapName(dgdName) + container.VolumeMounts = append( + container.VolumeMounts, + corev1.VolumeMount{Name: selectedCyborgInfiniBandName, MountPath: selectedCyborgInfiniBandPath}, + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + ) + container.Env = append( + container.Env, + corev1.EnvVar{ + Name: selectedCyborgTokenizerDirEnv, + ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, + Key: "tokenizer_dir", + }}, + }, + corev1.EnvVar{Name: selectedCyborgTotalReplicasEnv, Value: strconv.FormatInt(int64(totalReplicas), 10)}, + ) + cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, totalReplicas) + if err != nil { + return err + } + // SWA cache IDs depend on the final, user-overridable batch size and are + // bound only after the main-container override is merged. + applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) + lpxContainer := findMainContainer(lpxPodSpec.Containers) + mountIndex := slices.IndexFunc(lpxContainer.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == commonconsts.ModelStorageVolumeName + }) + if mountIndex < 0 { + return fmt.Errorf("selected LPX main container requires model storage volume mount %q", commonconsts.ModelStorageVolumeName) + } + modelStoragePath := lpxContainer.VolumeMounts[mountIndex].MountPath + if strings.TrimSpace(modelStoragePath) == "" { + return fmt.Errorf("model storage volume %q has no mount path", commonconsts.ModelStorageVolumeName) + } + if err := applyCyborgManifestPath(container, projection, modelStoragePath); err != nil { + return err + } + hostsFile := lpuConfigMountPath + "/lpu_servers" + if workload.engineReplicas > 1 { + // Resolve the replica through the downward API before Kubernetes expands + // SERVER_HOSTS_FILE. The image entrypoint remains untouched. + container.Env = append(container.Env, corev1.EnvVar{ + Name: "LPX_ENGINE_REPLICA", + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels['" + grovecommon.LabelPodCliqueScalingGroupReplicaIndex + "']", + }}, + }) + hostsFile += "-$(LPX_ENGINE_REPLICA)" + } + container.Env = append(container.Env, corev1.EnvVar{ + Name: selectedCyborgServerHostsFileEnv, + Value: hostsFile, + }) + return nil +} + +// ApplySelectedCyborgPodDefaults installs the operator-owned V2 Cyborg +// volumes before the user's PodSpec override is merged. podSpec must be non-nil +// and is mutated in place. +func ApplySelectedCyborgPodDefaults(podSpec *corev1.PodSpec, dgdName string) { + podSpec.Volumes = append( + podSpec.Volumes, + corev1.Volume{ + Name: lpuConfigVolumeName, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: selectedCyborgConfigMapName(dgdName)}, + }}, + }, + corev1.Volume{ + Name: selectedCyborgInfiniBandName, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: selectedCyborgInfiniBandPath, + Type: ptr.To(corev1.HostPathDirectory), + }}, + }, + ) +} + +// renderSelectedCyborgConfigMap requires a nonnil build and at least one Agent template name. +func renderSelectedCyborgConfigMap( + namespace string, + dgdName string, + plan *MaterializationPlan, + modelStoragePath string, + build *Build, +) (*corev1.ConfigMap, error) { + tokenizerDir := "" + if build.Path != "" { + tokenizerPath := build.RuntimeTokenizerPath + if strings.TrimSpace(tokenizerPath) == "" { + return nil, fmt.Errorf("capnp manifest build is missing model.tokenizer.path") + } + runtimePath, runtimeErr := buildRuntimePath(build.Path, modelStoragePath) + if runtimeErr != nil { + return nil, fmt.Errorf("failed to determine tokenizer_dir: %w", runtimeErr) + } + tokenizerDir = filepath.Join(runtimePath, tokenizerPath) + } + + data := map[string]string{"tokenizer_dir": tokenizerDir} + for replica := int32(0); replica < plan.Replicas; replica++ { + // Cyborg supplies the PCS prefix, so only materialize the relative Agent name. + serverPrefix := materializedCliqueNameForReplica(plan.LPXScalingGroupTemplate, plan.Agents[0].TemplateName, replica) + "-" + var servers strings.Builder + offset := 0 + for index, partition := range build.Partitions { + if index != 0 { + servers.WriteByte('\n') + } + servers.WriteString(serverPrefix) + servers.WriteString(strconv.Itoa(offset)) + offset += partition.effectiveNodeCount() + } + key := "lpu_servers" + if plan.Replicas > 1 { + key += "-" + strconv.Itoa(int(replica)) + } + data[key] = servers.String() + } + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: selectedCyborgConfigMapName(dgdName), + Namespace: namespace, + }, + Data: data, + }, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go new file mode 100644 index 000000000000..f203859eea55 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "path/filepath" + "strconv" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + "github.com/stretchr/testify/require" +) + +func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { + t.Parallel() + + t.Log("Build a hybrid V2 workload with two uncollapsed runtime partitions") + fixture := newV2CompilerFixture() + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + fixture.selectedPropSyncChains = nil + snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, + Models: []string{"default"}, + RuntimeBuildRef: "model-build", + BuildSnapshot: normalizeTestSnapshot(t, snapshot), + ModelSettings: json.RawMessage(`{"prop_sync":false}`), + }) + require.NoError(t, err) + projection := projectionBatch[0] + projection.stage = testRenderComponentName + workload := &SelectedWorkload{ + modelProjections: []*ModelProjection{projection}, + engineReplicas: 1, + } + + t.Log("Resolve the projected runtime path") + plan, err := workload.PlanNodeLocalMaterialization("test-dgd-lpx") + require.NoError(t, err) + runtimePath, err := buildRuntimePath(projection.configuredBuild.Path, "/models") + require.NoError(t, err) + + t.Log("Verify every engine replica addresses only its own Agents") + for _, replicas := range []int32{1, 3, 9} { + t.Run(strconv.Itoa(int(replicas)), func(t *testing.T) { + workload.engineReplicas = replicas + plan, err := workload.PlanNodeLocalMaterialization("test-dgd-lpx") + require.NoError(t, err) + configMap, err := renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + require.NoError(t, err) + require.Len(t, configMap.Data, int(replicas)+1) + for replica := int32(0); replica < replicas; replica++ { + key := "lpu_servers" + if replicas > 1 { + key += "-" + strconv.Itoa(int(replica)) + } + prefix := plan.LPXScalingGroupTemplate + "-" + strconv.Itoa(int(replica)) + "-" + require.Equal(t, prefix+"lpu-wkr-m-0-0\n"+prefix+"lpu-wkr-m-0-2", configMap.Data[key]) + } + require.Equal(t, filepath.Join(runtimePath, "tokenizer"), configMap.Data["tokenizer_dir"]) + }) + } + + t.Log("Render a Cap'n Proto build with a nested tokenizer path") + projection.configuredBuild.RuntimeTokenizerPath = "metadata/tokenizer" + configMap, err := renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + + t.Log("Verify the nested tokenizer path is rooted in the runtime build") + require.NoError(t, err) + require.Equal(t, filepath.Join(runtimePath, "metadata/tokenizer"), configMap.Data["tokenizer_dir"]) + + t.Log("Render a Cap'n Proto build without tokenizer metadata") + projection.configuredBuild.RuntimeTokenizerPath = "" + projection.configuredBuild.Path = "relative/build" + _, err = renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + + t.Log("Verify missing tokenizer metadata is rejected") + require.EqualError(t, err, "capnp manifest build is missing model.tokenizer.path") +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands.go b/deploy/operator/internal/dynamo/lpx/lpu_commands.go new file mode 100644 index 000000000000..77dfc110a01e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_commands.go @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + _ "embed" + "fmt" + "strconv" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // LPUSSHPort is the SSH port used by Nova and LPU Agent workers. + LPUSSHPort = commonconsts.MpiRunSshPort + // LPUReadinessPort is the HTTP readiness port used by the LPU runtime. + LPUReadinessPort = 19878 + lpuRDMAPort = 19877 + + lpuCommandTemplateTokenPrefix = "@@LPX_" + lpuConfigMountPathToken = "@@LPX_CONFIG_MOUNT_PATH@@" + lpuSSHPortToken = "@@LPX_SSH_PORT@@" + lpuReadinessPortToken = "@@LPX_READINESS_PORT@@" + lpuRDMAPortToken = "@@LPX_RDMA_PORT@@" + + embeddedShellSPDXHeader = `# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +` + + lpuPartitionSetEnvCommands = `SETENV_LINE="${SETENV_LINE} PARTITION_ID=\"$PARTITION_ID\"" +SETENV_LINE="${SETENV_LINE} RANK_IN_PARTITION=\"$RANK_IN_PARTITION\"" +` + + lpuV2ProbeCommandTemplate = `if (( NODE_COUNT > 1 )) && [[ "${PARTITION_RANK}" != "0" ]]; then + cat < /dev/null > "/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-@@LPX_SSH_PORT@@}" +else + exec 3<>"/dev/tcp/127.0.0.1/${READINESS_PORT:-@@LPX_READINESS_PORT@@}" + printf "GET /readyz HTTP/1.0\r\n\r\n" >&3 + read -r STATUS <&3 + [[ "${STATUS}" == *" 200 "* ]] +fi` +) + +var ( + //go:embed commands/ssh_setup.sh + sshSetupCommandSource string + //go:embed commands/lpu_system_init.sh + lpuSystemInitCommandSource string + //go:embed commands/lpu_partition_metadata.sh + lpuPartitionMetadataCommandSource string + //go:embed commands/lpu_partition_env.sh + lpuPartitionEnvironmentCommandSource string + //go:embed commands/lpu_partition_agent.sh + lpuPartitionRunCommandSource string + //go:embed commands/lpu_worker.sh + lpuWorkerRunCommandSource string + + lpuCommandTemplateReplacer = strings.NewReplacer( + lpuConfigMountPathToken, lpuConfigMountPath, + lpuSSHPortToken, strconv.Itoa(LPUSSHPort), + lpuReadinessPortToken, strconv.Itoa(LPUReadinessPort), + lpuRDMAPortToken, strconv.Itoa(lpuRDMAPort), + ) + + sshSetupCommands = mustExpandLPUCommandTemplate(sshSetupCommandSource) + lpuSystemInitCommands = mustExpandLPUCommandTemplate(lpuSystemInitCommandSource) + lpuPartitionMetadataCommand = mustExpandLPUCommandTemplate(lpuPartitionMetadataCommandSource) + lpuPartitionEnvironmentCommand = mustExpandLPUCommandTemplate(lpuPartitionEnvironmentCommandSource) + lpuPartitionRunCommand = lpuSystemInitCommands + "\n" + lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + "\n" + mustExpandLPUCommandTemplate(lpuPartitionRunCommandSource) + lpuWorkerRunCommandTemplate = mustExpandLPUCommandTemplate(lpuWorkerRunCommandSource) + lpuV2ProbeCommand = lpuPartitionMetadataCommand + "\n" + mustExpandLPUCommandTemplate(lpuV2ProbeCommandTemplate) + + lpuWorkerRunCommand = lpuSystemInitCommands + "\n" + fmt.Sprintf(lpuWorkerRunCommandTemplate, sshSetupCommands, "") + lpuPartitionWorkerRunCommand = lpuSystemInitCommands + "\n" + lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + "\n" + fmt.Sprintf(lpuWorkerRunCommandTemplate, sshSetupCommands, lpuPartitionSetEnvCommands) +) + +func mustExpandLPUCommandTemplate(commandTemplate string) string { + // Remove only the standard source header so adding SPDX does not roll rendered workloads. + commandTemplate = strings.TrimPrefix(commandTemplate, embeddedShellSPDXHeader) + + // Resolve every Go-owned default through the package's single source of truth. + command := lpuCommandTemplateReplacer.Replace(commandTemplate) + if strings.Contains(command, lpuCommandTemplateTokenPrefix) { + panic("unexpanded LPX command template token") + } + return command +} + +const ( + lpuV2HTTPStartupFailureThreshold int32 = 60 + lpuV2ExecStartupFailureThreshold int32 = 240 +) + +func lpuV2StartupProbe(useHTTP bool) *corev1.Probe { + probe := lpuV2ReadinessProbe(useHTTP) + if useHTTP { + probe.FailureThreshold = lpuV2HTTPStartupFailureThreshold + } else { + probe.FailureThreshold = lpuV2ExecStartupFailureThreshold + } + return probe +} + +func lpuV2ReadinessProbe(useHTTP bool) *corev1.Probe { + probe := &corev1.Probe{ + FailureThreshold: 6, + PeriodSeconds: 5, + SuccessThreshold: 1, + TimeoutSeconds: 1, + } + if useHTTP { + probe.HTTPGet = &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(LPUReadinessPort), + Scheme: corev1.URISchemeHTTP, + } + } else { + probe.Exec = &corev1.ExecAction{ + Command: []string{"/bin/bash", "-c", lpuV2ProbeCommand}, + } + } + return probe +} + +func lpuSSHProbe() *corev1.Probe { + return &corev1.Probe{ + FailureThreshold: 3, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + SuccessThreshold: 1, + TimeoutSeconds: 1, + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(LPUSSHPort), + }, + }, + } +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go b/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go new file mode 100644 index 000000000000..be7b74bf10b4 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLPUPartitionMetadataDerivesConfigRowAndRank(t *testing.T) { + t.Parallel() + + t.Log("Resolve the Bash interpreter used by the embedded metadata command") + bash, err := exec.LookPath("bash") + require.NoError(t, err) + + t.Log("Define single-model, SpecDecode, and invalid pod-index metadata fixtures") + tests := []struct { + name string + files map[string]string + podIndex string + model string + want string + wantError string + }{ + { + name: "single model", + files: map[string]string{ + "nodes_per_partition": "2\n1\n", + "partition_node_offsets": "0\n2\n", + }, + podIndex: "1", + want: "0,0,1,2,0,0,1", + }, + { + name: "specdecode model with reset offsets", + files: map[string]string{ + "nodes_per_partition": "2\n2\n2\n2\n", + "partition_node_offsets": "0\n2\n0\n2\n", + "partition_models": "draft0\ndraft0\ntarget\ntarget\n", + "partition_indices": "0\n1\n0\n1\n", + }, + podIndex: "1", + model: "target", + want: "2,0,1,2,0,0,1", + }, + { + name: "rejects out of range pod index", + files: map[string]string{ + "nodes_per_partition": "2\n1\n", + "partition_node_offsets": "0\n2\n", + }, + podIndex: "3", + wantError: "matched 0 partition rows", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Write the node-local partition metadata files") + configDir := t.TempDir() + for name, contents := range tt.files { + require.NoError(t, os.WriteFile(filepath.Join(configDir, name), []byte(contents), 0o600)) + } + + t.Log("Execute the embedded metadata and environment commands") + command := lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + ` +printf '%s,%s,%s,%s,%s,%s,%s' "${CONFIG_PARTITION_INDEX}" "${LOGICAL_PARTITION_INDEX}" "${PARTITION_RANK}" "${NODE_COUNT}" "${NODE_OFFSET}" "${PARTITION_ID}" "${RANK_IN_PARTITION}"` + cmd := exec.Command(bash, "-c", command) + cmd.Env = []string{ + "LPU_CONFIG_DIR=" + configDir, + "GROVE_PCLQ_POD_INDEX=" + tt.podIndex, + "LPU_MODEL_NAME=" + tt.model, + } + out, err := cmd.CombinedOutput() + if tt.wantError != "" { + t.Log("Verify invalid partition metadata is rejected") + require.Error(t, err) + require.Contains(t, string(out), tt.wantError) + return + } + + t.Log("Verify the derived configuration row, logical partition, and rank") + require.NoError(t, err, string(out)) + require.Equal(t, tt.want, string(out)) + }) + } +} + +func TestLPUCommandDigests(t *testing.T) { + t.Parallel() + + t.Log("Define reviewed SHA-256 identities for assembled runtime commands") + tests := []struct { + name string + command string + want string + }{ + {name: "partition", command: lpuPartitionRunCommand, want: "d6f8a2ce0f1031bcf8e08d0aff75e8aed33cbd07aae1fd3e09d813a03b879911"}, + {name: "worker", command: lpuWorkerRunCommand, want: "a6c8c03474d3feb459a329719ee961e2488a5e4886581adc10deec0691b103aa"}, + {name: "partition worker", command: lpuPartitionWorkerRunCommand, want: "65b022e0e8785f0ce2f5ff49d798db40cf8e538d310f3044473088e78d655886"}, + {name: "probe", command: lpuV2ProbeCommand, want: "96ac74ae5e413a21fe98077f4d3d5b1dba7a741064e2734ae822ad77ab30e854"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Bind the assembled embedded command to its reviewed SHA-256 digest") + require.Equal(t, test.want, fmt.Sprintf("%x", sha256.Sum256([]byte(test.command)))) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go new file mode 100644 index 000000000000..9920996346c4 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -0,0 +1,427 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "crypto/sha256" + "fmt" + "maps" + "net/url" + "path/filepath" + "slices" + "strconv" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/pelletier/go-toml/v2" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + lpuConfigVolumeName = "config" + lpuConfigMountPath = "/configs" +) + +type lpuModelStorage struct { + volume corev1.Volume + mount corev1.VolumeMount +} + +func renderLPUConfigMap( + namespace string, + dgdName string, + modelStoragePath string, + projections []*ModelProjection, + agents []ExpectedAgent, +) (*corev1.ConfigMap, error) { + modelConfig, err := lpuModelConfig(projections, modelStoragePath) + if err != nil { + return nil, fmt.Errorf("render model_config.toml: %w", err) + } + var modelTOML strings.Builder + if err := toml.NewEncoder(&modelTOML).Encode(modelConfig); err != nil { + return nil, fmt.Errorf("render model_config.toml: %w", err) + } + + datacenterRacks := make(map[string]any, len(agents)) + for _, agent := range agents { + availableNodes := make([]int, agent.Replicas) + for index := range availableNodes { + availableNodes[index] = index + } + datacenterRacks[agent.TemplateName] = map[string]any{ + "available_nodes": availableNodes, + "preferred_start": 0, + "node_name_template": fmt.Sprintf( + "%s-%s-{node}.${GROVE_HEADLESS_SERVICE}", + "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}", + agent.TemplateName, + ), + } + } + var datacenterTOML strings.Builder + _ = toml.NewEncoder(&datacenterTOML).Encode(map[string]any{ + "model_base_paths": []string{modelStoragePath}, + "datacenters": map[string]any{"racks": datacenterRacks}, + }) + + data := resolvedPartitionData(projections) + if projections[0].pipeline == PipelineLPX { + data["gas_dir"] = modelConfig["iop"].(map[string]any)["model_path"].(string) + } + data["model_config.toml"] = modelTOML.String() + data["datacenter.toml"] = datacenterTOML.String() + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgdName + "-lpu", + Namespace: namespace, + }, + Data: data, + }, nil +} + +// LPUConfigMapHash returns the hash stamped on LPU runtime Pod templates. +// configMap must be non-nil, with valid native metadata from rendering or Kubernetes. +// Its concrete content is always serializable; the input is not mutated. +func LPUConfigMapHash(configMap *corev1.ConfigMap) string { + contentHash, _ := controllercommon.GetSpecHash(configMap) + + // The graph's extra-resource annotation hashes each resource's spec hash. + // Apply the same second hash here so LPX can render that annotation directly. + return fmt.Sprintf("%x", sha256.Sum256([]byte(contentHash))) +} + +func lpuModelStorageBinding(spec corev1.PodSpec) (lpuModelStorage, error) { + container := findMainContainer(spec.Containers) + mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == commonconsts.ModelStorageVolumeName + }) + if mountIndex < 0 { + return lpuModelStorage{}, fmt.Errorf( + "selected LPX main container requires model storage volume mount %q", + commonconsts.ModelStorageVolumeName, + ) + } + mount := container.VolumeMounts[mountIndex] + if strings.TrimSpace(mount.MountPath) == "" { + return lpuModelStorage{}, fmt.Errorf("model storage volume %q has no mount path", mount.Name) + } + volumeIndex := slices.IndexFunc(spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) + if volumeIndex < 0 { + return lpuModelStorage{}, fmt.Errorf("selected LPX podTemplate has no model storage volume %q", mount.Name) + } + return lpuModelStorage{volume: spec.Volumes[volumeIndex], mount: mount}, nil +} + +// withLPUModelStorage requires a nonnil spec and a unique container retained from its unchanged Containers slice. +func withLPUModelStorage(spec *corev1.PodSpec, container *corev1.Container, storage lpuModelStorage) error { + volumeIndex := slices.IndexFunc(spec.Volumes, func(existing corev1.Volume) bool { return existing.Name == storage.volume.Name }) + if volumeIndex < 0 { + spec.Volumes = append(spec.Volumes, storage.volume) + } else if !apiequality.Semantic.DeepEqual(spec.Volumes[volumeIndex], storage.volume) { + return fmt.Errorf("selected Cyborg podTemplate volume %q conflicts with model storage", storage.volume.Name) + } + + mountIndex := slices.IndexFunc(container.VolumeMounts, func(existing corev1.VolumeMount) bool { + return existing.Name == storage.mount.Name || existing.MountPath == storage.mount.MountPath + }) + if mountIndex < 0 { + container.VolumeMounts = append(container.VolumeMounts, storage.mount) + } else if !apiequality.Semantic.DeepEqual(container.VolumeMounts[mountIndex], storage.mount) { + return fmt.Errorf("selected Cyborg main container conflicts with model storage mount %q", storage.mount.MountPath) + } + return nil +} + +func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (map[string]any, error) { + if projections[0].pipeline != PipelineSpecDecode { + config, err := nestedLPUModelConfig(projections[0], modelStoragePath) + if err != nil { + return nil, err + } + config["type"] = "Single" + return config, nil + } + + // Derive the pipeline-wide speculative-decoding controls from the draft model. + config := make(map[string]any) + + // Promote the draft's setup timeouts so Nova applies them to every model. + setup, err := modelObjectSetting(projections[0], "setup") + if err != nil { + return nil, err + } + delete(setup, "setup_ops_format") + delete(setup, "resolved_partitions_dir") + if len(setup) > 0 { + config["setup"] = setup + } + + // Promote the only model-provided speculative-decoding scalar currently consumed by the runtime. + if value, present := projections[0].configuredBuild.runtimeSettings[v3MaxSWADKVCBlocksDraft]; present { + config[v3MaxSWADKVCBlocksDraft] = value + } + + draftCount := len(projections) - 1 + draft, err := nestedLPUModelConfig(projections[0], modelStoragePath) + if err != nil { + return nil, err + } + // Draft setup values configure the SpecDecode parent. Keep the child setup + // limited to the operator-owned LPU runtime bindings so Nova can inherit the + // parent timeouts exactly as it did with the legacy pipeline field. + draft["setup"] = map[string]any{ + "setup_ops_format": agentSetupOpsFormat, + "resolved_partitions_dir": lpuConfigMountPath, + } + target, err := nestedLPUModelConfig(projections[draftCount], modelStoragePath) + if err != nil { + return nil, err + } + config["type"] = "SpecDecode" + config["draft"] = draft + config["target"] = target + config["num_drafts"] = draftCount + setConfigDefault(config, "draft_to_target_port", 23456) + setConfigDefault(config, "target_to_draft_port", 23457) + setConfigDefault(config, "head_to_head_port", 23458) + return config, nil +} + +func nestedLPUModelConfig(projection *ModelProjection, modelStoragePath string) (map[string]any, error) { + settings := maps.Clone(projection.configuredBuild.runtimeSettings) + + // Lift Nova's model-level sections out of the IOP settings object. + scheduler, err := modelObjectSetting(projection, "scheduler") + if err != nil { + return nil, err + } + if scheduler == nil { + scheduler = map[string]any{} + } + delete(settings, "scheduler") + setup, err := modelObjectSetting(projection, "setup") + if err != nil { + return nil, err + } + if setup == nil { + setup = map[string]any{} + } + delete(settings, "setup") + + // Keep operator-owned LPU runtime setup bindings authoritative. + setup["setup_ops_format"] = agentSetupOpsFormat + setup["resolved_partitions_dir"] = lpuConfigMountPath + delete(settings, v3MaxSWADKVCBlocksDraft) + + // Lift additional programs into the Nova model object when configured. + var extraPrograms []any + if configured, present := settings["extra_programs"]; present { + var ok bool + extraPrograms, ok = configured.([]any) + if !ok { + return nil, fmt.Errorf("model %q settings.extra_programs must be an array", projection.model) + } + delete(settings, "extra_programs") + if projection.configuredBuild.Family == BuildFamilyXT && + projection.pipeline == PipelineSingle && len(extraPrograms) > 0 { + return nil, fmt.Errorf("model %q settings.extra_programs is not supported with setup_ops_format %q", projection.model, agentSetupOpsFormat) + } + } + + build := &projection.configuredBuild + buildRef := build.Path + if projection.pipeline == PipelineLPX || + projection.configuredBuild.Family == BuildFamilyHX { + buildRef = lpuRuntimeBuildRef(projection, modelStoragePath) + } + modelPath, err := buildRuntimePath(buildRef, modelStoragePath) + if err != nil { + return nil, fmt.Errorf("model %q model_path: %w", projection.model, err) + } + if _, overridden := settings["tokenizer_path"]; !overridden && + strings.TrimSpace(build.RuntimeTokenizerPath) != "" { + settings["tokenizer_path"] = filepath.Join(modelPath, build.RuntimeTokenizerPath) + } + if projection.configuredBuild.Family == BuildFamilyHX || + projection.pipeline == PipelineSingle { + if err := validateRuntimeTokenizerSettings(settings); err != nil { + return nil, fmt.Errorf("model %q settings: %w", projection.model, err) + } + } + if build.SupportsCPUEmbeddings { + if enabled, _ := settings["cpu_embeddings"].(bool); enabled { + if _, overridden := settings["embedding_path"]; !overridden && + strings.TrimSpace(build.RuntimeTokenEmbeddingsPath) != "" { + settings["embedding_path"] = filepath.Join(modelPath, build.RuntimeTokenEmbeddingsPath) + } + } + } + if projection.configuredBuild.Family == BuildFamilyXT && + projection.pipeline == PipelineSpecDecode { + setConfigDefault(settings, "model_path", modelPath) + } else { + settings["model_path"] = modelPath + } + config := map[string]any{ + "scheduler": scheduler, + "setup": setup, + "iop": settings, + } + if extraPrograms != nil { + config["extra_programs"] = extraPrograms + } + return config, nil +} + +func modelObjectSetting(projection *ModelProjection, name string) (map[string]any, error) { + configured, present := projection.configuredBuild.runtimeSettings[name] + if !present { + return nil, nil + } + object, ok := configured.(map[string]any) + if !ok { + return nil, fmt.Errorf("model %q settings.%s must be an object", projection.model, name) + } + return maps.Clone(object), nil +} + +func lpuRuntimeBuildRef(projection *ModelProjection, modelStoragePath string) string { + buildRef := projection.configuredBuild.Path + snapshotRef, snapshotErr := url.Parse(buildRef) + runtimeRef := strings.TrimSpace(projection.runtimeBuildRef) + runtimeURL, runtimeErr := url.Parse(runtimeRef) + if snapshotErr != nil || runtimeErr != nil || snapshotRef.Scheme != BuildSchemeFile || + runtimeRef == "" || runtimeURL.Scheme != "" || filepath.IsAbs(runtimeRef) { + return buildRef + } + cleaned := filepath.Clean(runtimeRef) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) { + return buildRef + } + return (&url.URL{Scheme: BuildSchemeFile, Path: filepath.Join(modelStoragePath, cleaned)}).String() +} + +func resolvedPartitionData(projections []*ModelProjection) map[string]string { + keys := [...]string{"nodes_per_partition", "partition_indices", "partition_ids", "partition_models", + "partition_node_offsets", "partition_paths", "topologies"} + + // Omit model-identity columns that the XT Single runtime never consumes. + includeModelColumns := projections[0].configuredBuild.Family != BuildFamilyXT || + projections[0].pipeline != PipelineSingle + var columns [len(keys)]strings.Builder + + // Accumulate each projection's runtime partitions into the surviving columns. + for _, projection := range projections { + // Render configured V2 runtime metadata without reapplying scheduler-shape validation to collapsed partitions. + partitions := projection.partitions + v2Runtime := projection.configuredBuild.Family == BuildFamilyXT && + len(projection.configuredBuild.Partitions) != 0 + if v2Runtime { + partitions = projection.configuredBuild.Partitions + } + + offset := int64(0) + for index, partition := range partitions { + var nodes string + var endpointCount int64 + if v2Runtime { + nodeCount := partition.effectiveNodeCount() + nodes, endpointCount = strconv.Itoa(nodeCount), int64(nodeCount) + } else { + endpointCount = partition.HXExtent[1] * partition.HXExtent[2] * partition.HXExtent[3] + nodes = strconv.FormatInt(endpointCount, 10) + } + // Project one row and populate optional model identity only when consumed. + row := [len(keys)]string{nodes, "", + strconv.FormatUint(uint64(uint32(partition.SourcePartitionID)), 10), "", + strconv.FormatInt(offset, 10), partition.PartPath, partition.Topology.Raw} + if includeModelColumns { + row[1] = strconv.Itoa(index) + row[3] = projection.model + } + + // Write only columns that survive into the ConfigMap. + for column := range row { + if !includeModelColumns && (column == 1 || column == 3) { + continue + } + columns[column].WriteString(row[column]) + columns[column].WriteByte('\n') + } + offset += endpointCount + } + } + data := make(map[string]string, len(keys)) + + // Materialize only the runtime-visible columns. + for column := range keys { + if !includeModelColumns && (column == 1 || column == 3) { + continue + } + data[keys[column]] = strings.TrimSuffix(columns[column].String(), "\n") + } + return data +} + +func setConfigDefault(values map[string]any, key string, value any) { + if _, present := values[key]; !present { + values[key] = value + } +} + +func withLPUConfigVolume(spec *corev1.PodSpec, configMapName string, allowOverrides bool) error { + found := false + for _, volume := range spec.Volumes { + if volume.Name != lpuConfigVolumeName { + continue + } + if !allowOverrides && (found || + volume.ConfigMap == nil || + volume.ConfigMap.Name != configMapName || + len(volume.ConfigMap.Items) != 0 || + volume.ConfigMap.DefaultMode != nil || + volume.ConfigMap.Optional != nil) { + return fmt.Errorf("selected LPX podTemplate volume %q is reserved for ConfigMap %q", lpuConfigVolumeName, configMapName) + } + found = true + } + if !found { + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: lpuConfigVolumeName, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, + }}, + }) + } + container := findMainContainer(spec.Containers) + found = false + for _, mount := range container.VolumeMounts { + if mount.Name == lpuConfigVolumeName || mount.MountPath == lpuConfigMountPath { + if !allowOverrides && (found || + mount.Name != lpuConfigVolumeName || + mount.MountPath != lpuConfigMountPath || + mount.ReadOnly || + mount.RecursiveReadOnly != nil || + mount.SubPath != "" || + mount.MountPropagation != nil || + mount.SubPathExpr != "") { + return fmt.Errorf("selected LPX main container reserves volume %q at %q", lpuConfigVolumeName, lpuConfigMountPath) + } + found = found || mount.MountPath == lpuConfigMountPath + } + } + if !found { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: lpuConfigVolumeName, + MountPath: lpuConfigMountPath, + }) + } + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go new file mode 100644 index 000000000000..e4ac6bbcebec --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLPURuntimeBuildRef(t *testing.T) { + t.Parallel() + + t.Log("Define runtime build-reference selection contracts") + tests := []struct { + name string + snapshot string + runtime string + want string + }{ + {name: "relative runtime", snapshot: "file:///snapshot", runtime: "model-build", want: "file:///models/model-build"}, + {name: "cleaned runtime", snapshot: "file:///snapshot", runtime: " a/../b ", want: "file:///models/b"}, + {name: "GCS snapshot", snapshot: "gs://bucket/snapshot", runtime: "model-build", want: "gs://bucket/snapshot"}, + {name: "malformed snapshot", snapshot: "%", runtime: "model-build", want: "%"}, + {name: "empty runtime", snapshot: "file:///snapshot", want: "file:///snapshot"}, + {name: "malformed runtime", snapshot: "file:///snapshot", runtime: "%", want: "file:///snapshot"}, + {name: "runtime URL", snapshot: "file:///snapshot", runtime: "gs://bucket/build", want: "file:///snapshot"}, + {name: "absolute runtime", snapshot: "file:///snapshot", runtime: "/model-build", want: "file:///snapshot"}, + {name: "dot runtime", snapshot: "file:///snapshot", runtime: ".", want: "file:///snapshot"}, + {name: "parent runtime", snapshot: "file:///snapshot", runtime: "..", want: "file:///snapshot"}, + {name: "escaping runtime", snapshot: "file:///snapshot", runtime: "a/../../b", want: "file:///snapshot"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Select the runtime build reference") + got := lpuRuntimeBuildRef(&ModelProjection{ + runtimeBuildRef: test.runtime, + configuredBuild: Build{Path: test.snapshot}, + }, "/models") + + t.Log("Preserve valid remapping and every fallback byte exactly") + require.Equal(t, test.want, got) + }) + } +} + +func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { + t.Parallel() + + t.Log("Construct a V2 hybrid projection with an image-owned runtime build") + projection := &ModelProjection{ + model: "default", + pipeline: PipelineLPX, + runtimeBuildRef: "model-build", + configuredBuild: Build{ + Path: "file:///snapshot-build", + Family: BuildFamilyXT, + runtimeSettings: map[string]any{}, + }, + } + + t.Log("Render the hybrid ConfigMap from the selected pipeline") + configMap, err := renderLPUConfigMap( + "test-namespace", + "test-dgd", + "/models", + []*ModelProjection{projection}, + nil, + ) + + t.Log("Keep gas_dir rooted at the selected image-owned build") + require.NoError(t, err) + require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") + require.Equal(t, "/models/model-build", configMap.Data["gas_dir"]) + + t.Log("Render the preserved runtime with an invalid snapshot build reference") + projection.configuredBuild.Path = "relative-build" + _, err = renderLPUConfigMap( + "test-namespace", + "test-dgd", + "/models", + []*ModelProjection{projection}, + nil, + ) + + t.Log("Return the model-path error before projecting gas_dir") + require.ErrorContains(t, err, "render model_config.toml: model \"default\" model_path: parse build path \"relative-build\": ref \"relative-build\" must be an absolute path or URL") +} + +func TestResolvedPartitionDataOmitsXTModelColumnsBeforeMaterialization(t *testing.T) { + t.Parallel() + + t.Log("Construct an XT Single projection with two physical runtime partitions") + projection := &ModelProjection{ + model: "default", + pipeline: PipelineSingle, + configuredBuild: Build{ + Family: BuildFamilyXT, + Partitions: []BuildPartition{ + {SourcePartitionID: 7, PartPath: "part-7", Topology: Topology{ChipCount: 16, Raw: "topology-7"}}, + {SourcePartitionID: 9, PartPath: "part-9", Topology: Topology{ChipCount: 8, Raw: "topology-9"}}, + }, + }, + } + + t.Log("Project only the five columns consumed by the XT Single runtime") + data := resolvedPartitionData([]*ModelProjection{projection}) + + t.Log("Verify omitted model columns never enter the final map and retained bytes remain exact") + require.Equal(t, map[string]string{ + "nodes_per_partition": "2\n1", + "partition_ids": "7\n9", + "partition_node_offsets": "0\n2", + "partition_paths": "part-7\npart-9", + "topologies": "topology-7\ntopology-9", + }, data) +} + +func TestV3SingleConfigUsesManifestTopologyAndArbitraryModelSettings(t *testing.T) { + t.Parallel() + + t.Log("Project the V3 manifest with arbitrary deployment-time model settings") + snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, RuntimeBuildRef: "model-build", BuildSnapshot: normalizeTestSnapshot(t, snapshot), + ModelSettings: json.RawMessage(`{ + "batch_size":4, + "custom_runtime_knob":"enabled", + "scheduler":{"max_inflight_tasks":3}, + "tokenizer_path":"/nfs/models/gemma4-31b", + "stop_tokens":[1] + }`), + }) + require.NoError(t, err) + projection := projectionBatch[0] + + t.Log("Render the complete LPU ConfigMap") + configMap, err := renderLPUConfigMap( + "test", + "test-dgd", + "/models", + []*ModelProjection{projection}, + []ExpectedAgent{{TemplateName: "rack", Replicas: 1}}, + ) + require.NoError(t, err) + + t.Log("Verify the generated files need no manual topology or model-setting patches") + require.Equal(t, "part-1", configMap.Data["partition_paths"]) + require.Equal(t, v3OpaqueTopology, configMap.Data["topologies"]) + require.Contains(t, configMap.Data["model_config.toml"], "sequence_length = 8192") + require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") + require.Contains(t, configMap.Data["model_config.toml"], "batch_size = 4") + require.Contains(t, configMap.Data["model_config.toml"], "custom_runtime_knob = 'enabled'") + require.Contains(t, configMap.Data["model_config.toml"], "max_inflight_tasks = 3") + require.Contains(t, configMap.Data["model_config.toml"], "tokenizer_path = '/nfs/models/gemma4-31b'") + require.Contains(t, configMap.Data["model_config.toml"], "stop_tokens = [1]") +} + +func TestNestedLPUModelConfigJoinsRuntimeAssetsToRemappedRoot(t *testing.T) { + t.Log("Construct a projection with build-relative runtime assets") + projection := &ModelProjection{ + model: "default", pipeline: PipelineLPX, runtimeBuildRef: "model-build", + configuredBuild: Build{ + Path: "file:///snapshot-build", SupportsCPUEmbeddings: true, + RuntimeTokenizerPath: ".", RuntimeTokenEmbeddingsPath: "runtime/text_embeddings.npz", + runtimeSettings: map[string]any{"cpu_embeddings": true}, + }, + } + + t.Log("Render model configuration under the remapped runtime root") + config, err := nestedLPUModelConfig(projection, "/models") + + t.Log("Verify default runtime asset paths are joined to the remapped build") + require.NoError(t, err) + iop := config["iop"].(map[string]any) + require.Equal(t, "/models/model-build", iop["model_path"]) + require.Equal(t, "/models/model-build", iop["tokenizer_path"]) + require.Equal(t, "/models/model-build/runtime/text_embeddings.npz", iop["embedding_path"]) + + t.Log("Render an explicit absolute embedding-path override") + projection.configuredBuild.runtimeSettings["embedding_path"] = "/custom/embeddings" + config, err = nestedLPUModelConfig(projection, "/models") + + t.Log("Verify the absolute override remains unchanged") + require.NoError(t, err) + require.Equal(t, "/custom/embeddings", config["iop"].(map[string]any)["embedding_path"]) +} + +func TestV2SingleConfigRejectsExtraPrograms(t *testing.T) { + t.Log("Render a V2 single-model configuration with unsupported extra programs") + _, err := lpuModelConfig([]*ModelProjection{{ + model: "default", + pipeline: PipelineSingle, + configuredBuild: Build{ + Family: BuildFamilyXT, + runtimeSettings: map[string]any{"extra_programs": []any{"unsupported"}}, + }, + }}, "/models") + + t.Log("Verify the unsupported runtime setting is rejected") + require.ErrorContains(t, err, "settings.extra_programs is not supported") +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go new file mode 100644 index 000000000000..c9709efb0832 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "slices" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +const lpuAgentContainerName = "agent" + +func applyLPUHostDeviceVolumes(podSpec *corev1.PodSpec, replaceExisting bool) { + apply := appendVolumeIfMissing + if replaceExisting { + apply = setVolumeByName + } + + podSpec.Volumes = apply(podSpec.Volumes, lpuHostDeviceVolume("host-dev", "/dev")) + podSpec.Volumes = apply(podSpec.Volumes, lpuHostDeviceVolume("host-sys", "/sys")) +} + +func lpuHostDeviceVolume(name, path string) corev1.Volume { + return corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: path, + Type: ptr.To(corev1.HostPathDirectory), + }}, + } +} + +func updateWorkerPodSpec(podSpec *corev1.PodSpec) { + // TODO can we remove some of this when we get the device-plugin? + podSpec.HostIPC = true + podSpec.HostNetwork = true + podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet +} + +// ensureLPUNodeTolerations adds standard LPU-node tolerations to podSpec, which must be nonnil. +func ensureLPUNodeTolerations(podSpec *corev1.PodSpec) { + // Derive each complete toleration from the platform's ordered taint keys. + for _, key := range [...]string{"lpu.nvidia.com/node", "lpu.nvidia.com/lpu", "lpu.nvidia.com/node-v2"} { + toleration := corev1.Toleration{Key: key, Operator: corev1.TolerationOpExists} + if !slices.Contains(podSpec.Tolerations, toleration) { + podSpec.Tolerations = append(podSpec.Tolerations, toleration) + } + } +} + +func applyLPUWorkerContainerBase(container *corev1.Container, preserveCommand bool) { + if !preserveCommand { + container.Command = []string{"/bin/bash"} + } + container.SecurityContext = &corev1.SecurityContext{ + Privileged: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + } + container.TTY = true + container.Stdin = true + container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ + Name: "hugepages", + MountPath: "/dev/hugepages", + }) + + // Agent scheduling already initialized device resources; add runtime CPU and hugepage requirements. + container.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("62") + container.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") + container.Resources.Limits[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") +} + +func addLPUHostDeviceVolumeMounts(container *corev1.Container) { + container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ + Name: "host-sys", + MountPath: "/sys", + }) + container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ + Name: "host-dev", + MountPath: "/dev", + }) +} diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go new file mode 100644 index 000000000000..75b4a40ea7cb --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -0,0 +1,228 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "crypto/sha256" + "fmt" + "math" + "slices" + "strconv" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + "k8s.io/apimachinery/pkg/util/validation" +) + +const minimumHashedCliqueNameLength = 8 + +const maximumCyborgPodIndex = math.MaxInt32 + +// ExpectedAgent binds one projected model to the exact Grove +// PodClique identity that must materialize it. The controller uses this plan +// to observe objects; it never reconstructs a row from a generated Pod name. +type ExpectedAgent struct { + // CliqueName is the materialized Grove PodClique name for the selected replica. + CliqueName string + // TemplateName is the model's Grove PodClique template name. + TemplateName string + // Replicas is the number of Agent pods in the clique. + Replicas int +} + +// MaterializationPlan is the deterministic identity projection shared by the +// renderer and the controller's API-object observer. +type MaterializationPlan struct { + // PodCliqueSetName is the actual PCS used for every generated child address. + PodCliqueSetName string + // ConductorClique is the materialized conductor PodClique name, if present. + ConductorClique string + // ConductorTemplate is the conductor PodClique template name, if present. + ConductorTemplate string + // Agents contains the expected materialized Agent identities. + Agents []ExpectedAgent + // CyborgClique is the materialized Cyborg PodClique name, if present. + CyborgClique string + // CyborgTemplate is the Cyborg PodClique template name, if present. + CyborgTemplate string + // LPXScalingGroup is the materialized Grove scaling-group name. + LPXScalingGroup string + // LPXScalingGroupTemplate is the Grove scaling-group template name. + LPXScalingGroupTemplate string + // ReplicaIndex is the scaling-group replica represented by this projection. + ReplicaIndex int32 + // Replicas is the total scaling-group replica count. + Replicas int32 +} + +// PlanNodeLocalMaterialization derives the exact identities shared by graph +// rendering and lifecycle observation without mutating the workload. +func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*MaterializationPlan, error) { + componentName := w.LPXComponentName() + cyborgTemplateName := w.CyborgTemplateName() + if strings.TrimSpace(pcsName) == "" || strings.TrimSpace(componentName) == "" { + return nil, fmt.Errorf("PodCliqueSet and selected LPX component names are required") + } + + conductorTemplate := "" + if w.Pipeline() != PipelineLPX { + conductorTemplate = strings.ToLower(fmt.Sprintf("%s-%s", componentName, commonconsts.GroveRoleSuffixLeader)) + if problems := validation.IsDNS1123Label(conductorTemplate); len(problems) != 0 { + return nil, fmt.Errorf("conductor template name %q is invalid: %s", conductorTemplate, strings.Join(problems, "; ")) + } + } + agents := make([]ExpectedAgent, 0, len(w.modelProjections)) + agentScalingGroupTemplateBudget := validation.DNS1123LabelMaxLength + for index, projection := range w.modelProjections { + name := strings.ToLower(fmt.Sprintf("%s-%s-m-%d", componentName, commonconsts.GroveRoleSuffixWorker, index)) + if problems := validation.IsDNS1123Label(name); len(problems) != 0 { + return nil, fmt.Errorf("Agent template name %q is invalid: %s", name, strings.Join(problems, "; ")) + } + agents = append(agents, ExpectedAgent{TemplateName: name, Replicas: projection.agentReplicas}) + agentScalingGroupTemplateBudget = min( + agentScalingGroupTemplateBudget, + materializedScalingGroupTemplateNameBudget(pcsName, name, projection.agentReplicas-1), + ) + } + maxScalingGroupTemplateLength := validation.DNS1123LabelMaxLength + if conductorTemplate != "" { + maxScalingGroupTemplateLength = materializedScalingGroupTemplateNameBudget(pcsName, conductorTemplate, 0) + } + maxScalingGroupTemplateLength = min(maxScalingGroupTemplateLength, agentScalingGroupTemplateBudget) + if cyborgTemplateName != "" { + maxScalingGroupTemplateLength = min( + maxScalingGroupTemplateLength, + materializedScalingGroupTemplateNameBudget( + pcsName, + cyborgTemplateName, + maximumCyborgPodIndex, + ), + ) + } + lpxScalingGroupTemplate, err := boundedNameTo(strings.ToLower(componentName), maxScalingGroupTemplateLength) + if err != nil { + return nil, fmt.Errorf("derive LPU scaling-group template name: %w", err) + } + + // Construct the plan with Grove's canonical replica-zero scaling-group identity. + plan := &MaterializationPlan{ + PodCliqueSetName: pcsName, + ConductorTemplate: conductorTemplate, + CyborgTemplate: cyborgTemplateName, + Agents: agents, + LPXScalingGroup: grovecommon.GeneratePodCliqueScalingGroupName( + grovecommon.ResourceNameReplica{Name: pcsName, Replica: 0}, lpxScalingGroupTemplate, + ), + LPXScalingGroupTemplate: lpxScalingGroupTemplate, + Replicas: w.engineReplicas, + } + plan = plan.ForReplica(0) + + // All admitted replica indices are 0..8, so their names have the same length. + // Validate final hostnames once, retaining conductor/Agent/Cyborg error order. + if plan.ConductorClique != "" { + if err := validateMaterializedPodHostnameAtIndex("conductor", plan.ConductorClique, 0); err != nil { + return nil, err + } + } + for _, agent := range plan.Agents { + if err := validateMaterializedPodHostnameAtIndex("Agent", agent.CliqueName, agent.Replicas-1); err != nil { + return nil, err + } + } + if plan.CyborgClique != "" { + if err := validateMaterializedPodHostnameAtIndex("Cyborg", plan.CyborgClique, maximumCyborgPodIndex); err != nil { + return nil, err + } + } + return plan, nil +} + +// ForReplica returns the child clique identities for one scaling-group replica. +// The receiver is a successfully constructed plan and 0 <= index < Replicas. +// Neither the receiver nor its Agents slice is mutated. +func (p *MaterializationPlan) ForReplica(index int32) *MaterializationPlan { + out := *p + out.ReplicaIndex = index + out.Agents = slices.Clone(p.Agents) + + // Materialize this replica beneath the same Grove scaling-group identity. + if p.ConductorTemplate != "" { + out.ConductorClique = materializedCliqueNameForReplica(p.LPXScalingGroup, p.ConductorTemplate, index) + } + for agentIndex, agent := range out.Agents { + out.Agents[agentIndex].CliqueName = materializedCliqueNameForReplica( + p.LPXScalingGroup, + agent.TemplateName, + index, + ) + } + if p.CyborgTemplate != "" { + out.CyborgClique = materializedCliqueNameForReplica(p.LPXScalingGroup, p.CyborgTemplate, index) + } + return &out +} + +func materializedCliqueNameForReplica(pcsName, templateName string, replica int32) string { + return grovecommon.GeneratePodCliqueName( + grovecommon.ResourceNameReplica{Name: pcsName, Replica: int(replica)}, + templateName, + ) +} + +func validateMaterializedPodHostnameAtIndex(role, cliqueName string, podIndex int) error { + hostname := materializedPodHostname(cliqueName, podIndex) + if problems := validation.IsDNS1123Label(hostname); len(problems) != 0 { + return fmt.Errorf( + "materialized %s Pod hostname %q is invalid: %s", + role, + hostname, + strings.Join(problems, "; "), + ) + } + return nil +} + +func materializedPodHostname(cliqueName string, podIndex int) string { + return fmt.Sprintf("%s-%d", cliqueName, podIndex) +} + +func materializedScalingGroupTemplateNameBudget(pcsName, childTemplateName string, podIndex int) int { + // Grove materializes "-0--0--". + return validation.DNS1123LabelMaxLength - + len(pcsName) - + len(childTemplateName) - + 7 - + len(strconv.Itoa(podIndex)) +} + +func boundedNameTo(candidate string, maxLength int) (string, error) { + if maxLength > validation.DNS1123LabelMaxLength { + maxLength = validation.DNS1123LabelMaxLength + } + if maxLength < minimumHashedCliqueNameLength { + return "", fmt.Errorf( + "available DNS label budget %d is smaller than the %d-character collision-resistant minimum", + maxLength, + minimumHashedCliqueNameLength, + ) + } + if len(candidate) <= maxLength && len(validation.IsDNS1123Label(candidate)) == 0 { + return candidate, nil + } + digest := sha256.Sum256([]byte(candidate)) + hash := fmt.Sprintf("%x", digest[:4]) + prefixLength := maxLength - len(hash) - 1 + if prefixLength <= 0 { + return hash, nil + } + prefix := strings.TrimRight(candidate[:min(len(candidate), prefixLength)], "-.") + if prefix == "" { + return hash, nil + } + return prefix + "-" + hash, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go new file mode 100644 index 000000000000..d65a00240ff8 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "strings" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/validation" +) + +func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { + t.Parallel() + + t.Log("Project hybrid and LPU-only materialization fixtures") + snapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) + hybrid := newV2CompilerFixture() + hybrid.compilationMode = manifestcapnp.CompilationMode_lpx + hybridProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, hybrid))) + lpuOnlyProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) + + t.Log("Use component-derived names at the minimum collision-resistant group budget") + tests := []struct { + name string + pcsLength int + projection *ModelProjection + }{ + { + name: "hybrid", pcsLength: 14, + projection: hybridProjection, + }, + { + name: "LPU-only", pcsLength: 26, + projection: lpuOnlyProjection, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.projection.stage = "longcomponent" + workload := &SelectedWorkload{modelProjections: []*ModelProjection{test.projection}, engineReplicas: 1} + pcsName := strings.Repeat("a", test.pcsLength) + t.Log("Plan and verify materialization within the DNS hostname budget") + plan, err := workload.PlanNodeLocalMaterialization(pcsName) + require.NoError(t, err) + require.Equal(t, "longcomponent-wkr-m-0", plan.Agents[0].TemplateName) + require.Len(t, plan.LPXScalingGroupTemplate, 8) + hostname := materializedPodHostname(plan.Agents[0].CliqueName, plan.Agents[0].Replicas-1) + if test.projection.pipeline == PipelineLPX { + require.Equal(t, "longcomponent-engine-gpu", plan.CyborgTemplate) + hostname = materializedPodHostname(plan.CyborgClique, maximumCyborgPodIndex) + } + require.Len(t, hostname, validation.DNS1123LabelMaxLength) + require.Empty(t, validation.IsDNS1123Label(hostname)) + + t.Log("Reject a PCS name that leaves one character too little for the group") + _, err = workload.PlanNodeLocalMaterialization(pcsName + "a") + require.ErrorContains(t, err, "derive LPU scaling-group template name") + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go new file mode 100644 index 000000000000..7f226e08969d --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go @@ -0,0 +1,203 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "io" + "net/url" + "testing" + + "capnproto.org/go/capnp/v3" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +const registryTestTopology = "URSA_V2_1__Q8__8C__G_106__KP_FEC__GHZ_1_0__NO_FPGA" + +const gbuildManifestJSONFile = "manifest.json" + +func normalizeRegistryFixtureBuild(ctx context.Context, registry *ModelRegistry, id string) (*Build, error) { + snapshot, err := registry.AcquireBuildSnapshot(ctx, id) + if err != nil { + return nil, err + } + normalized, err := normalizeBuildSnapshot(snapshot) + if err != nil { + return nil, err + } + return normalized.build, nil +} + +type fakeModelServiceClient struct { + request *modelpb.ModelDownloadRequest + stream grpc.ServerStreamingClient[modelpb.ModelStatusUpdate] + err error + filesRequests []*modelpb.ModelFilesRequest + filesContext context.Context + fileStreams []*fakeModelFileStream + filesErr error + listRequests []*modelpb.ModelFilesRequest + list *modelpb.ModelFileList + listErr error +} + +func (c *fakeModelServiceClient) EnsureModelDownloaded( + _ context.Context, + request *modelpb.ModelDownloadRequest, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[modelpb.ModelStatusUpdate], error) { + c.request = request + if c.err != nil { + return nil, c.err + } + return c.stream, nil +} + +func (c *fakeModelServiceClient) StreamModelFiles( + ctx context.Context, + request *modelpb.ModelFilesRequest, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[modelpb.FileChunk], error) { + c.filesRequests = append(c.filesRequests, request) + c.filesContext = ctx + if c.filesErr != nil { + return nil, c.filesErr + } + stream := c.fileStreams[0] + c.fileStreams = c.fileStreams[1:] + return stream, nil +} + +func (c *fakeModelServiceClient) ListModelFiles( + _ context.Context, + request *modelpb.ModelFilesRequest, + _ ...grpc.CallOption, +) (*modelpb.ModelFileList, error) { + c.listRequests = append(c.listRequests, request) + if c.listErr != nil { + return nil, c.listErr + } + if c.list != nil { + return c.list, nil + } + return &modelpb.ModelFileList{}, nil +} + +type fakeModelDownloadStream struct { + grpc.ClientStream + update *modelpb.ModelStatusUpdate + err error +} + +func (s *fakeModelDownloadStream) Recv() (*modelpb.ModelStatusUpdate, error) { + if s.err != nil { + return nil, s.err + } + return s.update, nil +} + +type fakeModelFileStream struct { + grpc.ClientStream + chunks []*modelpb.FileChunk + err error +} + +func (s *fakeModelFileStream) Recv() (*modelpb.FileChunk, error) { + if len(s.chunks) > 0 { + chunk := s.chunks[0] + s.chunks = s.chunks[1:] + return chunk, nil + } + if s.err != nil { + return nil, s.err + } + return nil, io.EOF +} + +func modelFileChunks(relativePath string, parts ...string) []*modelpb.FileChunk { + totalSize := 0 + for _, part := range parts { + totalSize += len(part) + } + + var offset uint64 + chunks := make([]*modelpb.FileChunk, 0, len(parts)) + for i, part := range parts { + chunks = append(chunks, &modelpb.FileChunk{ + RelativePath: relativePath, + Data: []byte(part), + Offset: offset, + TotalSize: uint64(totalSize), + IsLastChunk: i == len(parts)-1, + }) + offset += uint64(len(part)) + } + return chunks +} + +func mustParseURL(t *testing.T, rawURL string) url.URL { + t.Helper() + + buildURL, err := url.Parse(rawURL) + if err != nil { + t.Fatalf("url.Parse(%q) error = %v", rawURL, err) + } + return *buildURL +} + +func defaultTestString(value string, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func newManifestV2ContractFixture(t *testing.T) manifestcapnpv2.Manifest { + t.Helper() + + _, seg := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnpv2.NewRootManifest(seg) + require.NoError(t, err) + manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) + deployment, err := manifest.NewDeployment() + require.NoError(t, err) + deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpx) + deployment.SetNumLpuNodes(1) + program, err := deployment.NewProgram() + require.NoError(t, err) + program.SetBatchSize(8) + runtimeIO, err := deployment.NewRuntimeIo() + require.NoError(t, err) + runtimeIO.SetProtocol(runtimeIOProtocolMultiEndpoint) + runtimeIO.SetReserved1(2) + runtimeIO.SetIoFpgaCount(4) + runtimeIO.SetFanoutFactor(2) + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(1) + require.NoError(t, err) + setManifestV2LPUArtifact(t, partitions.At(0), 0) + + return manifest +} + +func setManifestV2LPUArtifact(t *testing.T, partition manifestcapnpv2.PartitionInfo, id uint32) { + t.Helper() + + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetPartitionId(id) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath("part-0")) + require.NoError(t, detail.SetTopology(registryTestTopology)) + detail.SetNumChips(8) + detail.SetDevicesPerNode(8) +} diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go new file mode 100644 index 000000000000..b7e7c70996ca --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go @@ -0,0 +1,288 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "capnproto.org/go/capnp/v3" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" +) + +const hxTopologyFamily = "16x8x2x3" + +func addManifestTextDefault( + defaults map[string]any, + key string, + subject string, + has bool, + read func() (string, error), +) error { + if !has { + return nil + } + value, err := read() + if err != nil { + return fmt.Errorf("reading %s: %w", subject, err) + } + value = strings.TrimSpace(value) + if value != "" { + defaults[key] = value + } + return nil +} + +func childSwaDefaults(defaults map[string]any) map[string]any { + child, _ := defaults["swa"].(map[string]any) + if child == nil { + child = make(map[string]any) + defaults["swa"] = child + } + return child +} + +func capnpUInt32List(values capnp.UInt32List) []uint32 { + seen := make(map[uint32]struct{}, values.Len()) + out := make([]uint32, 0, values.Len()) + for i := 0; i < values.Len(); i++ { + value := values.At(i) + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + +func classifyManifestPartitions(filename string, partitions []BuildPartition, partSelect bool) (BuildFamily, int, int, error) { + family := BuildFamilyXT + packagedNodes, partitionZeroNodes := 0, 0 + seen := make(map[int]struct{}, len(partitions)) + for _, partition := range partitions { + partitionFamily := BuildFamilyXT + if len(partition.HXExtent) != 0 { + partitionFamily = BuildFamilyHX + } + if len(seen) != 0 && family != partitionFamily { + return "", 0, 0, fmt.Errorf("%s mixes XT and HX LPU partitions", filename) + } + family = partitionFamily + if _, duplicate := seen[partition.SourcePartitionID]; duplicate { + if family == BuildFamilyHX { + return "", 0, 0, fmt.Errorf("V3 %s repeats LPU partition ID %d", filename, partition.SourcePartitionID) + } + return "", 0, 0, fmt.Errorf("%s has duplicate LPU partition id %d", filename, partition.SourcePartitionID) + } + seen[partition.SourcePartitionID] = struct{}{} + + // Retain both deployment geometries during the mandatory artifact traversal. + nodes := partition.Topology.Replicas() + if len(partition.HXExtent) == 4 { + nodes = int(partition.HXExtent[1] * partition.HXExtent[2] * partition.HXExtent[3]) + } + packagedNodes += nodes + if partition.SourcePartitionID == 0 { + partitionZeroNodes += nodes + } + } + if family == BuildFamilyHX { + if partSelect { + return "", 0, 0, fmt.Errorf("V3 %s partSelect builds are not supported", filename) + } + return family, packagedNodes, partitionZeroNodes, nil + } + sort.Slice(partitions, func(i, j int) bool { return partitions[i].SourcePartitionID < partitions[j].SourcePartitionID }) + return family, packagedNodes, partitionZeroNodes, nil +} + +func buildXTPartition(subject, value string, raw manifestcapnpv2.LpuPartitionArtifact) (BuildPartition, error) { + // Decode XT's named topology before checking its declared geometry. + topology, err := parse(value) + if err != nil { + return BuildPartition{}, fmt.Errorf("parsing %s topology %q: %w", subject, value, err) + } + + // Reject nonpositive chip counts and nonintegral multi-node XT topologies. + if topology.ChipCount <= 0 { + return BuildPartition{}, fmt.Errorf("%s topology has invalid chip count %d", subject, topology.ChipCount) + } + if topology.ChipCount%lpuChipsPerNode != 0 && topology.ChipCount > lpuChipsPerNode { + return BuildPartition{}, fmt.Errorf("%s topology has %d chips, not divisible by %d LPU devices per node", subject, topology.ChipCount, lpuChipsPerNode) + } + + // Require the manifest's chip count and device count to agree with XT geometry. + numChips, err := positiveManifestUInt32ToInt(subject+" numChips", raw.NumChips()) + if err != nil { + return BuildPartition{}, err + } + if topology.ChipCount != numChips { + return BuildPartition{}, fmt.Errorf("%s topology chip count %d does not match numChips %d", subject, topology.ChipCount, numChips) + } + devicesPerNode, err := positiveManifestUInt32ToInt(subject+" devicesPerNode", raw.DevicesPerNode()) + if err != nil { + return BuildPartition{}, err + } + if devicesPerNode != lpuChipsPerNode { + return BuildPartition{}, fmt.Errorf("%s devicesPerNode %d, want %d for LPU device version 2", subject, devicesPerNode, lpuChipsPerNode) + } + return BuildPartition{Topology: topology}, nil +} + +func buildHXPartition(subject, value string, raw manifestcapnpv2.LpuPartitionArtifact) (BuildPartition, bool, error) { + // HX topology names are opaque; metadata supplies the geometry when present. + topology := strings.TrimSpace(value) + if topology == "" { + return BuildPartition{}, false, fmt.Errorf("V3 %s topology must not be empty", subject) + } + partition := BuildPartition{Topology: Topology{Raw: topology, ChipCount: int(raw.NumChips())}} + if !raw.HasTopologyMetadata() { + // The caller selects metadata-less HX only for the 16-chip, 16-device case. + partition.HXExtent = []int64{16, 1, 1, 1} + return partition, true, nil + } + + // Decode HX metadata directly and report read errors at their source. + metadata, err := raw.TopologyMetadata() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading V3 %s topologyMetadata: %w", subject, err) + } + family, err := metadata.TopologyFamily() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading V3 %s topologyMetadata.topologyFamily: %w", subject, err) + } + if strings.TrimSpace(family) != hxTopologyFamily { + return BuildPartition{}, false, fmt.Errorf("V3 %s topologyMetadata.topologyFamily = %q, want %q", subject, strings.TrimSpace(family), hxTopologyFamily) + } + shape, err := metadata.PartitionShape() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading V3 %s topologyMetadata.partitionShape: %w", subject, err) + } + extent := make([]int64, shape.Len()) + for index := range extent { + extent[index] = int64(shape.At(index)) + } + + // Preserve every supported HX shape and its chip/device consistency checks. + if len(extent) != 4 || extent[0] != 16 || extent[1] < 1 || extent[1] > 8 || + (!((extent[2] == 1 || extent[2] == 2) && extent[3] == 1) && !(extent[1] == 8 && extent[2] == 2 && extent[3] == 2)) { + return BuildPartition{}, false, fmt.Errorf("V3 %s has unsupported HX extent %v", subject, extent) + } + count := extent[0] * extent[1] * extent[2] * extent[3] + if count != int64(raw.NumChips()) { + return BuildPartition{}, false, fmt.Errorf("V3 %s topologyMetadata.partitionShape contains %d chips, want numChips %d", subject, count, raw.NumChips()) + } + if extent[0] != int64(raw.DevicesPerNode()) { + return BuildPartition{}, false, fmt.Errorf("V3 %s topologyMetadata.partitionShape first dimension %d does not match devicesPerNode %d", subject, extent[0], raw.DevicesPerNode()) + } + partition.HXExtent = extent + return partition, false, nil +} + +func buildCompilationMode(filename, mode string) (BuildCompilationMode, error) { + switch mode { + case string(BuildCompilationModeLPUOnly): + return BuildCompilationModeLPUOnly, nil + case string(BuildCompilationModeHybrid): + return BuildCompilationModeHybrid, nil + default: + return BuildCompilationModeUnknown, fmt.Errorf("%s deployment.compilationMode %q is not supported", filename, mode) + } +} + +// decodePropSyncChain normalizes one selected-chain list from the compiler manifest. +func decodePropSyncChain(rawIDs capnp.UInt32List, chainPath string) ([]int, error) { + if rawIDs.Len() < 2 { + return nil, fmt.Errorf("%s must contain at least two partitionIds", chainPath) + } + + partitionIDs := make([]int, rawIDs.Len()) + for index := range rawIDs.Len() { + partitionIDs[index] = int(rawIDs.At(index)) + } + return partitionIDs, nil +} + +func cleanManifestRelativeBuildPath(field, rawPath string, allowBuildRoot bool) (string, error) { + if strings.ContainsAny(rawPath, "\x00\r\n") { + return "", fmt.Errorf("%s %q must not contain NUL bytes or line breaks", field, rawPath) + } + assetPath := strings.TrimSpace(rawPath) + if assetPath == "" { + return "", fmt.Errorf("%s is empty", field) + } + if filepath.IsAbs(assetPath) { + return "", fmt.Errorf("%s %q must be relative and stay within build directory", field, rawPath) + } + assetPath = filepath.ToSlash(filepath.Clean(assetPath)) + if assetPath == "." && allowBuildRoot { + return assetPath, nil + } + if assetPath == "." || assetPath == ".." || strings.HasPrefix(assetPath, "../") { + return "", fmt.Errorf("%s %q must be relative and stay within build directory", field, rawPath) + } + return assetPath, nil +} + +func validateManifestPartitionNodeCount( + filename string, + want int, + build *Build, + partialSelection bool, + hxDoubleNodeCount bool, + packagedNodes, partitionZeroNodes int, +) error { + // Compare the declaration with the packaged and host-embedding partition inventories. + hostEmbeddingNodes := packagedNodes + if build.SupportsCPUEmbeddings && build.StandaloneTokenEmbeddings { + hostEmbeddingNodes -= partitionZeroNodes + } + if build.Family == BuildFamilyHX { + if want == packagedNodes || (hxDoubleNodeCount && want == 2*packagedNodes) { + return nil + } + return fmt.Errorf("V3 %s deployment.numLpuNodes = %d, but partition extents require %d LPU nodes", filename, want, packagedNodes) + } + + // A host-embedding deployment must retain at least one model partition. + if hostEmbeddingNodes == 0 { + return fmt.Errorf("%s LPU partitions use 0 LPU nodes, want deployment.numLpuNodes %d", filename, want) + } + + // Accept either supported deployment mode without discarding packaged partitions. + if want == packagedNodes || want == hostEmbeddingNodes { + return nil + } + + // partSelect artifacts contain only the selected partitions, while numLpuNodes + // describes the complete deployment geometry. + if partialSelection && (want >= packagedNodes || want >= hostEmbeddingNodes) { + return nil + } + + // Keep the existing error concise when both deployment modes use the same node count. + if packagedNodes == hostEmbeddingNodes { + return fmt.Errorf("%s LPU partitions use %d LPU nodes, want deployment.numLpuNodes %d", filename, packagedNodes, want) + } + + return fmt.Errorf( + "%s LPU partitions use %d packaged LPU nodes or %d with host embeddings, want deployment.numLpuNodes %d", + filename, + packagedNodes, + hostEmbeddingNodes, + want, + ) +} + +func positiveManifestUInt32ToInt(field string, value uint32) (int, error) { + if value == 0 { + return 0, fmt.Errorf("%s must be >= 1, got 0", field) + } + return int(value), nil +} diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common_test.go new file mode 100644 index 000000000000..2afef7eb4d11 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common_test.go @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildPartitionFromManifestV2SelectsFamily(t *testing.T) { + t.Parallel() + + t.Log("Define XT and HX classification outcomes from actual revision-2 artifacts") + tests := []struct { + name, topology, family string + numChips, devicesPerNode uint32 + extent, wantExtent []uint32 + wantCompatible bool + wantErr string + }{ + {name: "ordinary XT", topology: registryTestTopology, numChips: 8, devicesPerNode: 8}, + {name: "multi-node XT", topology: strings.Replace(registryTestTopology, "8C", "16C", 1), numChips: 16, devicesPerNode: 8}, + {name: "metadata-less HX opaque topology", topology: " " + v3OpaqueTopology + " ", numChips: 16, devicesPerNode: 16, wantExtent: []uint32{16, 1, 1, 1}, wantCompatible: true}, + {name: "metadata-less HX XT-looking topology", topology: registryTestTopology, numChips: 16, devicesPerNode: 16, wantExtent: []uint32{16, 1, 1, 1}, wantCompatible: true}, + {name: "metadata HX opaque topology", topology: v3OpaqueTopology, numChips: 16, devicesPerNode: 16, family: " " + hxTopologyFamily + " ", extent: []uint32{16, 1, 1, 1}, wantExtent: []uint32{16, 1, 1, 1}}, + {name: "metadata HX XT-looking topology", topology: registryTestTopology, numChips: 16, devicesPerNode: 16, family: hxTopologyFamily, extent: []uint32{16, 1, 1, 1}, wantExtent: []uint32{16, 1, 1, 1}}, + {name: "full HX geometry", topology: v3OpaqueTopology, numChips: 512, devicesPerNode: 16, family: hxTopologyFamily, extent: []uint32{16, 8, 2, 2}, wantExtent: []uint32{16, 8, 2, 2}}, + {name: "metadata forbids XT fallback", topology: registryTestTopology, numChips: 8, devicesPerNode: 8, family: hxTopologyFamily, extent: []uint32{16, 1, 1, 1}, wantErr: "contains 16 chips, want numChips 8"}, + {name: "unknown HX family", topology: v3OpaqueTopology, numChips: 16, devicesPerNode: 16, family: "unknown", extent: []uint32{16, 1, 1, 1}, wantErr: "topologyMetadata.topologyFamily"}, + {name: "missing HX extent", topology: v3OpaqueTopology, numChips: 16, devicesPerNode: 16, family: hxTopologyFamily, wantErr: "unsupported HX extent"}, + {name: "unsupported HX extent", topology: v3OpaqueTopology, numChips: 64, devicesPerNode: 16, family: hxTopologyFamily, extent: []uint32{16, 1, 2, 2}, wantErr: "unsupported HX extent"}, + {name: "HX devices per node mismatch", topology: v3OpaqueTopology, numChips: 16, devicesPerNode: 8, family: hxTopologyFamily, extent: []uint32{16, 1, 1, 1}, wantErr: "does not match devicesPerNode 8"}, + {name: "opaque topology needs HX geometry", topology: v3OpaqueTopology, numChips: 16, devicesPerNode: 8, wantErr: "parsing"}, + {name: "metadata-less HX wrong chip count", topology: v3OpaqueTopology, numChips: 8, devicesPerNode: 16, wantErr: "does not match expected pattern"}, + {name: "metadata-less HX empty topology", topology: " ", numChips: 16, devicesPerNode: 16, wantErr: "topology must not be empty"}, + {name: "metadata-less HX unsafe topology", topology: v3OpaqueTopology + "\n", numChips: 16, devicesPerNode: 16, wantErr: "must not contain NUL bytes or line breaks"}, + {name: "metadata HX NUL topology", topology: hxTopologyFamily + "\x00other", numChips: 16, devicesPerNode: 16, family: hxTopologyFamily, extent: []uint32{16, 1, 1, 1}, wantErr: "topology must not contain NUL bytes or line breaks"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Construct a manifest partition with the selected geometry") + manifest := newManifestV2ContractFixture(t) + artifacts, err := manifest.Artifacts() + require.NoError(t, err) + partitions, err := artifacts.Partitions() + require.NoError(t, err) + raw := partitions.At(0) + setManifestV2LPUArtifact(t, raw, 7) + detail, err := raw.Detail().Lpu() + require.NoError(t, err) + require.NoError(t, detail.SetTopology(test.topology)) + detail.SetNumChips(test.numChips) + detail.SetDevicesPerNode(test.devicesPerNode) + if test.family != "" { + metadata, err := detail.NewTopologyMetadata() + require.NoError(t, err) + require.NoError(t, metadata.SetTopologyFamily(test.family)) + extent, err := metadata.NewPartitionShape(int32(len(test.extent))) + require.NoError(t, err) + for index, value := range test.extent { + extent.Set(index, value) + } + } + + t.Log("Decode exactly the supported geometry and preserve the compatibility flag") + partition, compatible, err := buildPartitionFromManifestV2(raw) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + require.Equal(t, BuildPartition{}, partition) + require.False(t, compatible) + return + } + require.NoError(t, err) + require.Equal(t, test.wantCompatible, compatible) + require.Equal(t, 7, partition.SourcePartitionID) + require.Equal(t, "part-0", partition.PartPath) + require.Equal(t, strings.TrimSpace(test.topology), partition.Topology.Raw) + require.EqualValues(t, test.numChips, partition.Topology.ChipCount) + require.Len(t, partition.HXExtent, len(test.wantExtent)) + for index, value := range test.wantExtent { + require.EqualValues(t, value, partition.HXExtent[index]) + } + + t.Log("Reject unsafe paths for every supported partition family") + require.NoError(t, detail.SetPath("../outside")) + partition, compatible, err = buildPartitionFromManifestV2(raw) + require.ErrorContains(t, err, "path") + require.Equal(t, BuildPartition{}, partition) + require.False(t, compatible) + }) + } +} + +func TestManifestPartitionFamilyOrdering(t *testing.T) { + t.Log("Classify HX partitions while preserving manifest order") + hx := []BuildPartition{{SourcePartitionID: 7, HXExtent: []int64{16, 1, 1, 1}}, {SourcePartitionID: 3, HXExtent: []int64{16, 1, 1, 1}}} + family, _, _, err := classifyManifestPartitions(gbuildManifestV2CapnpFile, hx, false) + require.NoError(t, err) + require.Equal(t, BuildFamilyHX, family) + require.Equal(t, []int{7, 3}, []int{hx[0].SourcePartitionID, hx[1].SourcePartitionID}) + + t.Log("Classify XT partitions while sorting by source partition identity") + xt := []BuildPartition{{SourcePartitionID: 7}, {SourcePartitionID: 3}} + family, _, _, err = classifyManifestPartitions(gbuildManifestV2CapnpFile, xt, false) + require.NoError(t, err) + require.Equal(t, BuildFamilyXT, family) + require.Equal(t, []int{3, 7}, []int{xt[0].SourcePartitionID, xt[1].SourcePartitionID}) +} diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go new file mode 100644 index 000000000000..883500e02159 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -0,0 +1,493 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "math" + "sort" + "strings" + + "capnproto.org/go/capnp/v3" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" +) + +const ( + gbuildManifestV2CapnpFile = "manifest.v2.capnp.bin" + runtimeIOProtocolHost uint16 = 0 + runtimeIOProtocolMultiEndpoint uint16 = 1 + runtimeIOMaxMode uint16 = 2 +) + +func decodeGbuildManifestV2(data []byte) (manifestcapnpv2.Manifest, error) { + msg, err := capnp.Unmarshal(data) + if err != nil { + return manifestcapnpv2.Manifest{}, fmt.Errorf("parsing %s: %w", gbuildManifestV2CapnpFile, err) + } + manifest, err := manifestcapnpv2.ReadRootManifest(msg) + if err != nil { + return manifestcapnpv2.Manifest{}, fmt.Errorf("reading %s root: %w", gbuildManifestV2CapnpFile, err) + } + return manifest, nil +} + +// buildFromGbuildManifestV2 validates compiler input using the acquired canonical build reference. +func buildFromGbuildManifestV2(buildRef string, manifest manifestcapnpv2.Manifest) (*Build, error) { + if manifest.ContractRevision() != manifestcapnpv2.CurrentContractRevision { + return nil, fmt.Errorf( + "%s contractRevision = %d, want %d", + gbuildManifestV2CapnpFile, + manifest.ContractRevision(), + manifestcapnpv2.CurrentContractRevision, + ) + } + if !manifest.HasDeployment() { + return nil, fmt.Errorf("%s is missing deployment", gbuildManifestV2CapnpFile) + } + deployment, err := manifest.Deployment() + if err != nil { + return nil, fmt.Errorf("reading %s deployment: %w", gbuildManifestV2CapnpFile, err) + } + compilationMode, err := buildCompilationMode(gbuildManifestV2CapnpFile, deployment.CompilationMode().String()) + if err != nil { + return nil, err + } + if !deployment.HasProgram() { + return nil, fmt.Errorf("%s deployment.program is missing", gbuildManifestV2CapnpFile) + } + program, err := deployment.Program() + if err != nil { + return nil, fmt.Errorf("reading %s deployment.program: %w", gbuildManifestV2CapnpFile, err) + } + batchSize, err := positiveManifestUInt32ToInt( + fmt.Sprintf("%s deployment.program.batchSize", gbuildManifestV2CapnpFile), + program.BatchSize(), + ) + if err != nil { + return nil, err + } + chains, err := selectedPropSyncChainsFromManifestV2(deployment) + if err != nil { + return nil, err + } + ioFPGACount, ioFanoutFactor, err := runtimeIOFromManifestV2(deployment) + if err != nil { + return nil, err + } + // Reject an incomplete split-I/O batch before any runtime path consumes it. + if batchSize%int(ioFPGACount) != 0 { + return nil, fmt.Errorf( + "%s deployment.program.batchSize %d must be divisible by deployment.runtimeIo.ioFpgaCount %d", + gbuildManifestV2CapnpFile, + batchSize, + ioFPGACount, + ) + } + + // Reject incomplete client-owned transaction regions after endpoint splitting. + perEndpointBatchSize := batchSize / int(ioFPGACount) + if perEndpointBatchSize%int(ioFanoutFactor) != 0 { + return nil, fmt.Errorf( + "%s deployment.program.batchSize per endpoint %d must be divisible by deployment.runtimeIo.fanoutFactor %d", + gbuildManifestV2CapnpFile, + perEndpointBatchSize, + ioFanoutFactor, + ) + } + defaults, tokenizerPath, err := defaultsFromManifestV2(manifest, program, compilationMode, chains) + if err != nil { + return nil, err + } + if !manifest.HasArtifacts() { + return nil, fmt.Errorf("%s is missing artifacts", gbuildManifestV2CapnpFile) + } + artifacts, err := manifest.Artifacts() + if err != nil { + return nil, fmt.Errorf("reading %s artifacts: %w", gbuildManifestV2CapnpFile, err) + } + runtimeTokenEmbeddingsPath, err := runtimeTokenEmbeddingsPathFromManifestV2(artifacts) + if err != nil { + return nil, err + } + // Validate runtime embedding assets before projecting scheduler-facing artifacts. + if runtimeTokenEmbeddingsPath != "" && !program.SupportsCpuEmbeddings() { + return nil, fmt.Errorf("%s artifacts.runtimeAssets.tokenEmbeddingsPath requires supportsCpuEmbeddings=true", gbuildManifestV2CapnpFile) + } + if program.SupportsCpuEmbeddings() && program.StandaloneTokenEmbeddings() && runtimeTokenEmbeddingsPath == "" { + return nil, fmt.Errorf("%s artifacts.runtimeAssets.tokenEmbeddingsPath is required when standaloneTokenEmbeddings=true", gbuildManifestV2CapnpFile) + } + + build := &Build{ + Path: buildRef, + CompilationMode: compilationMode, + BatchSize: batchSize, + SelectedPropSyncChains: chains, + StandaloneTokenEmbeddings: program.StandaloneTokenEmbeddings(), + SupportsCPUEmbeddings: program.SupportsCpuEmbeddings(), + RuntimeTokenEmbeddingsPath: runtimeTokenEmbeddingsPath, + IOFPGACount: ioFPGACount, + IOFanoutFactor: ioFanoutFactor, + runtimeSettings: defaults, + RuntimeTokenizerPath: tokenizerPath, + } + + // Complete the normalized build with scheduler-facing LPU artifacts. + if err := addLPUArtifactsFromManifestV2(artifacts, deployment, build); err != nil { + return nil, err + } + return build, nil +} + +func runtimeIOFromManifestV2(deployment manifestcapnpv2.DeploymentInfo) (int32, int32, error) { + if !deployment.HasRuntimeIo() { + return 0, 0, fmt.Errorf("%s deployment.runtimeIo is missing", gbuildManifestV2CapnpFile) + } + runtimeIO, err := deployment.RuntimeIo() + if err != nil { + return 0, 0, fmt.Errorf("reading %s deployment.runtimeIo: %w", gbuildManifestV2CapnpFile, err) + } + count := runtimeIO.IoFpgaCount() + if count == 0 || count > math.MaxInt32 { + return 0, 0, fmt.Errorf("%s deployment.runtimeIo.ioFpgaCount must be in [1, %d], got %d", gbuildManifestV2CapnpFile, math.MaxInt32, count) + } + + // Fanout is a separate positive client count for each physical endpoint. + fanoutFactor := runtimeIO.FanoutFactor() + if fanoutFactor == 0 || fanoutFactor > math.MaxInt32 { + return 0, 0, fmt.Errorf("%s deployment.runtimeIo.fanoutFactor must be in [1, %d], got %d", gbuildManifestV2CapnpFile, math.MaxInt32, fanoutFactor) + } + switch runtimeIO.Protocol() { + case runtimeIOProtocolHost: + if count != 1 { + return 0, 0, fmt.Errorf("%s host runtime I/O requires ioFpgaCount 1, got %d", gbuildManifestV2CapnpFile, count) + } + case runtimeIOProtocolMultiEndpoint: + default: + return 0, 0, fmt.Errorf("%s deployment.runtimeIo.protocol %d is not supported", gbuildManifestV2CapnpFile, runtimeIO.Protocol()) + } + if mode := runtimeIO.Reserved1(); mode > runtimeIOMaxMode { + return 0, 0, fmt.Errorf("%s deployment.runtimeIo mode %d is not supported", gbuildManifestV2CapnpFile, mode) + } + return int32(count), int32(fanoutFactor), nil +} + +func selectedPropSyncChainsFromManifestV2(deployment manifestcapnpv2.DeploymentInfo) ([][]int, error) { + if !deployment.HasSelectedPropSyncChains() { + return nil, nil + } + rawChains, err := deployment.SelectedPropSyncChains() + if err != nil { + return nil, fmt.Errorf("reading %s deployment.selectedPropSyncChains: %w", gbuildManifestV2CapnpFile, err) + } + chains := make([][]int, 0, rawChains.Len()) + for chainIndex := 0; chainIndex < rawChains.Len(); chainIndex++ { + rawChain := rawChains.At(chainIndex) + if !rawChain.HasPartitionIds() { + return nil, fmt.Errorf("%s deployment.selectedPropSyncChains[%d].partitionIds is missing", gbuildManifestV2CapnpFile, chainIndex) + } + rawIDs, err := rawChain.PartitionIds() + if err != nil { + return nil, fmt.Errorf("reading %s deployment.selectedPropSyncChains[%d].partitionIds: %w", gbuildManifestV2CapnpFile, chainIndex, err) + } + + chainPath := fmt.Sprintf("%s deployment.selectedPropSyncChains[%d]", gbuildManifestV2CapnpFile, chainIndex) + chain, err := decodePropSyncChain(rawIDs, chainPath) + if err != nil { + return nil, err + } + chains = append(chains, chain) + } + return chains, nil +} + +func defaultsFromManifestV2( + manifest manifestcapnpv2.Manifest, + program manifestcapnpv2.ProgramConfig, + compilationMode BuildCompilationMode, + chains [][]int, +) (map[string]any, string, error) { + defaults := make(map[string]any) + tokenizerPath, err := addModelDefaultsFromManifestV2(defaults, manifest) + if err != nil { + return nil, "", err + } + defaults["batch_size"] = int64(program.BatchSize()) + if err := addManifestTextDefault(defaults, "program_name", gbuildManifestV2CapnpFile+" deployment.program.programName", program.HasProgramName(), program.ProgramName); err != nil { + return nil, "", err + } + if err := addManifestTextDefault(defaults, "package_name", gbuildManifestV2CapnpFile+" deployment.program.packageName", program.HasPackageName(), program.PackageName); err != nil { + return nil, "", err + } + defaults["sequence_length"] = int64(program.SequenceLength()) + defaults["input_size"] = int64(program.InputSize()) + defaults["output_size"] = int64(program.OutputSize()) + defaults["num_kv_caches"] = int64(program.NumKvCaches()) + defaults["num_batch_split_divisions"] = int64(program.NumBatchSplitDivisions()) + defaults["batch_folding"] = program.BatchFolding() + defaults["cpu_embeddings"] = program.SupportsCpuEmbeddings() + if program.HasTopk() { + topk, err := program.Topk() + if err != nil { + return nil, "", fmt.Errorf("reading %s deployment.program.topk: %w", gbuildManifestV2CapnpFile, err) + } + defaults["topk"] = int64(topk.Value()) + } + if program.EnableLogitMasking() { + defaults["enable_logit_masking"] = true + } + defaults["dkvc"] = program.HasDkvc() + if program.HasDkvc() { + dkvc, err := program.Dkvc() + if err != nil { + return nil, "", fmt.Errorf("reading %s deployment.program.dkvc: %w", gbuildManifestV2CapnpFile, err) + } + defaults["num_dkvc_blocks"] = int64(dkvc.NumBlocksPerKvCache()) + } + defaults["prop_sync"] = compilationMode == BuildCompilationModeLPUOnly && len(chains) > 0 + if _, hasSWA := defaults["swa"]; hasSWA || program.SwaChunked() || program.NumSwaDkvcBlocks() != 0 { + swaDefaults := childSwaDefaults(defaults) + swaDefaults["chunked"] = program.SwaChunked() + swaDefaults["num_swa_dkvc_blocks"] = int64(program.NumSwaDkvcBlocks()) + } + return defaults, tokenizerPath, nil +} + +func addModelDefaultsFromManifestV2(defaults map[string]any, manifest manifestcapnpv2.Manifest) (string, error) { + if !manifest.HasModel() { + return "", nil + } + model, err := manifest.Model() + if err != nil { + return "", fmt.Errorf("reading %s model: %w", gbuildManifestV2CapnpFile, err) + } + if model.HasArch() { + arch, err := model.Arch() + if err != nil { + return "", fmt.Errorf("reading %s model.arch: %w", gbuildManifestV2CapnpFile, err) + } + defaults["num_layers"] = int64(arch.NLayers()) + defaults["vocab_size"] = int64(arch.VocabSize()) + if arch.HasSwa() { + swa, err := arch.Swa() + if err != nil { + return "", fmt.Errorf("reading %s model.arch.swa: %w", gbuildManifestV2CapnpFile, err) + } + swaDefaults := childSwaDefaults(defaults) + swaDefaults["swa_ctx_len"] = int64(swa.MaxSeqLen()) + swaDefaults["swa_padding_len"] = int64(swa.Padding()) + if swa.HasNumUsers() { + numUsers, err := swa.NumUsers() + if err != nil { + return "", fmt.Errorf("reading %s model.arch.swa.numUsers: %w", gbuildManifestV2CapnpFile, err) + } + swaDefaults["swa_num_users"] = int64(numUsers.Value()) + } + } + } + if !model.HasTokenizer() { + return "", nil + } + tokenizer, err := model.Tokenizer() + if err != nil { + return "", fmt.Errorf("reading %s model.tokenizer: %w", gbuildManifestV2CapnpFile, err) + } + if tokenizer.VocabSize() != 0 { + defaults["vocab_size"] = int64(tokenizer.VocabSize()) + } + tokenizerPath := "" + if tokenizer.HasPath() { + tokenizerPath, err = tokenizer.Path() + if err != nil { + return "", fmt.Errorf("reading %s model.tokenizer.path: %w", gbuildManifestV2CapnpFile, err) + } + tokenizerPath, err = cleanManifestRelativeBuildPath( + fmt.Sprintf("%s model.tokenizer.path", gbuildManifestV2CapnpFile), + tokenizerPath, + true, + ) + if err != nil { + return "", err + } + } + if tokenizer.HasStopTokens() { + stopTokens, err := tokenizer.StopTokens() + if err != nil { + return "", fmt.Errorf("reading %s model.tokenizer.stopTokens: %w", gbuildManifestV2CapnpFile, err) + } + defaults["stop_tokens"] = capnpUInt32List(stopTokens) + } + return tokenizerPath, nil +} + +func addLPUArtifactsFromManifestV2( + artifacts manifestcapnpv2.ArtifactInfo, + deployment manifestcapnpv2.DeploymentInfo, + build *Build, +) error { + rawPartitions, err := artifacts.Partitions() + if err != nil { + return fmt.Errorf("reading %s artifacts.partitions: %w", gbuildManifestV2CapnpFile, err) + } + partitions := make([]BuildPartition, 0, rawPartitions.Len()) + hxDoubleNodeCount := true + for index := 0; index < rawPartitions.Len(); index++ { + partition, compatible, err := buildPartitionFromManifestV2(rawPartitions.At(index)) + if err != nil { + return err + } + // Nonempty paths mark LPU artifacts; only metadata-less HX partitions permit the historical doubled node count. + if partition.PartPath != "" { + partitions = append(partitions, partition) + hxDoubleNodeCount = hxDoubleNodeCount && (len(partition.HXExtent) == 0 || compatible) + } + } + // An LPU-only build cannot silently discard packaged CUDA or CPU partitions. + if build.CompilationMode == BuildCompilationModeLPUOnly && len(partitions) != rawPartitions.Len() { + return fmt.Errorf( + "%s deployment.compilationMode lpuOnly requires every artifact partition to use deviceType lpu", + gbuildManifestV2CapnpFile, + ) + } + if len(partitions) == 0 { + return fmt.Errorf("%s artifacts contain no LPU partitions", gbuildManifestV2CapnpFile) + } + partialSelection := artifacts.HasPartSelect() + family, packagedNodes, partitionZeroNodes, err := classifyManifestPartitions(gbuildManifestV2CapnpFile, partitions, partialSelection) + if err == nil && family == BuildFamilyXT { + err = validateManifestV2PartSelect(artifacts, partitions) + } + if err != nil { + return err + } + + // Publish the artifact projection before validating the complete deployment geometry. + build.Partitions = partitions + build.Family = family + + want, err := positiveManifestUInt32ToInt( + fmt.Sprintf("%s deployment.numLpuNodes", gbuildManifestV2CapnpFile), + deployment.NumLpuNodes(), + ) + if err != nil { + return err + } + return validateManifestPartitionNodeCount(gbuildManifestV2CapnpFile, want, build, partialSelection, hxDoubleNodeCount, packagedNodes, partitionZeroNodes) +} + +func buildPartitionFromManifestV2(raw manifestcapnpv2.PartitionInfo) (BuildPartition, bool, error) { + if !raw.HasPartition() { + return BuildPartition{}, false, fmt.Errorf("%s artifact partition is missing partition ref", gbuildManifestV2CapnpFile) + } + ref, err := raw.Partition() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading %s artifact partition ref: %w", gbuildManifestV2CapnpFile, err) + } + if ref.DeviceType() != manifestcapnpv2.DeviceType_lpu { + return BuildPartition{}, false, nil + } + if raw.Detail().Which() != manifestcapnpv2.PartitionInfo_detail_Which_lpu || !raw.Detail().HasLpu() { + return BuildPartition{}, false, fmt.Errorf("%s artifact partition %d has deviceType lpu without LPU detail", gbuildManifestV2CapnpFile, ref.PartitionId()) + } + detail, err := raw.Detail().Lpu() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading %s LPU partition %d detail: %w", gbuildManifestV2CapnpFile, ref.PartitionId(), err) + } + topology, err := detail.Topology() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading %s LPU partition %d topology: %w", gbuildManifestV2CapnpFile, ref.PartitionId(), err) + } + path, err := detail.Path() + if err != nil { + return BuildPartition{}, false, fmt.Errorf("reading %s LPU partition %d path: %w", gbuildManifestV2CapnpFile, ref.PartitionId(), err) + } + + // Both families require a safe topology string, even when HX treats it as opaque. + subject := fmt.Sprintf("%s LPU partition %d", gbuildManifestV2CapnpFile, ref.PartitionId()) + if strings.ContainsAny(topology, "\x00\r\n") { + return BuildPartition{}, false, fmt.Errorf("%s topology must not contain NUL bytes or line breaks: %q", subject, topology) + } + + // Metadata selects HX; only the 16-chip, 16-device HX shape can omit it. + var partition BuildPartition + var compatible bool + if detail.HasTopologyMetadata() || (detail.NumChips() == 16 && detail.DevicesPerNode() == 16) { + partition, compatible, err = buildHXPartition(subject, topology, detail) + } else { + partition, err = buildXTPartition(subject, topology, detail) + } + if err != nil { + return BuildPartition{}, false, err + } + + // Validate shared artifact fields once, after the selected geometry is accepted. + partition.PartPath, err = cleanManifestRelativeBuildPath(subject+" path", path, false) + if err != nil { + return BuildPartition{}, false, err + } + partition.SourcePartitionID = int(ref.PartitionId()) + return partition, compatible, nil +} + +func validateManifestV2PartSelect(artifacts manifestcapnpv2.ArtifactInfo, partitions []BuildPartition) error { + if !artifacts.HasPartSelect() { + return nil + } + partSelect, err := artifacts.PartSelect() + if err != nil { + return fmt.Errorf("reading %s artifacts.partSelect: %w", gbuildManifestV2CapnpFile, err) + } + selected, err := partSelect.Partitions() + if err != nil { + return fmt.Errorf("reading %s artifacts.partSelect.partitions: %w", gbuildManifestV2CapnpFile, err) + } + if selected.Len() == 0 { + return fmt.Errorf("%s artifacts.partSelect.partitions is empty", gbuildManifestV2CapnpFile) + } + selectedLPU := make(map[int]struct{}, len(partitions)) + for index := 0; index < selected.Len(); index++ { + ref := selected.At(index) + if ref.DeviceType() != manifestcapnpv2.DeviceType_lpu { + continue + } + id := int(ref.PartitionId()) + if _, duplicate := selectedLPU[id]; duplicate { + return fmt.Errorf("%s artifacts.partSelect has duplicate LPU partition id %d", gbuildManifestV2CapnpFile, id) + } + partitionIndex := sort.Search(len(partitions), func(i int) bool { + return partitions[i].SourcePartitionID >= id + }) + if partitionIndex == len(partitions) || partitions[partitionIndex].SourcePartitionID != id { + return fmt.Errorf("%s artifacts.partSelect references unpackaged LPU partition id %d", gbuildManifestV2CapnpFile, id) + } + selectedLPU[id] = struct{}{} + } + if len(selectedLPU) != len(partitions) { + return fmt.Errorf("%s artifacts.partSelect selects %d LPU partitions, but artifacts package %d", gbuildManifestV2CapnpFile, len(selectedLPU), len(partitions)) + } + return nil +} + +func runtimeTokenEmbeddingsPathFromManifestV2(artifacts manifestcapnpv2.ArtifactInfo) (string, error) { + if !artifacts.HasRuntimeAssets() { + return "", nil + } + runtimeAssets, err := artifacts.RuntimeAssets() + if err != nil { + return "", fmt.Errorf("reading %s artifacts.runtimeAssets: %w", gbuildManifestV2CapnpFile, err) + } + if !runtimeAssets.HasTokenEmbeddingsPath() { + return "", nil + } + rawPath, err := runtimeAssets.TokenEmbeddingsPath() + if err != nil { + return "", fmt.Errorf("reading %s artifacts.runtimeAssets.tokenEmbeddingsPath: %w", gbuildManifestV2CapnpFile, err) + } + return cleanManifestRelativeBuildPath( + fmt.Sprintf("%s artifacts.runtimeAssets.tokenEmbeddingsPath", gbuildManifestV2CapnpFile), + rawPath, + false, + ) +} diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go new file mode 100644 index 000000000000..b047d49daee1 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -0,0 +1,374 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + "capnproto.org/go/capnp/v3" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + "github.com/stretchr/testify/require" +) + +func TestBuildFromGbuildManifestV2ProjectsRuntimeIO(t *testing.T) { + t.Parallel() + + t.Log("Build a valid revision 2 manifest") + manifest := newManifestV2ContractFixture(t) + + t.Log("Project the complete build contract") + build, err := buildFromGbuildManifestV2("gs://models/build", manifest) + require.NoError(t, err) + require.EqualValues(t, 4, build.IOFPGACount) + require.EqualValues(t, 2, build.IOFanoutFactor) + require.Len(t, build.Partitions, 1) + + t.Log("Accept compat FPGA I/O mode without changing the normalized build contract") + deployment, err := manifest.Deployment() + require.NoError(t, err) + runtimeIO, err := deployment.RuntimeIo() + require.NoError(t, err) + runtimeIO.SetReserved1(1) + compat, err := buildFromGbuildManifestV2("gs://models/build", manifest) + require.NoError(t, err) + require.Equal(t, build, compat) + + t.Log("Reject an unsupported contract revision before lowering the manifest") + manifest.SetContractRevision(manifest.ContractRevision() + 1) + _, err = buildFromGbuildManifestV2("gs://models/build", manifest) + require.ErrorContains(t, err, "contractRevision") +} + +func TestNormalizeBuildSnapshotRejectsInvalidHXArtifacts(t *testing.T) { + t.Log("Define malformed HX artifact inventories") + tests := []struct { + name string + wantErr string + }{ + {name: "duplicate LPU partition ID", wantErr: "repeats LPU partition ID 1"}, + {name: "topology metadata node-count mismatch", wantErr: "deployment.numLpuNodes = 2, but partition extents require 1 LPU nodes"}, + {name: "partial build", wantErr: "partSelect builds are not supported"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Acquire stable manifest bytes with the selected malformed HX inventory") + fixture := newV3CompilerFixture() + switch test.name { + case "duplicate LPU partition ID": + fixture.partitions = append(fixture.partitions, fixture.partitions[0]) + case "topology metadata node-count mismatch": + fixture.partitions[0].topology = v3HXTopologyFamily + fixture.partitions[0].topologyFamily = v3HXTopologyFamily + fixture.partitions[0].partitionShape = []uint32{16, 1, 1, 1} + case "partial build": + fixture.partSelect = true + } + snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) + + t.Log("Reject malformed HX artifacts during manifest normalization") + _, err := normalizeBuildSnapshot(snapshot) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func TestBuildFromGbuildManifestV2ValidatesPartSelect(t *testing.T) { + t.Parallel() + + t.Log("Define valid and invalid revision-2 partSelect inventories") + tests := []struct { + name string + selectedIDs []uint32 + wantErr string + }{ + {name: "valid", selectedIDs: []uint32{1, 0}}, + { + name: "duplicate", + selectedIDs: []uint32{0, 0}, + wantErr: "artifacts.partSelect has duplicate LPU partition id 0", + }, + { + name: "unpackaged", + selectedIDs: []uint32{0, 2}, + wantErr: "artifacts.partSelect references unpackaged LPU partition id 2", + }, + { + name: "incomplete", + selectedIDs: []uint32{0}, + wantErr: "artifacts.partSelect selects 1 LPU partitions, but artifacts package 2", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Build an unsorted two-partition V2 artifact inventory and explicit selector") + manifest := newManifestV2ContractFixture(t) + deployment, err := manifest.Deployment() + require.NoError(t, err) + deployment.SetNumLpuNodes(2) + artifacts, err := manifest.Artifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(2) + require.NoError(t, err) + setManifestV2LPUArtifact(t, partitions.At(0), 1) + setManifestV2LPUArtifact(t, partitions.At(1), 0) + partSelect, err := artifacts.NewPartSelect() + require.NoError(t, err) + selected, err := partSelect.NewPartitions(int32(len(test.selectedIDs))) + require.NoError(t, err) + for index, id := range test.selectedIDs { + selected.At(index).SetPartitionId(id) + selected.At(index).SetDeviceType(manifestcapnpv2.DeviceType_lpu) + } + + t.Log("Validate against the sorted unique XT inventory produced by classification") + build, err := buildFromGbuildManifestV2("gs://models/build", manifest) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Len(t, build.Partitions, 2) + require.Equal(t, 0, build.Partitions[0].SourcePartitionID) + require.Equal(t, 1, build.Partitions[1].SourcePartitionID) + }) + } +} + +func TestBuildFromGbuildManifestV2ValidatesRuntimeInvariants(t *testing.T) { + t.Parallel() + + t.Log("Define invalid revision-2 runtime I/O contracts") + tests := []struct { + name string + batchSize, ioCount, fanoutFactor uint32 + wantErr string + }{ + { + name: "positive batch size", batchSize: 0, ioCount: 4, fanoutFactor: 2, + wantErr: "deployment.program.batchSize must be >= 1, got 0", + }, + { + name: "batch divisibility", batchSize: 3, ioCount: 4, fanoutFactor: 2, + wantErr: "deployment.program.batchSize 3 must be divisible by deployment.runtimeIo.ioFpgaCount 4", + }, + { + name: "fanout divisibility", batchSize: 8, ioCount: 4, fanoutFactor: 3, + wantErr: "deployment.program.batchSize per endpoint 2 must be divisible by deployment.runtimeIo.fanoutFactor 3", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Build a manifest with malformed runtime metadata") + manifest := newManifestV2ContractFixture(t) + deployment, err := manifest.Deployment() + require.NoError(t, err) + program, err := deployment.Program() + require.NoError(t, err) + program.SetBatchSize(test.batchSize) + runtimeIO, err := deployment.RuntimeIo() + require.NoError(t, err) + runtimeIO.SetIoFpgaCount(test.ioCount) + runtimeIO.SetFanoutFactor(test.fanoutFactor) + + t.Log("Reject it before scheduler-facing artifact projection") + _, err = buildFromGbuildManifestV2("gs://models/build", manifest) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { + t.Parallel() + + type geometryFixture struct { + numChips, devicesPerNode, numNodes uint32 + } + type embeddingFixture struct { + path string + cpu, standalone bool + } + + t.Log("Define revision-2 path, XT geometry, node-count, and embedding contracts") + tests := []struct { + name, partitionPath, tokenizerPath, wantErr, wantAsset string + geometry *geometryFixture + embedding *embeddingFixture + }{ + {name: "partition traversal", partitionPath: "../outside", wantErr: "LPU partition 0 path"}, + {name: "absolute tokenizer", tokenizerPath: "/outside", wantErr: "model.tokenizer.path"}, + { + name: "runtime asset line break", embedding: &embeddingFixture{path: "runtime\nasset", cpu: true}, + wantErr: "artifacts.runtimeAssets.tokenEmbeddingsPath", + }, + {name: "missing numChips", geometry: &geometryFixture{devicesPerNode: 8, numNodes: 1}, wantErr: "numChips must be >= 1"}, + { + name: "topology and numChips mismatch", geometry: &geometryFixture{numChips: 16, devicesPerNode: 8, numNodes: 2}, + wantErr: "topology chip count 8 does not match numChips 16", + }, + {name: "missing devicesPerNode", geometry: &geometryFixture{numChips: 8, numNodes: 1}, wantErr: "devicesPerNode must be >= 1"}, + { + name: "deployment node count mismatch", geometry: &geometryFixture{numChips: 8, devicesPerNode: 8, numNodes: 2}, + wantErr: "LPU partitions use 1 LPU nodes, want deployment.numLpuNodes 2", + }, + { + name: "asset without CPU embeddings", embedding: &embeddingFixture{path: "runtime/text_embeddings.npz"}, + wantErr: "requires supportsCpuEmbeddings=true", + }, + { + name: "missing standalone asset", embedding: &embeddingFixture{cpu: true, standalone: true}, + wantErr: "is required when standaloneTokenEmbeddings=true", + }, + { + name: "valid standalone asset", embedding: &embeddingFixture{path: "runtime/text_embeddings.npz", cpu: true, standalone: true}, + wantAsset: "runtime/text_embeddings.npz", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Apply one contract mutation to an otherwise valid revision-2 manifest") + manifest := newManifestV2ContractFixture(t) + if test.partitionPath != "" || test.geometry != nil { + artifacts, err := manifest.Artifacts() + require.NoError(t, err) + partitions, err := artifacts.Partitions() + require.NoError(t, err) + detail, err := partitions.At(0).Detail().Lpu() + require.NoError(t, err) + if test.partitionPath != "" { + require.NoError(t, detail.SetPath(test.partitionPath)) + } + if test.geometry != nil { + detail.SetNumChips(test.geometry.numChips) + detail.SetDevicesPerNode(test.geometry.devicesPerNode) + deployment, err := manifest.Deployment() + require.NoError(t, err) + deployment.SetNumLpuNodes(test.geometry.numNodes) + } + } + if test.tokenizerPath != "" { + model, err := manifest.NewModel() + require.NoError(t, err) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath(test.tokenizerPath)) + } + if test.embedding != nil { + deployment, err := manifest.Deployment() + require.NoError(t, err) + deployment.SetNumLpuNodes(2) + program, err := deployment.Program() + require.NoError(t, err) + program.SetSupportsCpuEmbeddings(test.embedding.cpu) + program.SetStandaloneTokenEmbeddings(test.embedding.standalone) + artifacts, err := manifest.Artifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(2) + require.NoError(t, err) + setManifestV2LPUArtifact(t, partitions.At(0), 0) + setManifestV2LPUArtifact(t, partitions.At(1), 1) + if test.embedding.path != "" { + runtimeAssets, err := artifacts.NewRuntimeAssets() + require.NoError(t, err) + require.NoError(t, runtimeAssets.SetTokenEmbeddingsPath(test.embedding.path)) + } + } + + t.Log("Accept only the safe and internally consistent revision-2 contract") + build, err := buildFromGbuildManifestV2("gs://models/build", manifest) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.wantAsset, build.RuntimeTokenEmbeddingsPath) + }) + } +} + +func TestNormalizedV2BuildTreatsManifestBuildIDAsOpaque(t *testing.T) { + t.Parallel() + + t.Log("Define relocated revision-2 build references and opaque manifest identities") + tests := []struct { + name string + ref string + buildID string + }{ + { + name: "local copy", + ref: "file:///models/copied-build", + buildID: "original-producer-id", + }, + { + name: "different GCS bucket and path", + ref: "gs://new-bucket/relocated/renamed-build", + buildID: "original-producer-id", + }, + { + name: "missing producer identity", + ref: "file:///models/build-without-producer-id", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Create a revision-2 snapshot under its deployment locator") + manifest := newManifestV2ContractFixture(t) + if test.buildID != "" { + build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) + require.NoError(t, err) + require.NoError(t, build.SetText(11, test.buildID)) + require.NoError(t, manifest.SetReserved3(build.ToPtr())) + } + payload, err := manifest.Message().Marshal() + require.NoError(t, err) + snapshot := &BuildSnapshot{ + ref: test.ref, + contentID: "sha256:test", + manifestBytes: payload, + } + + t.Log("Use the deployment locator without interpreting manifest build.buildId") + normalized, err := normalizeBuildSnapshot(snapshot) + require.NoError(t, err) + require.Equal(t, test.ref, normalized.build.Path) + }) + } +} + +func TestBuildFromGbuildManifestV2RejectsNonLPUArtifactForLPUOnly(t *testing.T) { + t.Parallel() + + t.Log("Build an lpuOnly manifest with one LPU and one CUDA artifact") + manifest := newManifestV2ContractFixture(t) + deployment, err := manifest.Deployment() + require.NoError(t, err) + deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpuOnly) + artifacts, err := manifest.Artifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(2) + require.NoError(t, err) + setManifestV2LPUArtifact(t, partitions.At(0), 0) + cudaRef, err := partitions.At(1).NewPartition() + require.NoError(t, err) + cudaRef.SetPartitionId(1) + cudaRef.SetDeviceType(manifestcapnpv2.DeviceType_cuda) + + t.Log("Reject the mixed-device lpuOnly artifact set") + _, err = buildFromGbuildManifestV2("gs://bucket/registry/build-id", manifest) + require.ErrorContains(t, err, "deployment.compilationMode lpuOnly requires every artifact partition to use deviceType lpu") +} diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go new file mode 100644 index 000000000000..170197299220 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -0,0 +1,150 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "fmt" + "slices" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + // WorkloadDigestAnnotation records the immutable Dynamo workload projection digest on rendered objects. + WorkloadDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-workload-digest" + // DGDGenerationAnnotation records the source DGD generation on rendered objects. + DGDGenerationAnnotation = "scheduling.lpu.nvidia.com/dgd-generation" + // DGDUIDAnnotation records the source DGD UID on rendered objects. + DGDUIDAnnotation = "scheduling.lpu.nvidia.com/dgd-uid" + // ExecutionBackendAnnotation selects the LPX execution backend on rendered objects. + ExecutionBackendAnnotation = commonconsts.KubeAnnotationLPXExecutionBackend + // WorkloadModeAnnotation records the projected LPX workload mode on rendered objects. + WorkloadModeAnnotation = "scheduling.lpu.nvidia.com/workload-mode" +) + +// WorkloadDigest is the SHA-256 identity of a projected LPX workload. +type WorkloadDigest [32]byte + +// String returns the digest in canonical sha256-prefixed hexadecimal form. +func (d WorkloadDigest) String() string { + return fmt.Sprintf("sha256:%x", d[:]) +} + +// ModelProjectionInput is the complete producer-owned input to one component's +// immutable model projections. It excludes Grove materialization and scheduler output. +type ModelProjectionInput struct { + // Pipeline selects the DGD pipeline shape being projected. + Pipeline Pipeline + // Models contains the validated, nonempty logical model identities in canonical order. + Models []string + // RuntimeBuildRef is the build reference projected into runtime configuration. + RuntimeBuildRef string + // BuildSnapshot is the normalized immutable build input. + BuildSnapshot NormalizedBuildSnapshot + // ModelSettings contains the optional per-model JSON settings. + ModelSettings json.RawMessage +} + +// ModelProjection holds scheduler request inputs and runtime rendering state +// derived for one model. Completed component data is shared read-only; only the +// resolver assigns the stage before publishing the immutable projections. +type ModelProjection struct { + digest WorkloadDigest + runtimeBuildRef string + model string + stage string + pipeline Pipeline + configuredBuild Build + allocationMetadata json.RawMessage + // partitions retains immutable physical build evidence before runtime collapse. + partitions []BuildPartition + connectors []lpxv1alpha1.PropSyncConnectorRequest + agentReplicas int +} + +// Digest returns the immutable projection digest. The receiver must be non-nil. +func (p *ModelProjection) Digest() WorkloadDigest { + return p.digest +} + +// Model returns the logical model identity. The receiver must be non-nil. +func (p *ModelProjection) Model() string { + return p.model +} + +// RequestSpec returns a fresh node-local LPR spec. The materialization owner +// supplies only workload references; scheduler-owned placement fields do not +// exist in this request arm. The receiver must be non-nil. A nil cyborgRef is +// supported and omits the Cyborg reference. Neither input is mutated. +func (p *ModelProjection) RequestSpec( + namespace string, + podGangName string, + cyborgRef *lpxv1alpha1.CyborgPodCliqueReference, +) lpxv1alpha1.LPUPipelineRequestSpec { + connectors := make([]lpxv1alpha1.PropSyncConnectorRequest, len(p.connectors)) + for i := range p.connectors { + p.connectors[i].DeepCopyInto(&connectors[i]) + } + partitions := make([]lpxv1alpha1.PartitionRequest, len(p.partitions)) + mappings := make([]lpxv1alpha1.NodeLocalPartitionMapping, len(p.partitions)) + for index, partition := range p.partitions { + partitionID := fmt.Sprintf("partition-%03d", index) + request := lpxv1alpha1.PartitionRequest{ + ID: partitionID, + Ordinal: int64(index), + CompilerPartitionID: int64(uint32(partition.SourcePartitionID)), + } + if p.configuredBuild.Family == BuildFamilyXT { + shape, _, _ := xtShape(partition.Topology.ChipCount) + request.XtShape = &shape + } + if partition.HXExtent != nil { + extent := slices.Clone(partition.HXExtent) + request.Extent = &extent + } + partitions[index] = request + mappings[index] = lpxv1alpha1.NodeLocalPartitionMapping{ + ModelPartitionID: int64(index), + PartitionID: partitionID, + } + } + ns := namespace + spec := lpxv1alpha1.LPUPipelineRequestSpec{ + AllocationMetadata: runtime.RawExtension{Raw: slices.Clone(p.allocationMetadata)}, + PodGangRef: lpxv1alpha1.NamespacedName{Name: podGangName, Namespace: &ns}, + PropSyncConnectors: connectors, + TargetFamily: lpxv1alpha1.TargetFamily(p.configuredBuild.Family), + WorkloadMode: p.schedulerWorkloadMode(), + ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, + NodeLocal: &lpxv1alpha1.NodeLocalRequest{ + Model: p.model, + PartitionMappings: mappings, + }, + Partitions: partitions, + } + if cyborgRef != nil { + spec.CyborgPodCliqueRef = cyborgRef.DeepCopy() + } + return spec +} + +// schedulerWorkloadMode translates the normalized build family and pipeline at the LPX wire boundary. +func (p *ModelProjection) schedulerWorkloadMode() lpxv1alpha1.WorkloadMode { + // HX and XT have separate wire values for the same two runtime shapes. + if p.configuredBuild.Family == BuildFamilyHX { + if p.pipeline == PipelineLPX { + return lpxv1alpha1.WorkloadModeV3HxStrictHybrid + } + return lpxv1alpha1.WorkloadModeV3HxLPUOnly + } + if p.pipeline == PipelineLPX { + return lpxv1alpha1.WorkloadModeV2StrictHybrid + } + return lpxv1alpha1.WorkloadModeV2LPUOnly +} diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go new file mode 100644 index 000000000000..3d2224d4c701 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -0,0 +1,366 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" +) + +var ( + errRelativeBuildRef = errors.New("relative ref") + errBuildFileTooLarge = errors.New("build metadata file exceeds its acquisition limit") +) + +const ( + // BuildSchemeFile identifies builds stored on the local filesystem. + BuildSchemeFile = "file" + // BuildSchemeGCS identifies builds stored in Google Cloud Storage. + BuildSchemeGCS = "gs" +) + +// ModelRegistry resolves build references and acquires their compiler metadata. +type ModelRegistry struct { + registryURL *url.URL + mxClient modelpb.ModelServiceClient +} + +// NewModelRegistry constructs a registry rooted at modelRegistryURL. An empty +// URL is supported for absolute build references. mxClient may be nil when GCS +// downloads are not required. +func NewModelRegistry(modelRegistryURL string, mxClient modelpb.ModelServiceClient) (*ModelRegistry, error) { + var registryURL *url.URL + + if modelRegistryURL != "" { + var err error + registryURL, err = parseBuildRef(modelRegistryURL) + if err != nil { + return nil, err + } + switch registryURL.Scheme { + case BuildSchemeFile, BuildSchemeGCS: + default: + return nil, fmt.Errorf("unsupported model registry URL scheme %q", registryURL.Scheme) + } + } + + return &ModelRegistry{ + registryURL: registryURL, + mxClient: mxClient, + }, nil +} + +// EnsureDownloaded ensures buildURL is locally available and reports whether +// the download is complete. For GCS builds it may initiate or advance a +// ModelExpress download. The receiver must be non-nil and is not mutated. +func (r *ModelRegistry) EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) { + switch buildURL.Scheme { + case BuildSchemeFile: + return true, nil + case BuildSchemeGCS: + if r.mxClient == nil { + return false, fmt.Errorf("Model Express client is required for GCS model downloads") + } + + // Request the GCS build through the current Model Express client and classify its first status. + rpcCtx, cancel := context.WithCancel(ctx) + defer cancel() + + modelName := buildURL.String() + + stream, err := r.mxClient.EnsureModelDownloaded(rpcCtx, &modelpb.ModelDownloadRequest{ + ModelName: modelName, + Provider: modelpb.ModelProvider_GCS, + }) + if err != nil { + return false, fmt.Errorf("ensure model express download for %q: %w", modelName, err) + } + + update, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return false, fmt.Errorf("model express returned no status for %q", modelName) + } + return false, fmt.Errorf("receive model express download status for %q: %w", modelName, err) + } + + switch update.GetStatus() { + case modelpb.ModelStatus_DOWNLOADED: + return true, nil + case modelpb.ModelStatus_DOWNLOADING: + return false, nil + case modelpb.ModelStatus_ERROR: + message := update.GetMessage() + if message == "" { + message = "model express reported download error" + } + return false, fmt.Errorf("model express download failed for %q: %s", modelName, message) + default: + return false, fmt.Errorf("model express returned unexpected status %s for %q", update.GetStatus(), modelName) + } + default: + return false, fmt.Errorf("unsupported build download scheme %q", buildURL.Scheme) + } +} + +// BuildURL resolves id against the configured registry URL. The receiver must +// be non-nil and is not mutated. +func (r *ModelRegistry) BuildURL(id string) (*url.URL, error) { + refURL, err := parseBuildRef(id) + if err == nil { + return refURL, nil + } + if !errors.Is(err, errRelativeBuildRef) { + return nil, err + } + + if r.registryURL == nil { + return nil, fmt.Errorf("model registry URL is not configured") + } + + relativeID := strings.TrimSpace(id) + if !filepath.IsLocal(relativeID) { + return nil, fmt.Errorf("build ID %q must remain within the configured model registry", id) + } + relativeID = filepath.Clean(relativeID) + if relativeID == "" || relativeID == "." { + return nil, fmt.Errorf("empty ref") + } + + if r.registryURL.Scheme == BuildSchemeFile { + return &url.URL{ + Scheme: BuildSchemeFile, + Path: filepath.Clean(filepath.Join(r.registryURL.Path, relativeID)), + }, nil + } + + return r.registryURL.JoinPath(relativeID), nil +} + +func parseBuildRef(ref string) (*url.URL, error) { + ref = strings.TrimSpace(ref) + if ref == "" { + return nil, fmt.Errorf("empty ref") + } + cleanRef := filepath.Clean(ref) + if filepath.IsAbs(cleanRef) { + return &url.URL{Scheme: BuildSchemeFile, Path: cleanRef}, nil + } + + uri, err := url.Parse(ref) + if err != nil { + return nil, fmt.Errorf("unable to parse ref %q: %w", ref, err) + } + if uri.Scheme == "" { + return nil, fmt.Errorf("ref %q must be an absolute path or URL: %w", ref, errRelativeBuildRef) + } + if uri.Scheme == BuildSchemeFile { + if uri.Host != "" { + return nil, fmt.Errorf("invalid ref %q: unexpected file host %q", ref, uri.Host) + } + if !filepath.IsAbs(uri.Path) { + return nil, fmt.Errorf("invalid ref %q: file path must be absolute", ref) + } + return &url.URL{Scheme: BuildSchemeFile, Path: filepath.Clean(uri.Path)}, nil + } + if uri.Host == "" { + return nil, fmt.Errorf("invalid ref %q: missing host", ref) + } + + return uri, nil +} + +// readBuildFileBounded reads one relative file. The receiver and buildURL must +// be non-nil, and maxBytes must be non-negative. +func (r *ModelRegistry) readBuildFileBounded( + ctx context.Context, + buildURL *url.URL, + relativePath string, + maxBytes int, +) ([]byte, error) { + switch buildURL.Scheme { + case BuildSchemeFile: + fileURL := buildURL.JoinPath(relativePath) + file, err := os.Open(filepath.Clean(fileURL.Path)) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", fileURL.String(), err) + } + defer func() { + _ = file.Close() + }() + if info, statErr := file.Stat(); statErr == nil && info.Size() > int64(maxBytes) { + return nil, fmt.Errorf( + "%w: %s is %d bytes, limit is %d", + errBuildFileTooLarge, + fileURL.String(), + info.Size(), + maxBytes, + ) + } + data, err := io.ReadAll(io.LimitReader(file, int64(maxBytes)+1)) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", fileURL.String(), err) + } + if len(data) > maxBytes { + return nil, fmt.Errorf( + "%w: %s is larger than %d bytes", + errBuildFileTooLarge, + fileURL.String(), + maxBytes, + ) + } + return data, nil + case BuildSchemeGCS: + if r.mxClient == nil { + return nil, fmt.Errorf("Model Express client is required for GCS model registry reads of %q", buildURL.String()) + } + + // Release the metadata stream even when chunk validation stops the read early. + rpcCtx, cancel := context.WithCancel(ctx) + defer cancel() + + modelName := buildURL.String() + stream, err := r.mxClient.StreamModelFiles(rpcCtx, &modelpb.ModelFilesRequest{ + ModelName: modelName, + Provider: modelpb.ModelProvider_GCS, + FileSelector: &modelpb.ModelFileSelector{ + Paths: []string{relativePath}, + }, + }) + if err != nil { + return nil, fmt.Errorf("streaming %s from %q with Model Express: %w", relativePath, modelName, err) + } + + var data []byte + receivedChunk := false + completedFile := false + + for { + chunk, err := stream.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("receiving %s from %q with Model Express: %w", relativePath, modelName, err) + } + if completedFile { + return nil, fmt.Errorf("received extra chunk for %s from %q after final chunk", relativePath, modelName) + } + if chunk.GetRelativePath() != relativePath { + return nil, fmt.Errorf("received unexpected file %q for %s from %q", chunk.GetRelativePath(), relativePath, modelName) + } + if chunk.GetOffset() != uint64(len(data)) { + return nil, fmt.Errorf("received out-of-order chunk for %s from %q: offset %d, want %d", relativePath, modelName, chunk.GetOffset(), len(data)) + } + + receivedChunk = true + chunkData := chunk.GetData() + if len(chunkData) > maxBytes-len(data) { + return nil, fmt.Errorf( + "%w: %s from %q is larger than %d bytes", + errBuildFileTooLarge, + relativePath, + modelName, + maxBytes, + ) + } + data = append(data, chunkData...) + + if chunk.GetIsLastChunk() { + completedFile = true + if totalSize := chunk.GetTotalSize(); totalSize != uint64(len(data)) { + return nil, fmt.Errorf("received incomplete %s from %q: final size %d, want %d", relativePath, modelName, len(data), totalSize) + } + } + } + + if !receivedChunk { + return nil, fmt.Errorf("Model Express returned no chunks for %s from %q", relativePath, modelName) + } + if !completedFile { + return nil, fmt.Errorf("Model Express stream for %s from %q ended before final chunk", relativePath, modelName) + } + + return data, nil + default: + return nil, fmt.Errorf("unsupported build metadata scheme %q", buildURL.Scheme) + } +} + +func (r *ModelRegistry) listBuildFiles(ctx context.Context, buildURL *url.URL) ([]string, error) { + var ( + paths []string + err error + ) + switch buildURL.Scheme { + case BuildSchemeFile: + paths, err = localBuildFilePaths(buildURL.Path) + case BuildSchemeGCS: + if r.mxClient == nil { + return nil, fmt.Errorf("Model Express client is required for GCS model registry reads of %q", buildURL.String()) + } + + // List the GCS build files through Model Express before validating the inventory. + modelName := buildURL.String() + list, listErr := r.mxClient.ListModelFiles(ctx, &modelpb.ModelFilesRequest{ + ModelName: modelName, + Provider: modelpb.ModelProvider_GCS, + }) + if listErr != nil { + return nil, fmt.Errorf("listing files from %q with Model Express: %w", modelName, listErr) + } + paths = make([]string, 0, len(list.GetFiles())) + for _, file := range list.GetFiles() { + if path := file.GetRelativePath(); path != "" { + paths = append(paths, path) + } + } + default: + err = fmt.Errorf("unsupported build metadata scheme %q", buildURL.Scheme) + } + if err != nil { + return nil, err + } + return normalizeBuildFilePaths(paths) +} + +func localBuildFilePaths(root string) ([]string, error) { + cleanRoot := filepath.Clean(root) + resolvedRoot, err := filepath.EvalSymlinks(cleanRoot) + if err != nil { + return nil, fmt.Errorf("resolving build directory %q: %w", root, err) + } + + // Keep a malformed manifest directory in the inventory so its required file read fails. + manifestPath := filepath.Join(resolvedRoot, gbuildManifestV2CapnpFile) + paths := make([]string, 0) + err = filepath.WalkDir(resolvedRoot, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && path != manifestPath { + return nil + } + rel, err := filepath.Rel(resolvedRoot, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("listing build files under %q: %w", root, err) + } + return paths, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/model_registry_test.go b/deploy/operator/internal/dynamo/lpx/model_registry_test.go new file mode 100644 index 000000000000..9e91d0833254 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/model_registry_test.go @@ -0,0 +1,360 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "errors" + "io" + "net/url" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" +) + +func TestRegistryBuildURL(t *testing.T) { + t.Parallel() + + t.Log("Reject unsupported registry schemes at construction") + _, err := NewModelRegistry("https://unsupported.invalid", nil) + require.EqualError(t, err, `unsupported model registry URL scheme "https"`) + + t.Log("Define absolute and registry-relative model build references") + tests := []struct { + name string + registryURL string + id string + want string + wantErrMessage string + }{ + {name: "direct gcs url", id: "gs://bucket/model/build", want: "gs://bucket/model/build"}, + {name: "direct file url", id: "file:///models/build", want: "file:///models/build"}, + {name: "relative id with gcs registry", registryURL: "gs://bucket/registry", id: "model/build", want: "gs://bucket/registry/model/build"}, + {name: "relative id with file registry", registryURL: "file:///registry", id: "model/build", want: "file:///registry/model/build"}, + {name: "absolute path", id: "/models/build", want: "file:///models/build"}, + {name: "relative id escaping registry", registryURL: "gs://bucket/registry", id: "../model/build", wantErrMessage: "build ID \"../model/build\" must remain within the configured model registry"}, + {name: "relative id without registry", id: "model/build", wantErrMessage: "model registry URL is not configured"}, + {name: "empty id", wantErrMessage: "empty ref"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Construct the selected model registry") + registry, err := NewModelRegistry(tt.registryURL, nil) + require.NoError(t, err) + + t.Log("Resolve the selected build reference") + buildURL, err := registry.BuildURL(tt.id) + if tt.wantErrMessage != "" { + require.EqualError(t, err, tt.wantErrMessage) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, buildURL.String()) + }) + } +} + +func TestLocalModelRegistrySnapshot_DanglingGbuildCapnpSymlinkFailsClosed(t *testing.T) { + t.Parallel() + + t.Log("Create a dangling symlink for the required binary manifest") + registryDir := t.TempDir() + buildDir := filepath.Join(registryDir, "build-id") + require.NoError(t, os.MkdirAll(buildDir, 0o700)) + require.NoError(t, os.Symlink("missing-manifest.v2.capnp.bin", filepath.Join(buildDir, gbuildManifestV2CapnpFile))) + + t.Log("Normalize the local registry build") + registry, err := NewModelRegistry(registryDir, nil) + require.NoError(t, err) + _, err = normalizeRegistryFixtureBuild(t.Context(), registry, "build-id") + + t.Log("Fail closed while preserving the missing manifest identity") + require.ErrorContains(t, err, gbuildManifestV2CapnpFile) +} + +func TestLocalModelRegistrySnapshot_ReportsMetadataDirectories(t *testing.T) { + t.Parallel() + + t.Log("Create a directory where compiler metadata must be a regular file") + registryDir := t.TempDir() + buildDir := filepath.Join(registryDir, "build-id") + require.NoError(t, os.MkdirAll(buildDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, gbuildManifestJSONFile), []byte(`{}`), 0o600)) + require.NoError(t, os.Mkdir(filepath.Join(buildDir, gbuildManifestV2CapnpFile), 0o755)) + + t.Log("Normalize the invalid local registry build") + registry, err := NewModelRegistry(registryDir, nil) + require.NoError(t, err) + _, err = normalizeRegistryFixtureBuild(t.Context(), registry, "build-id") + + t.Log("Report the invalid metadata path") + require.Error(t, err) + require.ErrorContains(t, err, gbuildManifestV2CapnpFile) +} + +func TestReadBuildFileBoundsLocalFileAtAcquisition(t *testing.T) { + t.Parallel() + + t.Log("Create a five-byte local compiler metadata file") + buildDir := t.TempDir() + const metadataFile = "metadata.bin" + require.NoError(t, os.WriteFile(filepath.Join(buildDir, metadataFile), []byte("12345"), 0o600)) + buildURL := &url.URL{Scheme: BuildSchemeFile, Path: buildDir} + registry := &ModelRegistry{} + + t.Log("Read the file at the inclusive acquisition limit") + data, err := registry.readBuildFileBounded(t.Context(), buildURL, metadataFile, 5) + require.NoError(t, err) + require.Equal(t, []byte("12345"), data) + + t.Log("Reject the same file above a four-byte limit") + _, err = registry.readBuildFileBounded(t.Context(), buildURL, metadataFile, 4) + require.ErrorIs(t, err, errBuildFileTooLarge) +} + +func TestReadBuildFileBoundsGCSAccumulatedChunks(t *testing.T) { + t.Parallel() + + t.Log("Create a GCS metadata stream whose chunks exceed the aggregate limit") + const metadataFile = "metadata.bin" + client := &fakeModelServiceClient{ + fileStreams: []*fakeModelFileStream{ + {chunks: modelFileChunks(metadataFile, "12", "345")}, + }, + } + registry := &ModelRegistry{mxClient: client} + buildURL := &url.URL{Scheme: BuildSchemeGCS, Host: "bucket", Path: "/model/build"} + + t.Log("Reject the accumulated stream before accepting oversized metadata") + _, err := registry.readBuildFileBounded(t.Context(), buildURL, metadataFile, 4) + require.ErrorIs(t, err, errBuildFileTooLarge) + require.ErrorContains(t, err, "larger than 4 bytes") + + t.Log("Release the rejected metadata stream without canceling the caller") + require.ErrorIs(t, client.filesContext.Err(), context.Canceled) + require.NoError(t, t.Context().Err()) +} + +func TestGCSModelRegistrySnapshot_RequiresModelExpressClient(t *testing.T) { + t.Parallel() + + t.Log("Construct a GCS registry without a Model Express client") + registry, err := NewModelRegistry("gs://bucket/registry", nil) + if err != nil { + t.Fatalf("NewModelRegistry() error = %v", err) + } + + t.Log("Attempt to normalize a remote build") + _, err = normalizeRegistryFixtureBuild(context.Background(), registry, "model/build") + if err == nil { + t.Fatal("snapshot normalization error = nil, want error") + } + + t.Log("Require Model Express for GCS registry reads") + if !strings.Contains(err.Error(), "Model Express client is required for GCS model registry reads") { + t.Fatalf("snapshot normalization error = %v, want Model Express required error", err) + } +} + +func TestGCSModelRegistrySnapshot_StreamFailures(t *testing.T) { + t.Parallel() + + t.Log("Define Model Express RPC and chunk-stream failures") + rpcErr := errors.New("rpc unavailable") + recvErr := errors.New("stream interrupted") + + tests := []struct { + name string + filesErr error + recvErr error + chunkPath string + chunkData string + chunkOffset uint64 + chunkSize uint64 + lastChunk bool + wantErr error + wantErrContains string + }{ + { + name: "stream rpc error", + filesErr: rpcErr, + wantErr: rpcErr, + }, + { + name: "receive error", + recvErr: recvErr, + wantErr: recvErr, + }, + { + name: "empty stream", + wantErrContains: "returned no chunks", + }, + { + name: "unexpected relative path", chunkPath: "other.json", chunkData: "{}", chunkSize: 2, lastChunk: true, + wantErrContains: `unexpected file "other.json"`, + }, + { + name: "out of order chunk", chunkPath: gbuildManifestV2CapnpFile, chunkData: "{}", + chunkOffset: 1, chunkSize: 2, lastChunk: true, + wantErrContains: "out-of-order chunk", + }, + { + name: "missing final chunk", chunkPath: gbuildManifestV2CapnpFile, chunkData: `{"topologies":[]}`, + wantErrContains: "ended before final chunk", + }, + { + name: "final size mismatch", chunkPath: gbuildManifestV2CapnpFile, chunkData: "{}", + chunkSize: 3, lastChunk: true, + wantErrContains: "incomplete", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Construct the selected Model Express stream failure") + client := &fakeModelServiceClient{filesErr: tt.filesErr} + if tt.filesErr == nil { + stream := &fakeModelFileStream{err: tt.recvErr} + if tt.chunkPath != "" { + stream.chunks = []*modelpb.FileChunk{{ + RelativePath: tt.chunkPath, Data: []byte(tt.chunkData), Offset: tt.chunkOffset, + TotalSize: tt.chunkSize, IsLastChunk: tt.lastChunk, + }} + } + client.fileStreams = []*fakeModelFileStream{stream} + } + client.list = &modelpb.ModelFileList{ + Files: []*modelpb.ModelFileInfo{{RelativePath: gbuildManifestV2CapnpFile}}, + } + + registry, err := NewModelRegistry("gs://bucket/registry", client) + if err != nil { + t.Fatalf("NewModelRegistry() error = %v", err) + } + + t.Log("Normalize through the selected failing stream") + _, err = normalizeRegistryFixtureBuild(context.Background(), registry, "model/build") + if err == nil { + t.Fatal("snapshot normalization error = nil, want error") + } + + t.Log("Preserve the selected transport or protocol failure") + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("snapshot normalization error = %v, want %v", err, tt.wantErr) + } + if tt.wantErrContains != "" && !strings.Contains(err.Error(), tt.wantErrContains) { + t.Fatalf("snapshot normalization error = %v, want substring %q", err, tt.wantErrContains) + } + }) + } +} + +func TestDefinitiveBuildMemberAbsenceIncludesLocalENOTDIR(t *testing.T) { + t.Parallel() + + t.Log("Classify local ENOTDIR as definitive build-member absence") + err := &os.PathError{Op: "open", Path: "/build/manifest.v2.capnp.bin", Err: syscall.ENOTDIR} + require.True(t, isDefinitiveBuildMemberAbsence(err)) + + t.Log("Keep transient read failures outside definitive absence") + require.False(t, isDefinitiveBuildMemberAbsence(errors.New("temporary read failure"))) +} + +func TestRegistryEnsureDownloadedSkipsLocalBuilds(t *testing.T) { + t.Log("Construct a registry without Model Express") + registry, err := NewModelRegistry("", nil) + require.NoError(t, err) + + t.Log("Accept an already-local build") + ready, err := registry.EnsureDownloaded(context.Background(), mustParseURL(t, "file:///models/build")) + require.NoError(t, err) + require.True(t, ready) +} + +func TestRegistryEnsureDownloadedReturnsFirstStatus(t *testing.T) { + t.Log("Define terminal and in-progress first Model Express statuses") + tests := []struct { + name string + status modelpb.ModelStatus + message string + }{ + {name: "downloaded", status: modelpb.ModelStatus_DOWNLOADED}, + {name: "downloading", status: modelpb.ModelStatus_DOWNLOADING}, + {name: "error", status: modelpb.ModelStatus_ERROR, message: "download failed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Construct the selected first-status Model Express stream") + client := &fakeModelServiceClient{ + stream: &fakeModelDownloadStream{ + update: &modelpb.ModelStatusUpdate{ + Status: tt.status, + Message: ptr.To(tt.message), + }, + }, + } + registry := &ModelRegistry{mxClient: client} + buildURL := mustParseURL(t, "gs://bucket/models/build") + + t.Log("Classify the first Model Express download status") + ready, err := registry.EnsureDownloaded(context.Background(), buildURL) + require.NotNil(t, client.request) + require.Equal(t, buildURL.String(), client.request.ModelName) + require.Equal(t, modelpb.ModelProvider_GCS, client.request.Provider) + if tt.status == modelpb.ModelStatus_ERROR { + require.ErrorContains(t, err, tt.message) + require.False(t, ready) + return + } + require.NoError(t, err) + require.Equal(t, tt.status == modelpb.ModelStatus_DOWNLOADED, ready) + }) + } +} + +func TestRegistryEnsureDownloadedPropagatesTransportFailures(t *testing.T) { + for _, scenario := range []struct { + name string + rpc bool + err error + }{ + {name: "RPC failure", rpc: true, err: errors.New("rpc unavailable")}, + {name: "receive failure", err: errors.New("stream interrupted")}, + {name: "empty stream", err: io.EOF}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Construct the selected transport failure") + client := &fakeModelServiceClient{stream: &fakeModelDownloadStream{err: scenario.err}} + if scenario.rpc { + client.err = scenario.err + } + registry := &ModelRegistry{mxClient: client} + + t.Log("Reject the failed or empty stream before classifying readiness") + ready, err := registry.EnsureDownloaded(t.Context(), mustParseURL(t, "gs://bucket/models/build")) + if scenario.err == io.EOF { + require.ErrorContains(t, err, "returned no status") + } else { + require.ErrorIs(t, err, scenario.err) + } + require.False(t, ready) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/modelexpress.go b/deploy/operator/internal/dynamo/lpx/modelexpress.go new file mode 100644 index 000000000000..763f8541572e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/modelexpress.go @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "crypto/tls" + "fmt" + "net/url" + + modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +const modelExpressMaxMessageSize = 100 * 1024 * 1024 + +// NewModelExpressClient validates modelExpressURL and constructs a gRPC-backed client. +func NewModelExpressClient(modelExpressURL string) (modelpb.ModelServiceClient, error) { + parsedModelExpressURL, err := url.Parse(modelExpressURL) + if err != nil { + return nil, fmt.Errorf("parse ModelExpress URL %q: %w", modelExpressURL, err) + } + + target := modelExpressURL + transportCredentials := insecure.NewCredentials() + switch parsedModelExpressURL.Scheme { + case "http", "https": + if parsedModelExpressURL.Host == "" { + return nil, fmt.Errorf("ModelExpress URL %q must include a host", modelExpressURL) + } + if parsedModelExpressURL.Path != "" && parsedModelExpressURL.Path != "/" { + return nil, fmt.Errorf("ModelExpress URL %q must not include a path", modelExpressURL) + } + if parsedModelExpressURL.ForceQuery || parsedModelExpressURL.RawQuery != "" || parsedModelExpressURL.Fragment != "" { + return nil, fmt.Errorf("ModelExpress URL %q must not include query or fragment", modelExpressURL) + } + + target = parsedModelExpressURL.Host + if parsedModelExpressURL.Scheme == "https" { + transportCredentials = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) + } + } + + conn, err := grpc.NewClient( + target, + grpc.WithTransportCredentials(transportCredentials), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(modelExpressMaxMessageSize), + grpc.MaxCallSendMsgSize(modelExpressMaxMessageSize), + ), + ) + if err != nil { + return nil, fmt.Errorf("create ModelExpress gRPC client for %q: %w", modelExpressURL, err) + } + + return modelpb.NewModelServiceClient(conn), nil +} diff --git a/deploy/operator/internal/dynamo/lpx/modelexpress_test.go b/deploy/operator/internal/dynamo/lpx/modelexpress_test.go new file mode 100644 index 000000000000..17d835f69462 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/modelexpress_test.go @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" +) + +func TestNewModelExpressClientAcceptsModelExpressURLs(t *testing.T) { + t.Log("Define accepted ModelExpress endpoint forms") + tests := []string{ + "model-express:8080", + "http://model-express:8080", + "http://model-express:8080/", + "https://model-express.example.com:443", + } + + for _, modelExpressURL := range tests { + t.Run(modelExpressURL, func(t *testing.T) { + t.Logf("Create a ModelExpress client for %q", modelExpressURL) + client, err := NewModelExpressClient(modelExpressURL) + + t.Log("Verify the endpoint produces a usable client") + if err != nil { + t.Fatalf("NewModelExpressClient() error = %v", err) + } + if client == nil { + t.Fatal("NewModelExpressClient() = nil, want client") + } + }) + } +} + +func TestNewModelExpressClientRejectsInvalidModelExpressURL(t *testing.T) { + t.Log("Define malformed or unsupported ModelExpress endpoint forms") + tests := []string{ + "://bad-url", + "http:model-express:8080", + "http://model-express:8080/mx", + "http://model-express:8080?version=1", + "http://model-express:8080#mx", + "https://model-express.example.com:443/mx", + } + + for _, modelExpressURL := range tests { + t.Run(modelExpressURL, func(t *testing.T) { + t.Logf("Attempt to create a ModelExpress client for %q", modelExpressURL) + client, err := NewModelExpressClient(modelExpressURL) + + t.Log("Verify the invalid endpoint is rejected without a client") + if err == nil { + t.Fatal("NewModelExpressClient() error = nil, want error") + } + if client != nil { + t.Fatalf("NewModelExpressClient() client = %#v, want nil", client) + } + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/pod_mutation.go b/deploy/operator/internal/dynamo/lpx/pod_mutation.go new file mode 100644 index 000000000000..583538fef9f9 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/pod_mutation.go @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "slices" + + corev1 "k8s.io/api/core/v1" +) + +func appendVolumeIfMissing(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume { + for _, existing := range volumes { + if existing.Name == volume.Name { + return volumes + } + } + return append(volumes, volume) +} + +func setVolumeByName(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume { + return replaceConflicts(volumes, volume, func(existing corev1.Volume) bool { + return existing.Name == volume.Name + }) +} + +func setVolumeMount(volumeMounts []corev1.VolumeMount, volumeMount corev1.VolumeMount) []corev1.VolumeMount { + return replaceConflicts(volumeMounts, volumeMount, func(existing corev1.VolumeMount) bool { + return existing.MountPath == volumeMount.MountPath + }) +} + +func setContainerByName(containers []corev1.Container, container corev1.Container) []corev1.Container { + return replaceConflicts(containers, container, func(existing corev1.Container) bool { + return existing.Name == container.Name + }) +} + +func replaceConflicts[T any](items []T, replacement T, conflicts func(T) bool) []T { + // Preserve the first conflict's position while removing later conflicts, or append when none exists. + first := slices.IndexFunc(items, conflicts) + if first < 0 { + return append(items, replacement) + } + + out := append(items[:first], replacement) + for _, item := range items[first+1:] { + if !conflicts(item) { + out = append(out, item) + } + } + return out +} diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go new file mode 100644 index 000000000000..102a5fd3b125 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + // SchedulerName is the Kubernetes scheduler used for LPX Agent pods. + SchedulerName = "lpx-scheduler" + + v2LPUResourceName corev1.ResourceName = "lpu.nvidia.com/lpu" + v3LPUResourceName corev1.ResourceName = "nvidia.com/lpu" + + lpuModelNameEnvVar = "LPU_MODEL_NAME" +) + +// configureAgentScheduling consumes a fresh, nonnil Agent PodSpec containing main. +func configureAgentScheduling( + agent *corev1.PodSpec, + targetFamily BuildFamily, +) { + // Preserve Agent placement while replacing authored LPU resources with the selected device. + stripLPUResources(agent) + agent.SchedulerName = SchedulerName + container := findMainContainer(agent.Containers) + // Model projection has already restricted the target family to XT or HX. + name, amount := v2LPUResourceName, resource.MustParse("8") + if targetFamily == BuildFamilyHX { + name, amount = v3LPUResourceName, resource.MustParse("16") + } + if container.Resources.Requests == nil { + container.Resources.Requests = make(corev1.ResourceList) + } + if container.Resources.Limits == nil { + container.Resources.Limits = make(corev1.ResourceList) + } + container.Resources.Requests[name], container.Resources.Limits[name] = amount, amount +} + +// findMainContainer returns the main container, or nil when absent. +func findMainContainer(containers []corev1.Container) *corev1.Container { + for i := range containers { + if containers[i].Name == commonconsts.MainContainerName { + return &containers[i] + } + } + return nil +} + +func stripLPUResources(spec *corev1.PodSpec) { + for index := range spec.InitContainers { + stripLPUResourceRequirements(&spec.InitContainers[index].Resources) + } + for index := range spec.Containers { + stripLPUResourceRequirements(&spec.Containers[index].Resources) + } + for index := range spec.EphemeralContainers { + stripLPUResourceRequirements(&spec.EphemeralContainers[index].Resources) + } + if spec.Resources != nil { + stripLPUResourceList(spec.Resources.Limits) + stripLPUResourceList(spec.Resources.Requests) + } +} + +func stripLPUResourceRequirements(resources *corev1.ResourceRequirements) { + stripLPUResourceList(resources.Limits) + stripLPUResourceList(resources.Requests) +} + +func stripLPUResourceList(resources corev1.ResourceList) { + for name := range resources { + if strings.HasPrefix(string(name), "lpu.nvidia.com/") || name == v3LPUResourceName { + delete(resources, name) + } + } +} diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go new file mode 100644 index 000000000000..543e842ccf2e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -0,0 +1,465 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "slices" + "testing" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestSelectedRolePodSpecsMaterializeFamilyResourceOnlyOnAgentMainContainer(t *testing.T) { + t.Parallel() + + t.Log("Construct a mixed-resource PodSpec spanning every container resource location") + lpuResources := corev1.ResourceList{ + v2LPUResourceName: resource.MustParse("8"), + corev1.ResourceName("lpu.nvidia.com/devices"): resource.MustParse("1"), + v3LPUResourceName: resource.MustParse("16"), + corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), + } + base := corev1.PodSpec{ + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelHostname, Operator: corev1.NodeSelectorOpIn, Values: []string{"lpu-node-a"}, + }}}}, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "init", Resources: corev1.ResourceRequirements{ + Limits: lpuResources.DeepCopy(), Requests: lpuResources.DeepCopy(), + }, + }}, + Containers: []corev1.Container{{ + Name: "main", Image: "runtime", + Resources: corev1.ResourceRequirements{ + Limits: lpuResources.DeepCopy(), Requests: lpuResources.DeepCopy(), + }, + }}, + EphemeralContainers: []corev1.EphemeralContainer{{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: "debug", Resources: corev1.ResourceRequirements{Limits: lpuResources.DeepCopy()}, + }, + }}, + Resources: &corev1.ResourceRequirements{ + Limits: lpuResources.DeepCopy(), Requests: lpuResources.DeepCopy(), + }, + } + + t.Log("Define V2 and V3 family-specific resource expectations") + tests := []struct { + name string + family BuildFamily + expectedResource corev1.ResourceName + expectedQuantity resource.Quantity + unexpectedResource corev1.ResourceName + }{ + { + name: "V2 XT8888", family: BuildFamilyXT, + expectedResource: v2LPUResourceName, expectedQuantity: resource.MustParse("8"), + unexpectedResource: v3LPUResourceName, + }, + { + name: "V3 HX", family: BuildFamilyHX, + expectedResource: v3LPUResourceName, expectedQuantity: resource.MustParse("16"), + unexpectedResource: v2LPUResourceName, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Shape LPU resources independently for the conductor and Agent") + conductor, agent := base.DeepCopy(), base.DeepCopy() + stripLPUResources(conductor) + configureAgentScheduling(agent, test.family) + + t.Log("Verify the input is immutable and affinity follows its owning role") + require.Equal(t, lpuResources, base.Containers[0].Resources.Limits, "input PodSpec must remain unchanged") + require.Equal(t, base.Affinity, agent.Affinity) + + t.Log("Remove generic and wrong-family LPU resources from both roles") + for _, spec := range []*corev1.PodSpec{conductor, agent} { + for _, resources := range lpuResourceLists(spec) { + require.NotContains(t, resources, corev1.ResourceName("lpu.nvidia.com/devices")) + require.NotContains(t, resources, test.unexpectedResource) + require.Contains(t, resources, corev1.ResourceName("nvidia.com/gpu")) + require.Contains(t, resources, corev1.ResourceCPU) + } + } + for _, resources := range lpuResourceLists(conductor) { + require.NotContains(t, resources, test.expectedResource) + } + for index, resources := range lpuResourceLists(agent) { + if index == 2 || index == 3 { + continue + } + require.NotContains(t, resources, test.expectedResource) + } + t.Log("Materialize only the selected family resource on the main Agent container") + require.Equal(t, test.expectedQuantity, agent.Containers[0].Resources.Requests[test.expectedResource]) + require.Equal(t, test.expectedQuantity, agent.Containers[0].Resources.Limits[test.expectedResource]) + }) + } +} + +func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { + lists := []corev1.ResourceList{ + spec.InitContainers[0].Resources.Limits, + spec.InitContainers[0].Resources.Requests, + spec.Containers[0].Resources.Limits, + spec.Containers[0].Resources.Requests, + spec.EphemeralContainers[0].Resources.Limits, + spec.Resources.Limits, + spec.Resources.Requests, + } + return lists +} + +func TestStripNovaOnlyArgs(t *testing.T) { + t.Parallel() + + t.Log("Define mixed Nova-leader and worker argument shapes") + for _, test := range []struct { + name string + args []string + want []string + wantNovaOnly bool + }{ + { + name: "worker args only", + args: []string{"custom-agent-command"}, + want: []string{"custom-agent-command"}, + }, + { + name: "split leader arg", + args: []string{"--instance-model-name", "custom-model-name", "custom-agent-command"}, + want: []string{"custom-agent-command"}, + }, + { + name: "equals leader arg", + args: []string{"--instance-model-name=custom-model-name", "custom-agent-command"}, + want: []string{"custom-agent-command"}, + }, + { + name: "leader args only", + args: []string{"--instance-model-name", "custom-model-name"}, + want: []string{}, wantNovaOnly: true, + }, + { + name: "Nova environment args", + args: []string{"--agent-env-vars", "FIRST=1", "--agent-env-vars=SECOND=2", "custom-agent-command"}, + want: []string{"custom-agent-command"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Strip Nova-only arguments from a copied input") + args := append([]string(nil), test.args...) + got, novaOnly := stripNovaOnlyArgs(args) + + t.Log("Verify worker arguments, classification, and input immutability") + require.Equal(t, test.want, got) + require.Equal(t, test.wantNovaOnly, novaOnly) + require.Equal(t, test.args, args) + }) + } +} + +func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { + t.Log("Start from the supported image-only V3 PodSpec") + base := corev1.PodSpec{ + HostUsers: ptr.To(false), + Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, + Image: "runtime:latest", + Args: []string{"--instance-model-name", "test/model", "--agent-env-vars", "PIPELINE_ENV=kept"}, + VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", ReadOnly: true}}, + Env: []corev1.EnvVar{ + {Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, + {Name: "DYN_KUBE_DISCOVERY_MODE", Value: "container"}, + {Name: "GROQ_V2_RESET_ON_OPEN", Value: "custom"}, + { + Name: "MAIN_CPU", + ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, + Resource: "limits.cpu", + }}, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + LivenessProbe: testExecProbe("check-live"), + StartupProbe: testExecProbe("check-started"), + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"stop-main"}}}, + }, + }}, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{}, + }, + }, + { + Name: "resources", + VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{Items: []corev1.DownwardAPIVolumeFile{{ + Path: "main-memory", + ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, + Resource: "limits.memory", + }, + }}}, + }, + }, + {Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "read-only-tmp"}}}, + }, + } + conductor := *base.DeepCopy() + agent := *base.DeepCopy() + + t.Log("Lower the image into the Nova conductor and privileged SSH worker roles") + require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret")) + require.NoError(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret")) + + t.Log("Verify the conductor starts Nova with immutable placement and source behavior") + require.Len(t, conductor.Containers, 1) + conductorContainer := conductor.Containers[0] + require.Equal(t, "conductor", conductorContainer.Name) + require.Equal(t, []string{"/bin/bash"}, conductorContainer.Command) + require.Equal(t, []string{"-c", lpuNovaEntrypointScript, "--"}, conductorContainer.Args[:3]) + require.Contains(t, conductorContainer.Args, "--instance-model-name") + requireFlagValue(t, conductorContainer.Args, "--allocation", "lpu-wkr-m-0") + require.GreaterOrEqual(t, len(conductorContainer.Args), len(nodeLocalHXAgentEnvironmentArgs)) + staticSuffixIndex := len(conductorContainer.Args) - len(nodeLocalHXAgentEnvironmentArgs) + require.Equal(t, nodeLocalHXAgentEnvironmentArgs, conductorContainer.Args[staticSuffixIndex:]) + pipelineEnvIndex := slices.Index(conductorContainer.Args, "PIPELINE_ENV=kept") + require.GreaterOrEqual(t, pipelineEnvIndex, 0) + require.Less(t, pipelineEnvIndex, staticSuffixIndex) + require.Equal(t, "1", testContainerEnvValue(conductorContainer.Env, "OMPI_ALLOW_RUN_AS_ROOT")) + require.Equal(t, "conductor", testContainerEnvValue(conductorContainer.Env, "CONTAINER_NAME")) + require.Equal(t, "conductor", testContainerEnvSource(conductorContainer.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) + require.Equal(t, "conductor", conductor.Volumes[1].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) + require.Equal(t, base.Containers[0].LivenessProbe, conductorContainer.LivenessProbe) + require.Equal(t, base.Containers[0].StartupProbe, conductorContainer.StartupProbe) + require.Equal(t, base.Containers[0].Lifecycle, conductorContainer.Lifecycle) + require.False(t, *conductor.HostUsers) + require.NotNil(t, testContainerEnvSource(conductorContainer.Env, "POD_IP")) + require.Contains(t, conductorContainer.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) + require.NotNil(t, testVolumeByName(t, conductor.Volumes, "tmp").EmptyDir) + require.Contains(t, testVolumeMountPaths(conductorContainer.VolumeMounts), conductorSSHKeyDir) + require.Len(t, conductor.InitContainers, 1) + require.Equal(t, conductorSSHKeyInitContainerName, conductor.InitContainers[0].Name) + require.Equal(t, int32(0600), *testVolumeByName(t, conductor.Volumes, runtimeSSHVolumeName).Secret.DefaultMode) + require.Equal(t, "ssh-secret", testVolumeByName(t, conductor.Volumes, runtimeSSHVolumeName).Secret.SecretName) + + t.Log("Verify the Agent hosts SSHD with local devices and the source resource profile") + require.Len(t, agent.Containers, 1) + agentContainer := agent.Containers[0] + require.Equal(t, lpuAgentContainerName, agentContainer.Name) + require.Equal(t, []string{"/bin/bash"}, agentContainer.Command) + require.Equal(t, []string{"-c", lpuWorkerRunCommand}, agentContainer.Args) + require.Empty(t, testContainerEnvValue(agentContainer.Env, "LPU_RUN_SYSTEM_INIT")) + require.Equal(t, "cvp", testContainerEnvValue(agentContainer.Env, "LPU_FPGA_RECONFIGURE_METHOD")) + require.Equal(t, "custom", testContainerEnvValue(agentContainer.Env, "GROQ_V2_RESET_ON_OPEN")) + require.Equal(t, "0", testContainerEnvValue(agentContainer.Env, "GROQ_PG613_SKIP_HOST_LINK_PROFILE")) + require.Equal(t, lpuAgentContainerName, testContainerEnvValue(agentContainer.Env, "CONTAINER_NAME")) + require.Equal(t, lpuAgentContainerName, testContainerEnvSource(agentContainer.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) + require.Equal(t, lpuAgentContainerName, agent.Volumes[1].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) + require.Equal(t, &corev1.SecurityContext{ + Privileged: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + RunAsGroup: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), + }, agentContainer.SecurityContext) + require.True(t, agentContainer.TTY) + require.True(t, agentContainer.Stdin) + require.Nil(t, agentContainer.LivenessProbe) + require.Nil(t, agentContainer.StartupProbe) + require.Nil(t, agentContainer.Lifecycle) + require.Nil(t, agent.HostUsers) + require.True(t, agent.HostIPC) + require.True(t, agent.HostNetwork) + require.Equal(t, corev1.DNSClusterFirstWithHostNet, agent.DNSPolicy) + require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/dev") + require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/sys") + require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/ssh-pk") + require.Contains(t, agentContainer.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) + require.NotNil(t, testVolumeByName(t, agent.Volumes, "tmp").EmptyDir) + require.Equal(t, resource.MustParse("2"), agentContainer.Resources.Requests[corev1.ResourceCPU]) + require.Equal(t, resource.MustParse("4Gi"), agentContainer.Resources.Limits[corev1.ResourceMemory]) + require.NotContains(t, testVolumeNames(agent.Volumes), "hugepages") + require.Equal(t, int32(0644), *testVolumeByName(t, agent.Volumes, runtimeSSHVolumeName).Secret.DefaultMode) + require.Len(t, agent.InitContainers, 1) + initContainer := agent.InitContainers[0] + require.Equal(t, lp30InitContainerName, initContainer.Name) + require.Equal(t, "runtime:latest", initContainer.Image) + require.Equal(t, []string{lp30InitPath}, initContainer.Command) + require.True(t, *initContainer.SecurityContext.Privileged) + require.Equal(t, "spec.nodeName", testContainerEnvSource(initContainer.Env, "NODE_NAME").FieldRef.FieldPath) + require.Contains(t, testVolumeMountPaths(initContainer.VolumeMounts), "/dev") + require.Contains(t, testVolumeMountPaths(initContainer.VolumeMounts), "/sys") +} + +func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T) { + t.Log("Define custom entrypoint and probe preservation cases") + customStartup := testExecProbe("custom-startup") + customReadiness := testExecProbe("custom-ready") + tests := []struct { + name string + container corev1.Container + wantCommand []string + wantArgs []string + wantStartup *corev1.Probe + wantReadiness *corev1.Probe + }{ + { + name: "args", + container: corev1.Container{Name: commonconsts.MainContainerName, Args: []string{"custom-agent"}}, + wantCommand: []string{"/bin/bash"}, + wantArgs: []string{"custom-agent"}, + wantStartup: lpuV2StartupProbe(true), + wantReadiness: lpuV2ReadinessProbe(true), + }, + { + name: "command", + container: corev1.Container{Name: commonconsts.MainContainerName, Command: []string{"custom-entrypoint"}}, + wantCommand: []string{"custom-entrypoint"}, + wantStartup: lpuV2StartupProbe(true), + wantReadiness: lpuV2ReadinessProbe(true), + }, + { + name: "user probes", + container: corev1.Container{ + Name: commonconsts.MainContainerName, + Args: []string{"custom-agent"}, + StartupProbe: customStartup, + ReadinessProbe: customReadiness, + }, + wantCommand: []string{"/bin/bash"}, + wantArgs: []string{"custom-agent"}, + wantStartup: customStartup, + wantReadiness: customReadiness, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Configure the direct LPU agent runtime from a user-owned entrypoint") + podSpec := corev1.PodSpec{Containers: []corev1.Container{*test.container.DeepCopy()}} + configureAgentScheduling(&podSpec, BuildFamilyXT) + err := configureDirectHybridAgentRuntime(&podSpec, "graph-lpu", "ssh-secret") + require.NoError(t, err) + + t.Log("Preserve the entrypoint and user probes while supplying defaults when absent") + agent := podSpec.Containers[0] + require.Equal(t, test.wantCommand, agent.Command) + require.Equal(t, test.wantArgs, agent.Args) + require.Equal(t, test.wantStartup, agent.StartupProbe) + require.Equal(t, test.wantReadiness, agent.ReadinessProbe) + }) + } +} + +func TestConfigureNodeLocalLPURuntimeRolesRejectsUnsupportedShape(t *testing.T) { + t.Log("Define unsupported node-local PodSpec shapes") + tests := []struct { + name string + mutate func(*corev1.PodSpec) + errorText string + }{ + { + name: "explicit command", + mutate: func(spec *corev1.PodSpec) { + spec.Containers[0].Command = []string{"/custom"} + }, + errorText: "image-only main container without command", + }, + { + name: "sidecar", + mutate: func(spec *corev1.PodSpec) { + spec.Containers = append(spec.Containers, corev1.Container{Name: "sidecar"}) + }, + errorText: `requires exactly one "main" container`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Reject input outside the supported image-only V3 contract") + base := corev1.PodSpec{Containers: []corev1.Container{{Name: commonconsts.MainContainerName}}} + test.mutate(&base) + conductor := *base.DeepCopy() + agent := *base.DeepCopy() + + require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agent", "ssh-secret"), test.errorText) + require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), test.errorText) + }) + } +} + +func testExecProbe(command string) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{command}}}, + } +} + +func requireFlagValue(t *testing.T, args []string, flag, value string) { + t.Helper() + index := slices.Index(args, flag) + require.GreaterOrEqual(t, index, 0) + require.Less(t, index+1, len(args)) + require.Equal(t, value, args[index+1]) +} + +func testContainerEnvValue(env []corev1.EnvVar, name string) string { + index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) + if index < 0 { + return "" + } + return env[index].Value +} + +func testContainerEnvSource(env []corev1.EnvVar, name string) *corev1.EnvVarSource { + index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) + if index < 0 { + return nil + } + return env[index].ValueFrom +} + +func testVolumeMountPaths(mounts []corev1.VolumeMount) []string { + paths := make([]string, 0, len(mounts)) + for _, mount := range mounts { + paths = append(paths, mount.MountPath) + } + return paths +} + +func testVolumeByName(t *testing.T, volumes []corev1.Volume, name string) corev1.VolumeSource { + t.Helper() + index := slices.IndexFunc(volumes, func(volume corev1.Volume) bool { return volume.Name == name }) + require.GreaterOrEqual(t, index, 0) + return volumes[index].VolumeSource +} + +func testVolumeNames(volumes []corev1.Volume) []string { + names := make([]string, 0, len(volumes)) + for _, volume := range volumes { + names = append(names, volume.Name) + } + return names +} diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go new file mode 100644 index 000000000000..5dd08b05970f --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -0,0 +1,322 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "maps" + "slices" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MaxRenderedPodCliqueSetBytes keeps headroom below the API server's request-size ceiling for +// admission metadata and transport overhead. +const MaxRenderedPodCliqueSetBytes = 1 << 20 + +const ( + // StageLabel records the authored LPX component association. + StageLabel = "lpx.nvidia.com/stage" + // ExecutionRoleLabel distinguishes internal LPU and GPU Pod templates. + ExecutionRoleLabel = "lpx.nvidia.com/execution-role" +) + +// RenderInput contains the source name and fresh stage templates consumed by rendering. +type RenderInput struct { + // DGDName is the source DynamoGraphDeployment name. + DGDName string + // MinAvailable is the minimum number of complete engine replicas in the gang. + MinAvailable *int32 + // Stages contains an independently merged LPU template for every projected stage. + Stages map[string]corev1.PodTemplateSpec + // Conductor supplies a fresh, independently merged template for the shared LPU + // conductor. Nil reuses the serving component's agent template. Hybrid pipelines + // do not emit this conductor. + Conductor *corev1.PodTemplateSpec + // SSHSecretName names the Secret containing runtime SSH credentials. + SSHSecretName string +} + +// RenderSelectedNodeLocal consumes fresh graph and render inputs. pcs, workload, +// and plan must be non-nil. The function mutates pcs and may mutate the +// reference-backed stage and conductor template contents in input; callers must +// pass independently owned values. pcs contains only the optional Cyborg clique +// rendered from the same workload, with no scaling groups. workload and plan are +// read without mutation. +// +//nolint:gocyclo // Rendering is one transactional validation-and-materialization pass. +func RenderSelectedNodeLocal( + pcs *grovev1alpha1.PodCliqueSet, + workload *SelectedWorkload, + plan *MaterializationPlan, + input RenderInput, +) ([]client.Object, error) { + projections := workload.modelProjections + + workloadDigest := workload.Digest().String() + agentTemplateNames := make([]string, 0, len(plan.Agents)) + for _, agent := range plan.Agents { + agentTemplateNames = append(agentTemplateNames, agent.TemplateName) + } + conductorTemplateName := plan.ConductorTemplate + allocation := strings.Join(agentTemplateNames, ",") + if projections[0].pipeline == PipelineSpecDecode { + allocation = strings.Join(agentTemplateNames, ":") + } + namespace := pcs.Namespace + + // The serving component owns conductor metadata and storage independently of model order. + conductorStage := projections[len(projections)-1].stage + conductorTemplate := input.Stages[conductorStage] + if input.Conductor != nil { + conductorTemplate = *input.Conductor + } + modelStorage, err := lpuModelStorageBinding(conductorTemplate.Spec) + if err != nil { + return nil, err + } + modelStorage.volume = *modelStorage.volume.DeepCopy() + modelStorage.mount = *modelStorage.mount.DeepCopy() + configMap, err := renderLPUConfigMap(namespace, input.DGDName, modelStorage.mount.MountPath, projections, plan.Agents) + if err != nil { + return nil, err + } + v2HybridRuntime := projections[0].configuredBuild.Family == BuildFamilyXT && + projections[0].pipeline == PipelineLPX + + // Render the optional Cyborg config and construct final resource order once. + var ( + cyborgConfigMap *corev1.ConfigMap + extraResources []client.Object + ) + if v2HybridRuntime { + cyborgConfigMap, err = renderSelectedCyborgConfigMap(namespace, input.DGDName, plan, modelStorage.mount.MountPath, &projections[0].configuredBuild) + if err != nil { + return nil, err + } + // Preserve the legacy graph order: Cyborg config first, LPU config last. + extraResources = []client.Object{cyborgConfigMap, configMap} + } else { + extraResources = []client.Object{configMap} + } + + configHash := LPUConfigMapHash(configMap) + + // Consume an explicit conductor; separate an inherited one before consuming its Agent. + var conductor *grovev1alpha1.PodCliqueTemplateSpec + if conductorTemplateName != "" { + conductorSpec := conductorTemplate.Spec + if input.Conductor == nil { + conductorSpec = *conductorSpec.DeepCopy() + conductorSpec.Affinity = nil + } + annotations := roleAnnotations(conductorTemplate.Annotations, lpxv1alpha1.PodRoleConductor, workloadDigest) + annotations[commonconsts.AnnotationExtraResourcesHash] = configHash + conductor = &grovev1alpha1.PodCliqueTemplateSpec{ + Name: conductorTemplateName, + Labels: maps.Clone(conductorTemplate.Labels), + Annotations: annotations, + Spec: grovev1alpha1.PodCliqueSpec{ + RoleName: conductorTemplateName, + PodSpec: conductorSpec, + Replicas: 1, + MinAvailable: ptr.To(int32(1)), + StartsAfter: agentTemplateNames, + }, + } + pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, conductor) + } + + // Canonical projections keep each component together; consume its last Agent instance. + var template corev1.PodTemplateSpec + for index, projection := range projections { + stage := projection.stage + if index == 0 || stage != projections[index-1].stage { + template = input.Stages[stage] + storage, err := lpuModelStorageBinding(template.Spec) + if err != nil { + return nil, fmt.Errorf("stage %s: %w", stage, err) + } + if storage.mount.MountPath != modelStorage.mount.MountPath { + return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStorage.mount.MountPath) + } + var conductorSpec *corev1.PodSpec + if stage == conductorStage && conductor != nil { + conductorSpec = &conductor.Spec.PodSpec + } + if err := configureLPURolePods(&template.Spec, conductorSpec, workload, configMap.Name, allocation, input.SSHSecretName); err != nil { + return nil, fmt.Errorf("stage %s: %w", stage, err) + } + } + podSpec := template.Spec + if index+1 < len(projections) && stage == projections[index+1].stage { + podSpec = *podSpec.DeepCopy() + } + + // Resolve per-model placeholders only after separating siblings' backing data. + for containerIndex := range podSpec.Containers { + env := podSpec.Containers[containerIndex].Env + for envIndex := range env { + if env[envIndex].Name == lpuModelNameEnvVar { + env[envIndex] = corev1.EnvVar{Name: lpuModelNameEnvVar, Value: projection.Model()} + } + } + } + annotations := roleAnnotations(template.Annotations, lpxv1alpha1.PodRoleAgent, projection.Digest().String()) + annotations[commonconsts.AnnotationExtraResourcesHash] = configHash + annotations[lpxv1alpha1.PodModelAnnotation] = projection.model + annotations[WorkloadModeAnnotation] = string(projection.schedulerWorkloadMode()) + agent := plan.Agents[index] + replicas := int32(agent.Replicas) + pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, &grovev1alpha1.PodCliqueTemplateSpec{ + Name: agent.TemplateName, + Labels: maps.Clone(template.Labels), + Annotations: annotations, + Spec: grovev1alpha1.PodCliqueSpec{ + RoleName: agent.TemplateName, + PodSpec: podSpec, + Replicas: replicas, + MinAvailable: ptr.To(replicas), + }, + }) + } + + pcs.Annotations = workloadAnnotations(pcs.Annotations, workloadDigest) + // Node-local uses canonical annotation absence for backward compatibility. + delete(pcs.Annotations, ExecutionBackendAnnotation) + var cyborg *grovev1alpha1.PodCliqueTemplateSpec + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name == plan.CyborgTemplate { + cyborg = clique + } + } + selectedTemplateNames := agentTemplateNames + if conductorTemplateName != "" { + selectedTemplateNames = append([]string{conductorTemplateName}, selectedTemplateNames...) + } + + hybrid := projections[0].pipeline == PipelineLPX + if hybrid { + if err := configureHybridCyborg( + cyborg, + projections[0], + workloadDigest, + modelStorage, + agentTemplateNames, + plan, + cyborgConfigMap, + ); err != nil { + return nil, err + } + } + + // The LPX-only PCS owns one scaling group for the complete engine. + members := selectedTemplateNames + if hybrid { + members = append(members, plan.CyborgTemplate) + } + pcs.Spec.Template.PodCliqueScalingGroupConfigs = []grovev1alpha1.PodCliqueScalingGroupConfig{{ + Name: plan.LPXScalingGroupTemplate, + CliqueNames: members, + Annotations: map[string]string{WorkloadDigestAnnotation: workloadDigest}, + Replicas: ptr.To(plan.Replicas), + MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), + }} + + explicit := grovev1alpha1.CliqueStartupTypeExplicit + pcs.Spec.Template.StartupType = &explicit + return extraResources, nil +} + +// configureLPURolePods consumes fresh, independently owned Agent and conductor +// specs. Agent and workload are nonnil; nil conductor means no emitted launcher. +func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, workload *SelectedWorkload, configMapName, allocation, sshSecretName string) error { + if err := withLPUConfigVolume(agentPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { + return err + } + configureAgentScheduling(agentPodSpec, workload.BuildFamily()) + // Placement is already resolved; shape only the actual conductor's LPX-owned fields. + if conductorPodSpec != nil { + conductorPodSpec.SchedulerName = corev1.DefaultSchedulerName + stripLPUResources(conductorPodSpec) + if err := withLPUConfigVolume(conductorPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { + return err + } + } + if workload.Pipeline() == PipelineSpecDecode || workload.Pipeline() == PipelineLPX { + agent := findMainContainer(agentPodSpec.Containers) + setContainerEnv(agent, false, corev1.EnvVar{Name: lpuModelNameEnvVar}) + } + + if workload.Pipeline() == PipelineLPX { + if err := configureDirectHybridAgentRuntime( + agentPodSpec, + configMapName, + sshSecretName, + ); err != nil { + return err + } + } else { + // Both node-local roles use the operator-managed key, including an agent-only draft. + if strings.TrimSpace(sshSecretName) == "" { + return fmt.Errorf("node-local LPU runtime requires an MPI SSH secret name") + } + if conductorPodSpec != nil { + if err := configureNodeLocalConductorRuntime(conductorPodSpec, workload.BuildFamily(), allocation, sshSecretName); err != nil { + return err + } + } + if err := configureNodeLocalAgentRuntime(agentPodSpec, workload.BuildFamily(), workload.Pipeline() == PipelineSingle, sshSecretName); err != nil { + return err + } + } + + ensureLPUNodeTolerations(agentPodSpec) + return nil +} + +func roleAnnotations( + base map[string]string, + role string, + workloadDigest string, +) map[string]string { + annotations := workloadAnnotations(maps.Clone(base), workloadDigest) + // Remove controller-owned role metadata before stamping canonical values. + for _, key := range []string{ + ExecutionBackendAnnotation, + WorkloadModeAnnotation, + lpxv1alpha1.PodModelAnnotation, + lpxv1alpha1.PodPartitionIDAnnotation, + lpxv1alpha1.PodRankInPartitionAnnotation, + } { + delete(annotations, key) + } + annotations[lpxv1alpha1.PodRoleAnnotation] = role + return annotations +} + +func workloadAnnotations(base map[string]string, digest string) map[string]string { + if base == nil { + base = make(map[string]string) + } + base[WorkloadDigestAnnotation] = digest + return base +} + +func appendUnique(existing []string, values ...string) []string { + for _, value := range values { + if !slices.Contains(existing, value) { + existing = append(existing, value) + } + } + return existing +} diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go new file mode 100644 index 000000000000..64546649385e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -0,0 +1,364 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + testRenderComponentName = "lpu" + testAgentTemplateName = "lpu-wkr-m-0" + testTargetStageName = "target" +) + +func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*ModelProjection, input RenderInput) (*grovev1alpha1.PodCliqueSet, error) { + // Attach test projections to their authored stage without inventing templates. + for _, projection := range projections { + projection.stage = testRenderComponentName + } + digest, err := workloadSetDigest(projections) + if err != nil { + return nil, err + } + workload := &SelectedWorkload{ + modelProjections: projections, + digest: digest, + engineReplicas: 1, + } + plan, err := workload.PlanNodeLocalMaterialization(pcs.Name) + if err != nil { + return nil, err + } + if _, err := RenderSelectedNodeLocal(pcs, workload, plan, input); err != nil { + return nil, err + } + return pcs, nil +} + +func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { + t.Parallel() + + t.Log("Cover XT configuration mounts and HX hybrid model-storage paths") + hybrid := newV3CompilerFixture() + hybrid.compilationMode = manifestcapnp.CompilationMode_lpx + tests := []struct { + name string + family lpxv1alpha1.TargetFamily + pipeline Pipeline + snapshot *BuildSnapshot + }{ + { + name: "XT config mount", family: lpxv1alpha1.TargetFamilyXt8888, pipeline: PipelineSingle, + snapshot: acquireTestSnapshot(t, writeV2CompilerFixture(t)), + }, + { + name: "HX hybrid storage", family: lpxv1alpha1.TargetFamilyHx16x8x2x3, pipeline: PipelineLPX, + snapshot: acquireTestSnapshot(t, writeCompilerFixture(t, hybrid)), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Seed conflicting runtime annotations and a nondefault mount") + projection := projectRenderFixture(t, test.family, test.pipeline, test.snapshot) + pcs := renderTestPCS(test.pipeline == PipelineLPX) + pcs.Annotations = map[string]string{ + ExecutionBackendAnnotation: "stale", + } + for _, clique := range pcs.Spec.Template.Cliques { + clique.Annotations = map[string]string{ExecutionBackendAnnotation: "stale"} + } + template := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + "user": "kept", + lpxv1alpha1.PodModelAnnotation: "spoofed", + ExecutionBackendAnnotation: "stale", + }}, + Spec: renderTestPodSpec(), + } + if test.family == lpxv1alpha1.TargetFamilyXt8888 { + template.Spec.Containers[0].VolumeMounts = append(template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) + } else { + template.Spec.Containers[0].VolumeMounts[0].MountPath = "/tmp" + } + + t.Log("Render into the fresh PCS without retaining stale runtime identity") + rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ + DGDName: "test-dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, + SSHSecretName: "ssh-secret", + }) + require.NoError(t, err) + require.Same(t, pcs, rendered) + require.Equal(t, projection.Digest().String(), rendered.Annotations[WorkloadDigestAnnotation]) + require.NotContains(t, rendered.Annotations, ExecutionBackendAnnotation) + for _, clique := range rendered.Spec.Template.Cliques { + require.NotContains(t, clique.Annotations, ExecutionBackendAnnotation) + } + agent := namedClique(t, rendered, testAgentTemplateName) + require.Equal(t, projection.Model(), agent.Annotations[lpxv1alpha1.PodModelAnnotation]) + require.Equal(t, projection.Digest().String(), agent.Annotations[WorkloadDigestAnnotation]) + + t.Log("Resolve runtime mounts using the authored storage path") + if test.family == lpxv1alpha1.TargetFamilyXt8888 { + conductor := namedClique(t, rendered, "lpu-ldr") + require.Equal(t, "kept", conductor.Annotations["user"]) + require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) + } else { + cyborg := namedClique(t, rendered, "lpu-engine-gpu").Spec.PodSpec.Containers[0] + require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/tmp"}) + require.Contains(t, cyborg.Env, corev1.EnvVar{ + Name: "GBUILD_MANIFEST_PATH", Value: "/tmp/model-build/manifest.v2.capnp.bin", + }) + } + }) + } +} + +func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { + t.Parallel() + + t.Log("Project an LPU-only model with nested runtime settings") + snapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) + projection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) + projection.configuredBuild.runtimeSettings["tokenizer_path"] = "tokenizer" + projection.configuredBuild.runtimeSettings["stop_tokens"] = []uint32{1} + projection.configuredBuild.runtimeSettings["swa"] = map[string]any{"chunked": true} + projection.configuredBuild.runtimeSettings["scheduler"] = map[string]any{"max_inflight_tasks": int64(3)} + retainedSetup := map[string]any{"custom": "kept"} + projection.configuredBuild.runtimeSettings["setup"] = retainedSetup + + t.Log("Render and verify isolated model configuration state") + firstConfig, err := lpuModelConfig([]*ModelProjection{projection}, "/models") + require.NoError(t, err) + require.EqualValues(t, 3, firstConfig["scheduler"].(map[string]any)["max_inflight_tasks"]) + require.NotContains(t, firstConfig["iop"], "scheduler") + require.NotContains(t, firstConfig["iop"], "setup") + + t.Log("Mutate one rendered config and verify the next render is isolated") + firstConfig["setup"].(map[string]any)["mutated"] = true + secondConfig, err := lpuModelConfig([]*ModelProjection{projection}, "/models") + require.NoError(t, err) + require.NotContains(t, secondConfig["setup"], "mutated") + require.Equal(t, map[string]any{"custom": "kept"}, retainedSetup) + require.Equal(t, map[string]any{"chunked": true}, projection.configuredBuild.runtimeSettings["swa"]) + + t.Log("Construct a base PodSpec with model binding and custom placement") + modelAnnotationSource := &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['" + lpxv1alpha1.PodModelAnnotation + "']", + }} + conductorPodSpec := renderTestPodSpec() + conductorPodSpec.NodeSelector = map[string]string{"node-pool": "lpu"} + conductorPodSpec.Containers[0].Command = []string{"/bin/bash"} + conductorPodSpec.Containers[0].Args = []string{"-c", "custom-agent"} + conductorPodSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelHostname, Operator: corev1.NodeSelectorOpIn, Values: []string{"lpu-node-a"}, + }}}}, + }, + }} + conductorPodSpec.Containers[0].Env = []corev1.EnvVar{{Name: lpuModelNameEnvVar, ValueFrom: modelAnnotationSource}} + + t.Log("Render conductor and Agent roles from the base PodSpec") + rendered, err := renderSelectedForTest(renderTestPCS(false), []*ModelProjection{projection}, RenderInput{ + DGDName: "test-dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, + SSHSecretName: "ssh-secret", + }) + require.NoError(t, err) + + t.Log("Verify conductor-owned placement and entrypoint behavior") + conductor := namedClique(t, rendered, "lpu-ldr") + require.Nil(t, conductor.Spec.PodSpec.Affinity) + require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) + require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) + require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) + require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) + + t.Log("Verify Agent-owned affinity and logical model binding") + agent := namedClique(t, rendered, testAgentTemplateName) + require.Equal(t, conductorPodSpec.Affinity, agent.Spec.PodSpec.Affinity) + require.Equal(t, conductorPodSpec.NodeSelector, agent.Spec.PodSpec.NodeSelector) + require.Equal(t, []string{"/bin/bash"}, agent.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"-c", "custom-agent"}, agent.Spec.PodSpec.Containers[0].Args) + require.Contains(t, agent.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: lpuModelNameEnvVar, + Value: projection.Model(), + }) +} + +func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { + t.Parallel() + + t.Log("Create distinct draft and target compiler fixtures") + draftFixture := newV3CompilerFixture() + draftFixture.pipelineName = runtimeModelDraft + draftSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, draftFixture)) + targetFixture := newV3CompilerFixture() + targetFixture.pipelineName = runtimeModelTarget + targetSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, targetFixture)) + + t.Log("Keep independent draft and target templates, with the conductor seeded by target") + draft := testLPXComponent(draftStageName, "draft-build", + v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, + ) + draft.Replicas = ptr.To(int32(2)) + draft.LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{ + "max_swa_dkvc_blocks_draft":2, + "setup":{"agent_connect_timeout":"30s","agent_setup_timeout":"180s"} + }`)} + target := testLPXComponent(testTargetStageName, "target-build", + v1beta1.ComponentRoleSpec{Name: "leader"}, + v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, + ) + source := newSelectedTestDGD(t, "specdecode", draft, target) + stages := make(map[string]corev1.PodTemplateSpec) + for _, component := range source.Spec.Components { + template := component.ComponentRole("worker").PodTemplate + template.Labels = map[string]string{"owner": component.ComponentName} + template.Annotations = map[string]string{"owner": component.ComponentName} + template.Spec.Containers[0].Image = component.ComponentName + "-runtime" + stages[component.ComponentName] = *template.DeepCopy() + } + before := source.DeepCopy() + + t.Log("Resolve and render the authored speculative workload") + selected, err := ResolveSelectedWorkload(t.Context(), source, staticBuildSnapshotSource{ + "draft-build": draftSnapshot, "target-build": targetSnapshot, + }) + require.NoError(t, err) + pcs := renderTestPCS(false) + plan, err := selected.PlanNodeLocalMaterialization(pcs.Name) + require.NoError(t, err) + extraResources, err := RenderSelectedNodeLocal(pcs, selected, plan, RenderInput{ + DGDName: source.Name, Stages: stages, SSHSecretName: "ssh-secret", + }) + require.NoError(t, err) + + t.Log("Retain each authored component's image and metadata on its Agent cliques") + for index, stage := range []string{draftStageName, draftStageName, testTargetStageName} { + agent := namedClique(t, pcs, plan.Agents[index].TemplateName) + require.Equal(t, stage+"-runtime", agent.Spec.PodSpec.Containers[0].Image) + require.Equal(t, stage, agent.Labels["owner"]) + require.Equal(t, stage, agent.Annotations["owner"]) + } + + t.Log("Mutate Agent 0 and verify sibling and conductor PodSpecs do not alias it") + firstAgent := namedClique(t, pcs, plan.Agents[0].TemplateName) + secondAgent := namedClique(t, pcs, plan.Agents[1].TemplateName) + conductor := namedClique(t, pcs, plan.ConductorTemplate) + require.Equal(t, "target-runtime", conductor.Spec.PodSpec.Containers[0].Image) + require.Equal(t, testTargetStageName, conductor.Labels["owner"]) + require.Equal(t, testTargetStageName, conductor.Annotations["owner"]) + secondAgentBefore := secondAgent.Spec.PodSpec.DeepCopy() + conductorBefore := conductor.Spec.PodSpec.DeepCopy() + require.NotEmpty(t, firstAgent.Spec.PodSpec.Containers[0].Env) + require.NotEmpty(t, firstAgent.Spec.PodSpec.Tolerations) + firstAgent.Spec.PodSpec.Containers[0].Env[0].Name = "MUTATED_AGENT_ENV" + firstAgent.Spec.PodSpec.Tolerations[0].Key = "mutated-agent-toleration" + require.Equal(t, secondAgentBefore, &secondAgent.Spec.PodSpec) + require.Equal(t, conductorBefore, &conductor.Spec.PodSpec) + + t.Log("Project draft settings once into the shared SpecDecode configuration") + configMap, ok := extraResources[0].(*corev1.ConfigMap) + require.True(t, ok) + require.Contains(t, configMap.Data["model_config.toml"], "SpecDecode") + require.Contains(t, configMap.Data["model_config.toml"], "[draft") + require.Contains(t, configMap.Data["model_config.toml"], "[target") + require.Contains(t, configMap.Data["model_config.toml"], "num_drafts = 2") + require.Contains(t, configMap.Data["model_config.toml"], "max_swa_dkvc_blocks_draft = 2") + require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_connect_timeout = '30s'")) + require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_setup_timeout = '180s'")) + require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "[draft.iop]")) + require.Equal(t, "draft0\ndraft1\ntarget", configMap.Data["partition_models"]) + require.Equal(t, before, source) +} + +func projectRenderFixture(t *testing.T, family lpxv1alpha1.TargetFamily, pipeline Pipeline, snapshot *BuildSnapshot) *ModelProjection { + t.Helper() + intent := ModelProjectionInput{ + Pipeline: pipeline, + Models: []string{"default"}, RuntimeBuildRef: "model-build", BuildSnapshot: normalizeTestSnapshot(t, snapshot), + } + if family == lpxv1alpha1.TargetFamilyXt8888 { + intent.ModelSettings = json.RawMessage(`{"prop_sync":true}`) + } + projectionBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + return projectionBatch[0] +} + +func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { + pcs := &grovev1alpha1.PodCliqueSet{} + pcs.Name = "test-pcs" + pcs.Namespace = "test" + if !hybrid { + return pcs + } + one := int32(1) + pcs.Spec.Template.Cliques = []*grovev1alpha1.PodCliqueTemplateSpec{{ + Name: "lpu-engine-gpu", + Labels: map[string]string{"kai.scheduler/queue": "legacy-queue"}, + Spec: grovev1alpha1.PodCliqueSpec{ + RoleName: "lpu-engine-gpu", Replicas: 1, MinAvailable: &one, + PodSpec: corev1.PodSpec{ + SchedulerName: corev1.DefaultSchedulerName, + Containers: []corev1.Container{{Name: "main", Image: "cyborg"}}, + ResourceClaims: []corev1.PodResourceClaim{{ + Name: "candidate", + ResourceClaimTemplateName: ptr.To("candidate"), + }}, + }, + }, + }} + return pcs +} + +func renderTestPodSpec() corev1.PodSpec { + return corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "model-storage", + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "model-storage", + }}, + }}, + Containers: []corev1.Container{{ + Name: "main", Image: "runtime", + VolumeMounts: []corev1.VolumeMount{{Name: "model-storage", MountPath: "/models"}}, + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{ + corev1.ResourceName("lpu.nvidia.com/devices"): resource.MustParse("1"), + }}, + }}, + } +} + +func namedClique(t *testing.T, pcs *grovev1alpha1.PodCliqueSet, name string) *grovev1alpha1.PodCliqueTemplateSpec { + t.Helper() + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name == name { + return clique + } + } + t.Fatalf("clique %q not found", name) + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go new file mode 100644 index 000000000000..ad8ebafa366b --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -0,0 +1,190 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" +) + +// ErrUnsupportedRuntime identifies input that the supported LPX runtimes cannot materialize. +var ErrUnsupportedRuntime = errors.New("unsupported LPX runtime") + +// ErrBuildSnapshotAcquisition identifies retryable failures while reading the +// immutable build snapshot from its backing store. +var ErrBuildSnapshotAcquisition = errors.New("acquiring immutable LPX build snapshot") + +// BuildSnapshotSource acquires immutable build inputs by build reference. +type BuildSnapshotSource interface { + // AcquireBuildSnapshot returns a complete snapshot for id or an error. + AcquireBuildSnapshot(context.Context, string) (*BuildSnapshot, error) +} + +// ResolveSelectedWorkload validates and projects the selected LPX workload in dgd. +// dgd and source must be non-nil. The function reads but does not mutate dgd. +func ResolveSelectedWorkload( + ctx context.Context, + dgd *dynamov1beta1.DynamoGraphDeployment, + source BuildSnapshotSource, +) (*SelectedWorkload, error) { + components, allErrs := validateSelectedIntent(dgd) + if err := allErrs.ToAggregate(); err != nil { + return nil, err + } + component := components[len(components)-1] + projections := make([]*ModelProjection, 0, len(components)) + var snapshot NormalizedBuildSnapshot + var pipeline Pipeline + for index, stage := range components { + model := stage.LPX + configuredModel := stage.ComponentName + var modelSettings json.RawMessage + if model.Settings != nil { + modelSettings = model.Settings.Raw + } + // A validated selection has at most two models, so only the preceding projection can share its build. + if len(projections) == 0 || projections[len(projections)-1].runtimeBuildRef != model.BuildID { + rawSnapshot, acquireErr := source.AcquireBuildSnapshot(ctx, model.BuildID) + if acquireErr != nil { + return nil, fmt.Errorf( + "%w for model %q build %q: %w", + ErrBuildSnapshotAcquisition, + configuredModel, + model.BuildID, + acquireErr, + ) + } + normalizedSnapshot, normalizeErr := normalizeBuildSnapshot(rawSnapshot) + if normalizeErr != nil { + return nil, fmt.Errorf( + "normalizing immutable LPX build snapshot for model %q build %q: %w", + configuredModel, + model.BuildID, + normalizeErr, + ) + } + snapshot = normalizedSnapshot + } + + // Validated model cardinality fixes one runtime shape for the selected workload. + if pipeline == "" { + pipeline = PipelineSingle + if len(components) == 2 { + pipeline = PipelineSpecDecode + } else if snapshot.build.CompilationMode == BuildCompilationModeHybrid { + pipeline = PipelineLPX + } + } + + if err := validateSelectedConductor(dgd, stage, pipeline, snapshot.build.CompilationMode); err != nil { + return nil, err + } + + // Project the component once into the resolver-owned aggregate destination. + modelNames := expandedSelectedModelNames(index, len(components), int(ptr.Deref(stage.Replicas, 1))) + intent := ModelProjectionInput{ + Pipeline: pipeline, + Models: modelNames, + RuntimeBuildRef: model.BuildID, + BuildSnapshot: snapshot, + ModelSettings: modelSettings, + } + projected, err := appendModelProjections(projections, intent) + if err != nil { + return nil, fmt.Errorf("project LPX model %q from build %q: %w", modelNames[0], model.BuildID, err) + } + + // Component geometry fixes the Agent count independently of draft fanout. + componentProjections := projected[len(projections):] + if count := stage.ComponentRole(dynamov1beta1.ComponentRoleWorker).Replicas; count != nil && int(*count) != componentProjections[0].agentReplicas { + return nil, fmt.Errorf("component %q agent replicas %d must match the compiled count %d", configuredModel, *count, componentProjections[0].agentReplicas) + } + + // Retain the authored stage association before acquiring the next component. + for _, projection := range componentProjections { + projection.stage = stage.ComponentName + } + projections = projected + } + replicas := ptr.Deref(component.Replicas, 1) + if pipeline != PipelineLPX && replicas != 1 { + return nil, fmt.Errorf("%w: fixed engine replicas above one currently require a hybrid engine", ErrUnsupportedRuntime) + } + + // Canonical roles expand into default or draft0..draft7 followed by target. + digest, err := workloadSetDigest(projections) + if err != nil { + return nil, err + } + return &SelectedWorkload{ + modelProjections: projections, + digest: digest, + engineReplicas: replicas, + }, nil +} + +// validateSelectedConductor checks role requirements that depend on the immutable build. +func validateSelectedConductor( + dgd *dynamov1beta1.DynamoGraphDeployment, + component *dynamov1beta1.DynamoComponentDeploymentSharedSpec, + pipeline Pipeline, + compilationMode BuildCompilationMode, +) error { + conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLeader) + // Hybrid execution is selected by immutable build metadata, not template presence. + if compilationMode == BuildCompilationModeHybrid { + if pipeline == PipelineSpecDecode { + return fmt.Errorf("%w: the shared speculative runtime requires LPU-only builds", ErrUnsupportedRuntime) + } + template := component.ComponentRole(dynamov1beta1.ComponentRoleWorker).PodTemplate + if conductor != nil && conductor.PodTemplate != nil { + template = conductor.PodTemplate + } + container := findMainContainer(template.Spec.Containers) + count, err := EffectiveCyborgGPUCount(container.Resources) + if err != nil { + return fmt.Errorf("component %q conductor resources: %w", component.ComponentName, err) + } + if len(template.Spec.ResourceClaims) == 0 && count <= 0 { + return fmt.Errorf("component %q conductor requires resourceClaims or a positive %s request", component.ComponentName, commonconsts.KubeResourceGPUNvidia) + } + return nil + } + if conductor == nil { + return nil + } + if ptr.Deref(conductor.Replicas, 1) != 1 { + return fmt.Errorf("%w: component %q conductor replicas must be one for LPU-only execution", ErrUnsupportedRuntime, component.ComponentName) + } + if conductor.PodTemplate == nil { + return nil + } + // Explicit LPU conductors receive the same appended launch arguments as inherited templates. + componentIndex := slices.IndexFunc(dgd.Spec.Components, func(candidate dynamov1beta1.DynamoComponentDeploymentSharedSpec) bool { + return candidate.ComponentName == component.ComponentName + }) + roleIndex := slices.IndexFunc(component.Roles, func(role dynamov1beta1.ComponentRoleSpec) bool { return role.Name == dynamov1beta1.ComponentRoleLeader }) + for containerIndex := range conductor.PodTemplate.Spec.Containers { + container := &conductor.PodTemplate.Spec.Containers[containerIndex] + if container.Name != commonconsts.MainContainerName { + continue + } + containerPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec", "containers").Index(containerIndex) + if err := validateAllocationInjectionTargetFields(container, containerPath).ToAggregate(); err != nil { + return err + } + } + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go new file mode 100644 index 000000000000..132137c8bf98 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -0,0 +1,320 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "slices" + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { + t.Log("Create one LPX component beside an unrelated conventional decode") + dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")})) + dgd.Annotations = nil + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "ordinary-decode", ComponentType: v1beta1.ComponentTypeDecode, + Replicas: ptr.To(int32(0)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "ordinary"}}}}, + }) + + t.Log("Resolve an LPU-only engine without selecting the conventional decode") + hxSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) + hx, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": hxSnapshot}) + require.NoError(t, err) + require.Equal(t, PipelineSingle, hx.Pipeline()) + require.Equal(t, BuildFamilyHX, hx.BuildFamily()) + require.Equal(t, lpxv1alpha1.WorkloadModeV3HxLPUOnly, hx.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Empty(t, hx.CyborgTemplateName()) + require.Equal(t, "LPX", hx.LPXComponentName()) + plan, err := hx.PlanNodeLocalMaterialization("test-pcs") + require.NoError(t, err) + require.Equal(t, "lpx", plan.LPXScalingGroupTemplate) + require.Equal(t, "lpx-ldr", plan.ConductorTemplate) + require.NotEmpty(t, plan.ConductorClique) + + t.Log("Require hybrid conductor resources in either the explicit or inherited template") + fixture := newV2CompilerFixture() + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + fixture.selectedPropSyncChains = nil + fixture.partitions = append(fixture.partitions, testV3CapnpPartition{id: 11, deviceType: manifestcapnp.DeviceType_cuda}) + snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) + source := staticBuildSnapshotSource{"build": snapshot} + _, err = ResolveSelectedWorkload(t.Context(), dgd, source) + require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") + dgd.Spec.Components[0].Roles = append(dgd.Spec.Components[0].Roles, v1beta1.ComponentRoleSpec{Name: "leader"}) + conductor := dgd.Spec.Components[0].ComponentRole("leader") + conductor.PodTemplate = testLPXPodTemplate("cyborg-runtime") + _, err = ResolveSelectedWorkload(t.Context(), dgd, source) + require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") + conductor.PodTemplate.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceName(commonconsts.KubeResourceGPUNvidia): resource.MustParse("1"), + } + dgd.Spec.Components[0].Replicas = ptr.To(int32(2)) + + t.Log("Project two complete hybrid replicas") + xt, err := ResolveSelectedWorkload(t.Context(), dgd, source) + require.NoError(t, err) + require.Equal(t, PipelineLPX, xt.Pipeline()) + require.Equal(t, BuildFamilyXT, xt.BuildFamily()) + require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, xt.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Equal(t, "lpx-engine-gpu", xt.CyborgTemplateName()) + require.Len(t, xt.modelProjections[0].configuredBuild.Partitions, 2) + plan, err = xt.PlanNodeLocalMaterialization("test-pcs") + require.NoError(t, err) + require.EqualValues(t, 2, plan.Replicas) + replica := plan.ForReplica(1) + require.NotEqual(t, plan.Agents[0].CliqueName, replica.Agents[0].CliqueName) + + t.Log("A scheduling deadline and annotation do not change hybrid launch") + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} + dgd.Annotations = map[string]string{commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend} + scheduled, err := ResolveSelectedWorkload(t.Context(), dgd, source) + require.NoError(t, err) + require.Equal(t, xt, scheduled) + scheduledPlan, err := scheduled.PlanNodeLocalMaterialization("test-pcs") + require.NoError(t, err) + require.Equal(t, plan, scheduledPlan) + require.Empty(t, scheduledPlan.ConductorTemplate) + + t.Log("Use the same agent template when the standalone conductor role is omitted") + agent := *dgd.Spec.Components[0].ComponentRole("worker") + agent.PodTemplate = conductor.PodTemplate.DeepCopy() + dgd.Spec.Components[0].Roles = []v1beta1.ComponentRoleSpec{agent} + before := dgd.DeepCopy() + implicit, err := ResolveSelectedWorkload(t.Context(), dgd, source) + require.NoError(t, err) + require.Equal(t, xt, implicit) + require.Same(t, &dgd.Spec.Components[0], ServingComponent(dgd)) + require.Equal(t, before, dgd) +} + +func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { + t.Log("Define revision-specific SpecDecode compiler snapshots") + tests := []struct { + name string + family BuildFamily + wantMode lpxv1alpha1.WorkloadMode + }{ + { + name: "v2", family: BuildFamilyXT, + wantMode: lpxv1alpha1.WorkloadModeV2LPUOnly, + }, + { + name: "v3", family: BuildFamilyHX, + wantMode: lpxv1alpha1.WorkloadModeV3HxLPUOnly, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Acquire the selected family fixtures: shared XT build or distinct HX snapshots") + var draftSnapshot, targetSnapshot *BuildSnapshot + var settings *apiextensionsv1.JSON + if test.family == BuildFamilyHX { + draftSnapshot = acquireTestSnapshot(t, writeV3CompilerFixture(t)) + targetSnapshot = acquireTestSnapshot(t, writeV3CompilerFixture(t)) + } else { + draftSnapshot = acquireTestSnapshot(t, writeV2CompilerFixture(t)) + targetSnapshot = draftSnapshot + settings = &apiextensionsv1.JSON{Raw: []byte(`{ "prop_sync": true }`)} + } + + t.Log("Build a selected SpecDecode DGD for the fixture's manifest generation") + dgd := newSelectedTestDGD(t, "specdecode", + testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: "leader"}, v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")}), + testLPXComponent("small", "draft-build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")}), + ) + dgd.Spec.Components[0].LPX.Settings = settings + dgd.Spec.Components[1].LPX.Settings = settings.DeepCopy() + dgd.Spec.Components[1].Replicas = ptr.To(int32(2)) + compiledAgentCount := int32(1) + if test.family == BuildFamilyXT { + compiledAgentCount = 4 + } + dgd.Spec.Components[1].ComponentRole("worker").Replicas = ptr.To(compiledAgentCount) + source := staticBuildSnapshotSource{ + "draft-build": draftSnapshot, + "target-build": targetSnapshot, + } + + t.Log("Project the selected SpecDecode workload") + before := dgd.DeepCopy() + selected, err := ResolveSelectedWorkload(t.Context(), dgd, source) + + t.Log("Project the selected family, workload mode, and pipeline") + require.NoError(t, err) + require.Equal(t, before, dgd, "canonical ordering must not rewrite the authored target-first list") + require.Equal(t, test.family, selected.BuildFamily()) + require.Equal(t, test.wantMode, selected.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Equal(t, PipelineSpecDecode, selected.Pipeline()) + require.Equal(t, "lpx", selected.LPXComponentName()) + + t.Log("Expand draft fanout while preserving model and template identity order") + projections := selected.ModelProjections() + require.Len(t, projections, 3) + require.Equal( + t, + []string{"draft0", "draft1", "target"}, + []string{projections[0].Model(), projections[1].Model(), projections[2].Model()}, + ) + plan, err := selected.PlanNodeLocalMaterialization("test-pcs") + require.NoError(t, err) + require.Equal(t, "small", projections[0].stage) + require.Equal(t, "lpx", projections[2].stage) + require.Equal( + t, + []string{"lpx-wkr-m-0", "lpx-wkr-m-1", "lpx-wkr-m-2"}, + []string{ + plan.Agents[0].TemplateName, + plan.Agents[1].TemplateName, + plan.Agents[2].TemplateName, + }, + "template identities follow canonical stage and draft-instance order", + ) + + t.Log("Reordering authored components does not change build identities or resources") + reordered := dgd.DeepCopy() + slices.Reverse(reordered.Spec.Components) + reselected, err := ResolveSelectedWorkload(t.Context(), reordered, source) + require.NoError(t, err) + require.Equal(t, selected.Digest(), reselected.Digest()) + reorderedPlan, err := reselected.PlanNodeLocalMaterialization("test-pcs") + require.NoError(t, err) + require.Equal(t, plan, reorderedPlan) + + t.Log("Agent replica assertions count one compiled model instance, not draft fanout") + invalidCount := dgd.DeepCopy() + invalidCount.Spec.Components[1].ComponentRole("worker").Replicas = ptr.To(compiledAgentCount * 2) + _, err = ResolveSelectedWorkload(t.Context(), invalidCount, source) + require.ErrorContains(t, err, "must match the compiled count") + + t.Log("Derive an aggregate digest and reject mixed-family aggregation") + require.NotEqual(t, WorkloadDigest{}, selected.Digest()) + require.NotEqual(t, projections[0].Digest(), selected.Digest()) + mixedFamily := *projections[2] + mixedFamily.configuredBuild.Family = BuildFamily("other") + _, err = workloadSetDigest([]*ModelProjection{projections[0], &mixedFamily}) + require.ErrorContains(t, err, "mixed target families") + + t.Log("Preserve family-specific runtime settings in every logical projection") + for _, projection := range projections { + require.EqualValues(t, compiledAgentCount, projection.agentReplicas) + if test.family == BuildFamilyHX { + require.EqualValues(t, 8192, projection.configuredBuild.runtimeSettings["sequence_length"]) + } + } + + for _, expansion := range []struct { + name string + count int32 + models []string + }{ + {name: "default", count: 1, models: []string{"draft0", "target"}}, + { + name: "maximum", + count: 8, + models: []string{"draft0", "draft1", "draft2", "draft3", "draft4", "draft5", "draft6", "draft7", "target"}, + }, + } { + t.Run(expansion.name, func(t *testing.T) { + t.Logf("Project SpecDecode draft fanout %d", expansion.count) + draft := &dgd.Spec.Components[1] + draft.Replicas = nil + if expansion.count > 1 { + draft.Replicas = ptr.To(expansion.count) + } + expanded, err := ResolveSelectedWorkload(t.Context(), dgd, source) + require.NoError(t, err) + models := make([]string, 0, len(expansion.models)) + for _, projection := range expanded.ModelProjections() { + models = append(models, projection.Model()) + } + + t.Log("Preserve the expected logical model ordering") + require.Equal(t, expansion.models, models) + }) + } + + if test.family == BuildFamilyHX { + t.Log("Project separate draft and target roles from the same immutable HX build") + draft := &dgd.Spec.Components[1] + draft.Replicas = nil + draft.LPX.BuildID = "target-build" + shared, err := ResolveSelectedWorkload(t.Context(), dgd, source) + require.NoError(t, err) + projections := shared.ModelProjections() + require.Len(t, projections, 2) + require.Equal(t, []string{"draft0", "target"}, []string{projections[0].Model(), projections[1].Model()}) + require.Equal(t, "target-build", projections[0].runtimeBuildRef) + require.Equal(t, "target-build", projections[1].runtimeBuildRef) + require.NotEqual(t, projections[0].Digest(), projections[1].Digest()) + } + }) + } +} + +func TestResolveConductorLaunchErrorUsesAuthoredRolePath(t *testing.T) { + t.Log("Place the target first and author a conductor argument owned by the operator") + conductor := testLPXPodTemplate("conductor-runtime") + conductor.Spec.Containers[0].Args = []string{"--allocation=forged"} + dgd := newSelectedTestDGD(t, "specdecode", + testLPXComponent("large", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("target-runtime")}, v1beta1.ComponentRoleSpec{Name: "leader", PodTemplate: conductor}), + testLPXComponent("small", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("draft-runtime")}), + ) + snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) + before := dgd.DeepCopy() + + t.Log("Report the authored target and conductor indices after canonical build selection") + _, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": snapshot}) + require.ErrorContains(t, err, "spec.components[0].roles[1].podTemplate.spec.containers[0].args") + require.ErrorContains(t, err, "must not set --allocation") + require.Equal(t, before, dgd) +} + +func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *testing.T) { + t.Log("Author independent launch and placement errors on both roles") + agent, conductor := testLPXPodTemplate("agent"), testLPXPodTemplate("conductor") + agent.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} + agent.Spec.Containers[0].Args = []string{"--allocation=forged", "--"} + agent.Spec.Hostname = "custom-host" + conductor.Spec.NodeName = "chosen-node" + conductor.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule, + }} + dgd := newSelectedTestDGD(t, "selected", testLPXComponent("lpx", "build", + v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: agent}, + v1beta1.ComponentRoleSpec{Name: "leader", PodTemplate: conductor}, + )) + + t.Log("Aggregate every actionable error at its authored path before acquiring a build") + _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) + for _, message := range []string{ + "spec.components[0].roles[0].podTemplate.spec.containers[0].args: Forbidden: selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list", + "spec.components[0].roles[0].podTemplate.spec.containers[0].args: Forbidden: selected LPX main container must not terminate arguments before Dynamo appends --allocation", + "spec.components[0].roles[0].podTemplate.spec.containers[0].command: Forbidden: selected LPX main container cannot use a shell because Dynamo appends --allocation", + "spec.components[0].roles[0].podTemplate.spec.hostname: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[1].podTemplate.spec.nodeName: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[1].podTemplate.spec.topologySpreadConstraints: Forbidden: LPX owns role placement", + } { + require.ErrorContains(t, err, message) + } + + t.Log("Reject missing main containers for both roles before acquiring a build") + agent.Spec.Containers, conductor.Spec.Containers = nil, nil + _, err = ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) + require.ErrorContains(t, err, `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX worker component requires a "main" runtime container`) + require.ErrorContains(t, err, `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX leader component requires a "main" runtime container`) +} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go new file mode 100644 index 000000000000..6044e9e76ddd --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -0,0 +1,446 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "slices" + "strings" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +// Preserve rendered workload identities and image-owned executable paths. +const ( + runtimeTemporaryStorageVolumeName = "tmp" + runtimeTemporaryStorageMountPath = "/tmp" + runtimeSSHVolumeName = "agent-v2-ssh" + lp30InitContainerName = "prepare-lp30" + lp30InitPath = "/bin/lp30-agent-v2-init" +) + +var lp30WorkerEnvironment = []corev1.EnvVar{ + {Name: "BMC_SSH_ENABLED", Value: "false"}, + {Name: "GROQ_HAL_V2_BACKGROUND_METRICS", Value: "false"}, + {Name: "GROQ_PG613_PORT_CONFIG", Value: "EB3"}, + {Name: "GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2", Value: "false"}, + {Name: "GROQ_PG613_SKIP_HOST_LINK_PROFILE", Value: "0"}, + {Name: "GROQ_V2_FPGA_BUS_ADDRESS_ID_0", Value: "0000:9a:00.0"}, + {Name: "GROQ_V2_FPGA_BUS_ADDRESS_ID_1", Value: "0000:7e:00.0"}, + {Name: "GROQ_V2_HEALTH_CHECK_ON_OPEN", Value: "0"}, + {Name: "GROQ_V2_RESET_ON_OPEN", Value: "0"}, + {Name: "LPU_FPGA_AUTOMATIC_JTAG_RECOVERY", Value: "false"}, + {Name: "LPU_FPGA_RECONFIGURE_METHOD", Value: "cvp"}, + {Name: "REDFISH_BMC_ENABLED", Value: "false"}, +} + +// nodeLocalHXAgentEnvironmentArgs is the immutable Nova environment suffix for HX. +var nodeLocalHXAgentEnvironmentArgs = []string{ + "--agent-env-vars", "BMC_SSH_ENABLED=false", + "--agent-env-vars", "GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0", + "--agent-env-vars", "GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false", + "--agent-env-vars", "GROQ_HAL_V2_BACKGROUND_METRICS=false", + "--agent-env-vars", "GROQ_PG613_PORT_CONFIG=EB3", + "--agent-env-vars", "GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false", + "--agent-env-vars", "GROQ_PG613_SKIP_HOST_LINK_PROFILE=1", + "--agent-env-vars", "GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0", + "--agent-env-vars", "GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0", + "--agent-env-vars", "GROQ_V2_HEALTH_CHECK_ON_OPEN=0", + "--agent-env-vars", "GROQ_V2_RESET_ON_OPEN=0", + "--agent-env-vars", "LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false", + "--agent-env-vars", "LPU_FPGA_RECONFIGURE_METHOD=cvp", + "--agent-env-vars", "LPU_PARTITION_RUNNER_DATAPLANE=pcie", + "--agent-env-vars", "LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000", + "--agent-env-vars", "LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10", + "--agent-env-vars", "LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true", + "--agent-env-vars", "LPU_VSAP_MAX_TENSORS_PER_CHUNK=4", + "--agent-env-vars", "REDFISH_BMC_ENABLED=false", +} + +// stripNovaOnlyArgs removes conductor-owned Nova CLI flags and reports whether +// every input argument was a well-formed conductor-owned flag. +func stripNovaOnlyArgs(args []string) ([]string, bool) { + if len(args) == 0 { + return args, true + } + + // Parse both accepted flag forms once while retaining worker arguments in order. + const modelNameFlag = "--instance-model-name" + const agentEnvFlag = "--agent-env-vars" + out := make([]string, 0, len(args)) + novaOnly := true + seenModelName := false + for i := 0; i < len(args); i++ { + argument := args[i] + switch argument { + case modelNameFlag, agentEnvFlag: + validValue := i+1 < len(args) && strings.TrimSpace(args[i+1]) != "" + if argument == modelNameFlag { + validValue = validValue && !seenModelName + seenModelName = true + } + novaOnly = novaOnly && validValue + if i+1 < len(args) { + i++ + } + default: + flag, value, hasValue := strings.Cut(argument, "=") + switch flag { + case modelNameFlag: + novaOnly = novaOnly && !seenModelName && hasValue && strings.TrimSpace(value) != "" + seenModelName = true + case agentEnvFlag: + novaOnly = novaOnly && hasValue && strings.TrimSpace(value) != "" + default: + novaOnly = false + out = append(out, argument) + } + } + } + return out, novaOnly +} + +// configureDirectHybridAgentRuntime lowers an LPX-scheduled PodSpec into the +// direct agent runtime used by conductorless hybrid workloads. +func configureDirectHybridAgentRuntime( + agentPodSpec *corev1.PodSpec, + lpuConfigMapName, sshSecretName string, +) error { + if strings.TrimSpace(sshSecretName) == "" { + return fmt.Errorf("direct hybrid agent runtime requires an MPI SSH secret name") + } + + agent := findMainContainer(agentPodSpec.Containers) + // Drop Nova-only flags before applying the direct Agent identity and worker defaults. + agent.Args, _ = stripNovaOnlyArgs(agent.Args) + hasCustomCommand := len(agent.Command) != 0 || len(agent.Args) != 0 + agent.Name = lpuAgentContainerName + applyLPUWorkerContainerBase(agent, len(agent.Command) != 0) + if len(agent.Args) == 0 && !hasCustomCommand { + agent.Args = []string{"-c", lpuPartitionRunCommand} + } + + if !hasCustomCommand || agent.StartupProbe == nil { + agent.StartupProbe = lpuV2StartupProbe(hasCustomCommand) + } + if !hasCustomCommand || agent.ReadinessProbe == nil { + agent.ReadinessProbe = lpuV2ReadinessProbe(hasCustomCommand) + } + + agent.Env = append(agent.Env, + corev1.EnvVar{Name: "GLUE_RDMA_PATH_MTU", Value: "1024"}, + corev1.EnvVar{Name: "NIC_NAME", Value: "mlx5_0"}, + corev1.EnvVar{Name: "UCX_TLS", Value: "tcp"}, + corev1.EnvVar{Name: "RDMA_PORT", Value: fmt.Sprintf("%d", lpuRDMAPort)}, + corev1.EnvVar{Name: "READINESS_PORT", Value: fmt.Sprintf("%d", LPUReadinessPort)}, + ) + + addLPUHostDeviceVolumeMounts(agent) + agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount(sshSecretName)) + retargetMainContainerReferences(agentPodSpec, agent) + + agent.Env = append(agent.Env, + corev1.EnvVar{ + Name: "TOPOLOGIES", + ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: lpuConfigMapName}, + Key: "topologies", + }}, + }, + corev1.EnvVar{ + Name: "GAS_DIR", + ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: lpuConfigMapName}, + Key: "gas_dir", + }}, + }, + ) + setNodeLocalPodIPEnv(agent, true) + addRuntimeTemporaryStorage(agentPodSpec, agent, false) + updateWorkerPodSpec(agentPodSpec) + agentPodSpec.SecurityContext = &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), + RunAsGroup: ptr.To(int64(0)), + } + addSSHVolumeWithDefaultMode(agentPodSpec, sshSecretName, sshSecretName, 0644) + applyLPUHostDeviceVolumes(agentPodSpec, false) + agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ + Name: "hugepages", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumHugePages, + }}, + }) + return nil +} + +// configureNodeLocalConductorRuntime consumes a nonnil conductor PodSpec. +// The caller has validated the runtime SSH secret name. +func configureNodeLocalConductorRuntime( + conductorPodSpec *corev1.PodSpec, + targetFamily BuildFamily, + allocation string, + sshSecretName string, +) error { + // Resolve the conductor container before changing its image-only contract. + conductor, err := runtimeMainContainer(conductorPodSpec, targetFamily) + if err != nil { + return fmt.Errorf("node-local LPU conductor: %w", err) + } + + isXT := targetFamily == BuildFamilyXT + if !isXT && len(conductor.Command) != 0 { + return fmt.Errorf("node-local LPU runtime requires an image-only main container without command") + } + + // Materialize the GPC conductor around the image's Nova binary. + setNodeLocalPodIPEnv(conductor, isXT) + addRuntimeTemporaryStorage(conductorPodSpec, conductor, !isXT) + updateLPUConductorContainer(conductor, allocation) + if targetFamily == BuildFamilyHX { + conductor.Args = append(conductor.Args, nodeLocalHXAgentEnvironmentArgs...) + } + retargetMainContainerReferences(conductorPodSpec, conductor) + sshVolumeName := runtimeSSHVolumeName + if isXT { + sshVolumeName = sshSecretName + } + addConductorSSHKey(conductorPodSpec, sshSecretName, sshVolumeName) + return nil +} + +// configureNodeLocalAgentRuntime consumes a nonnil Agent PodSpec. +// The caller has validated the runtime SSH secret name. +func configureNodeLocalAgentRuntime( + agentPodSpec *corev1.PodSpec, + targetFamily BuildFamily, + validateXTSingleArgs bool, + sshSecretName string, +) error { + // Validate the emitted Agent's own shape before changing its startup contract. + agent, err := runtimeMainContainer(agentPodSpec, targetFamily) + if err != nil { + return fmt.Errorf("node-local LPU agent: %w", err) + } + isXT := targetFamily == BuildFamilyXT + if isXT { + strippedArgs, novaOnly := stripNovaOnlyArgs(agent.Args) + if validateXTSingleArgs && len(agent.Command) == 0 && !novaOnly { + return fmt.Errorf("operator-managed XT single only supports --instance-model-name and --agent-env-vars; set command for custom startup behavior") + } + agent.Args = strippedArgs + } else if len(agent.Command) != 0 { + return fmt.Errorf("node-local LPU runtime requires an image-only main container without command") + } + preserveAgentEntrypoint := isXT && (len(agent.Command) != 0 || len(agent.Args) != 0) + sshVolumeName := runtimeSSHVolumeName + if isXT { + sshVolumeName = sshSecretName + } + + // Materialize the LPU role as the privileged SSH target that Nova launches. + setNodeLocalPodIPEnv(agent, isXT) + agent.Name = lpuAgentContainerName + retargetMainContainerReferences(agentPodSpec, agent) + configureNodeLocalAgentWorkerContainer(agent, isXT, preserveAgentEntrypoint) + addRuntimeTemporaryStorage(agentPodSpec, agent, !isXT) + agentPodSpec.HostUsers = nil + updateWorkerPodSpec(agentPodSpec) + applyLPUHostDeviceVolumes(agentPodSpec, !isXT) + addSSHVolumeWithDefaultMode(agentPodSpec, sshSecretName, sshVolumeName, 0644) + + // Expose node-local devices and hugetlbfs through host /dev without an unaccounted HugePages volume. + addLPUHostDeviceVolumeMounts(agent) + agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount(sshVolumeName)) + if isXT { + applyLPUWorkerContainerBase(agent, preserveAgentEntrypoint) + agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) + agent.SecurityContext.RunAsNonRoot = ptr.To(false) + agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ + Name: "hugepages", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumHugePages, + }}, + }) + } else { + addLP30InitContainer(agentPodSpec, agent) + } + + return nil +} + +// runtimeMainContainer consumes a fresh selected role with a validated main container. +func runtimeMainContainer( + podSpec *corev1.PodSpec, + targetFamily BuildFamily, +) (*corev1.Container, error) { + // Keep V3's image-only single-container contract unchanged. + if targetFamily == BuildFamilyHX && len(podSpec.Containers) != 1 { + return nil, fmt.Errorf("PodSpec requires exactly one %q container", commonconsts.MainContainerName) + } + return findMainContainer(podSpec.Containers), nil +} + +func configureNodeLocalAgentWorkerContainer( + container *corev1.Container, + isXT bool, + preserveEntrypoint bool, +) { + // The Agent pod hosts SSHD; Nova starts /bin/agent in a separate MPI process. + if !preserveEntrypoint { + runCommand := lpuWorkerRunCommand + if isXT { + runCommand = lpuPartitionWorkerRunCommand + } else { + container.Command = []string{"/bin/bash"} + } + container.Args = []string{"-c", runCommand} + } + + if isXT { + if !preserveEntrypoint || container.ReadinessProbe == nil { + container.ReadinessProbe = lpuSSHProbe() + } + setContainerEnv(container, true, corev1.EnvVar{Name: "LPU_RUN_SYSTEM_INIT", Value: "true"}) + } else { + container.SecurityContext = &corev1.SecurityContext{ + Privileged: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + RunAsGroup: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), + } + container.TTY = true + container.Stdin = true + + // Remove health and lifecycle behavior owned by the source image process. + container.LivenessProbe = nil + container.StartupProbe = nil + container.Lifecycle = nil + container.ReadinessProbe = lpuSSHProbe() + setContainerEnv(container, true, lp30WorkerEnvironment...) + } +} + +func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) { + initContainer := corev1.Container{ + Name: lp30InitContainerName, + Image: agent.Image, + ImagePullPolicy: agent.ImagePullPolicy, + Command: []string{lp30InitPath}, + Env: []corev1.EnvVar{{ + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }}, + }}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + RunAsGroup: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), + }, + } + addLPUHostDeviceVolumeMounts(&initContainer) + podSpec.InitContainers = setContainerByName(podSpec.InitContainers, initContainer) +} + +func setNodeLocalPodIPEnv(container *corev1.Container, isXT bool) { + if isXT && + slices.ContainsFunc(container.Env, func(existing corev1.EnvVar) bool { return existing.Name == "POD_IP" }) { + return + } + container.Env = append(container.Env, corev1.EnvVar{ + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }) +} + +// retargetMainContainerReferences requires its non-nil target to point into the non-nil PodSpec's Containers. +func retargetMainContainerReferences(podSpec *corev1.PodSpec, target *corev1.Container) { + targetName := target.Name + // Keep discovery identity and explicit resource selectors valid anywhere in the Pod after renaming main. + for index := range podSpec.Containers { + container := &podSpec.Containers[index] + retargetEnvReferences(container.Env, targetName, container == target) + } + for index := range podSpec.InitContainers { + retargetEnvReferences(podSpec.InitContainers[index].Env, targetName, false) + } + for index := range podSpec.EphemeralContainers { + retargetEnvReferences(podSpec.EphemeralContainers[index].Env, targetName, false) + } + + // Retarget both standalone and projected downwardAPI resource files. + for index := range podSpec.Volumes { + volume := &podSpec.Volumes[index] + if volume.DownwardAPI != nil { + retargetResourceFieldReferencesInDownwardAPI(volume.DownwardAPI.Items, targetName) + } + if volume.Projected == nil { + continue + } + for sourceIndex := range volume.Projected.Sources { + downwardAPI := volume.Projected.Sources[sourceIndex].DownwardAPI + if downwardAPI != nil { + retargetResourceFieldReferencesInDownwardAPI(downwardAPI.Items, targetName) + } + } + } +} + +func retargetEnvReferences(env []corev1.EnvVar, targetName string, retargetDiscoveryIdentity bool) { + // Replace the target's discovery opt-in before retargeting explicit resource selectors. + for index := range env { + if retargetDiscoveryIdentity && env[index].Name == "CONTAINER_NAME" { + env[index] = corev1.EnvVar{Name: "CONTAINER_NAME", Value: targetName} + } + source := env[index].ValueFrom + if source == nil || source.ResourceFieldRef == nil { + continue + } + if source.ResourceFieldRef.ContainerName == commonconsts.MainContainerName { + source.ResourceFieldRef.ContainerName = targetName + } + } +} + +func retargetResourceFieldReferencesInDownwardAPI(items []corev1.DownwardAPIVolumeFile, targetName string) { + // DownwardAPI files require the renamed container's exact Kubernetes name. + for index := range items { + selector := items[index].ResourceFieldRef + if selector != nil && selector.ContainerName == commonconsts.MainContainerName { + selector.ContainerName = targetName + } + } +} + +func addRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container, replaceExisting bool) { + // Give conductor expansion and Agent setup a pod-lifetime writable workspace. + volume := corev1.Volume{ + Name: runtimeTemporaryStorageVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + mount := corev1.VolumeMount{Name: runtimeTemporaryStorageVolumeName, MountPath: runtimeTemporaryStorageMountPath} + if replaceExisting { + podSpec.Volumes = setVolumeByName(podSpec.Volumes, volume) + container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) + return + } + if !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.MountPath == runtimeTemporaryStorageMountPath + }) { + container.VolumeMounts = append(container.VolumeMounts, mount) + } + podSpec.Volumes = appendVolumeIfMissing(podSpec.Volumes, volume) +} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings.go b/deploy/operator/internal/dynamo/lpx/runtime_settings.go new file mode 100644 index 000000000000..3b007238a871 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings.go @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "fmt" + "maps" + "math/big" + "slices" + "strconv" + "strings" +) + +// maxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. +const maxSpecDecodeNumDrafts = 8 + +// Preserve the setup-ops discriminator consumed by the LPU runtime image. +const agentSetupOpsFormat = "agent_v2" + +// resolveBuildSettings requires a nonnil build; it normalizes and consumes modelSettings, which callers must not reuse. +func resolveBuildSettings(build *Build, modelSettings map[string]any) error { + // Preserve normalized defaults before reusing the compiler-owned field for model overrides. + defaults := build.runtimeSettings + build.runtimeSettings = modelSettings + splitCount, overridesSplitCount := modelSettings["num_batch_split_divisions"] + + // Overlay model settings before consuming the selected prop-sync and embedding partitions. + merged, err := mergeRuntimeSettingOverride(defaults, build.runtimeSettings, "settings") + if err != nil { + return err + } + settings := merged.(map[string]any) + + // Restore the original split count for exact validation after merging settings. + if overridesSplitCount { + settings["num_batch_split_divisions"] = splitCount + } + + // Preserve the selected prop-sync and embedding transformations on the merged settings. + cpuEmbeddings, err := boolSetting(settings, "cpu_embeddings") + if err != nil { + return fmt.Errorf("settings.%w", err) + } + if err := build.consumeRuntimeSelectedPropSyncChain(); err != nil { + return err + } + if cpuEmbeddings { + build.omitStandaloneEmbeddingPartition() + } + build.runtimeSettings = settings + + return normalizeLegacyNovaSettings(build.runtimeSettings) +} + +func mergeRuntimeSettingOverride(base, override any, path string) (any, error) { + if override == nil { + return nil, fmt.Errorf("%s must not be null", path) + } + + switch typed := override.(type) { + case map[string]any: + baseMap, _ := base.(map[string]any) + merged := make(map[string]any, len(baseMap)+len(typed)) + for key, value := range baseMap { + if _, overridden := typed[key]; !overridden { + merged[key] = value + } + } + for _, key := range slices.Sorted(maps.Keys(typed)) { + overrideValue := typed[key] + value, err := mergeRuntimeSettingOverride(baseMap[key], overrideValue, path+"."+key) + if err != nil { + return nil, err + } + if _, numeric := overrideValue.(json.Number); numeric { + typed[key] = value + } + merged[key] = value + } + return merged, nil + case []any: + if typed == nil { + return []any{}, nil + } + for i, element := range typed { + value, err := mergeRuntimeSettingOverride(nil, element, fmt.Sprintf("%s[%d]", path, i)) + if err != nil { + return nil, err + } + typed[i] = value + } + return typed, nil + case json.Number: + number := typed.String() + if !strings.ContainsAny(number, ".eE") { + if integer, err := strconv.ParseInt(number, 10, 64); err == nil { + return integer, nil + } + } + if float, err := strconv.ParseFloat(number, 64); err == nil { + return float, nil + } + return typed, nil + default: + return override, nil + } +} + +// validateRuntimeTokenizerSettings validates and normalizes tokenizer settings shared by V2 and V3. +func validateRuntimeTokenizerSettings(iop map[string]any) error { + tokenizerPath, ok := iop["tokenizer_path"] + if !ok { + return fmt.Errorf("iop.tokenizer_path is required; set spec.components[].lpx.settings.tokenizer_path when the capnp manifest does not provide model.tokenizer.path") + } + path, ok := tokenizerPath.(string) + path = strings.TrimSpace(path) + if !ok || path == "" { + return fmt.Errorf("iop.tokenizer_path must be a non-empty string") + } + iop["tokenizer_path"] = path + + stopTokens, ok := iop["stop_tokens"] + if !ok { + return fmt.Errorf("iop.stop_tokens is required; set spec.components[].lpx.settings.stop_tokens when the capnp manifest does not provide model.tokenizer.stopTokens") + } + var normalized []uint32 + switch tokens := stopTokens.(type) { + case []uint32: + normalized = append([]uint32(nil), tokens...) + case []any: + normalized = make([]uint32, len(tokens)) + for i, value := range tokens { + integer, ok := value.(int64) + if !ok || integer < 0 || uint64(integer) > uint64(^uint32(0)) { + return fmt.Errorf("iop.stop_tokens[%d] must be an integer between 0 and %d", i, uint64(^uint32(0))) + } + normalized[i] = uint32(integer) + } + default: + return fmt.Errorf("iop.stop_tokens must be an array") + } + iop["stop_tokens"] = normalized + return nil +} + +// normalizeLegacyNovaSettings validates legacy Agent V2 controls and removes +// their accepted defaults. It consumes caller-owned settings without touching build geometry. +func normalizeLegacyNovaSettings(iop map[string]any) error { + // Reject unsupported folding before omitting its accepted false value. + batchFolding, err := boolSetting(iop, "batch_folding") + if err != nil { + return fmt.Errorf("iop.batch_folding must be a boolean") + } + if batchFolding { + return fmt.Errorf("iop.batch_folding=true is not supported with setup_ops_format %q", agentSetupOpsFormat) + } + + // Accept only zero or one, including JSON numbers from unresolved V2 LPX settings. + if value, ok := iop["num_batch_split_divisions"]; ok { + var numBatchSplitDivisions *big.Rat + switch typed := value.(type) { + case int64: + numBatchSplitDivisions = new(big.Rat).SetInt64(typed) + case float64: + numBatchSplitDivisions = new(big.Rat).SetFloat64(typed) + case json.Number: + numBatchSplitDivisions, _ = new(big.Rat).SetString(typed.String()) + } + if numBatchSplitDivisions == nil || !numBatchSplitDivisions.IsInt() { + return fmt.Errorf("iop.num_batch_split_divisions must be an integer") + } + if numBatchSplitDivisions.Sign() < 0 || numBatchSplitDivisions.Cmp(big.NewRat(1, 1)) > 0 { + return fmt.Errorf( + "iop.num_batch_split_divisions=%v is not supported with setup_ops_format %q", + value, + agentSetupOpsFormat, + ) + } + } + + // Nova no longer accepts either key, even when the value matches its default. + delete(iop, "batch_folding") + delete(iop, "num_batch_split_divisions") + return nil +} + +func boolSetting(values map[string]any, key string) (bool, error) { + value, ok := values[key] + if !ok { + return false, nil + } + boolValue, ok := value.(bool) + if !ok { + return false, fmt.Errorf("%s must be a boolean", key) + } + return boolValue, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go new file mode 100644 index 000000000000..2c00d1b8a9b5 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go @@ -0,0 +1,368 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "maps" + "slices" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + "github.com/stretchr/testify/require" +) + +func TestResolveBuildSettingsSelectedPropSyncChainWithCPUEmbeddings(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + cpuSupported, standalone bool + firstPartition int + }{ + {"standalone CPU embeddings", true, true, 1}, + {"CPU embeddings unsupported", false, true, 0}, + {"partition zero is runnable", true, false, 0}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Construct a selected chain with independent embedding placement requirements") + build := selectedPropSyncBuildWithTopologies(t, + []string{registryTestTopology, registryTestTopology, registryTestTopology}, [][]int{{0, 1, 2}}) + build.SupportsCPUEmbeddings, build.StandaloneTokenEmbeddings = test.cpuSupported, test.standalone + build.runtimeSettings = map[string]any{"cpu_embeddings": true} + partitions := slices.Clone(build.Partitions) + configured := *build + + t.Log("Resolve the selected chain and omit partition zero only when both flags permit it") + require.NoError(t, resolveBuildSettings(&configured, nil)) + require.Equal(t, partitions[test.firstPartition:], configured.Partitions) + require.Empty(t, configured.SelectedPropSyncChains) + + t.Log("Keep the original partition IDs, paths and selected chain unchanged") + require.Equal(t, partitions, build.Partitions) + require.Equal(t, [][]int{{0, 1, 2}}, build.SelectedPropSyncChains) + }) + } +} + +func TestResolveBuildSettingsRejectsInvalidSelectedPropSyncChain(t *testing.T) { + t.Parallel() + + t.Log("Define malformed or incompatible selected prop-sync chains") + tests := []struct { + name string + chains [][]int + wantErr string + }{ + { + name: "multiple chains", + chains: [][]int{{0, 1}, {2, 3}}, + wantErr: "LPU-only runtime requires exactly one selected prop-sync chain, got 2", + }, + { + name: "missing partition", + chains: [][]int{{3, 4}}, + wantErr: "references missing partition id 4", + }, + { + name: "duplicate partition", + chains: [][]int{{1, 2, 1}}, + wantErr: "contains duplicate partition id 1", + }, + { + name: "noncontiguous partitions", + chains: [][]int{{1, 3}}, + wantErr: "is not contiguous at partition id 3", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Configure malformed or incompatible LPU-only selected-chain metadata") + build := selectedPropSyncBuild(t, tt.chains) + build.runtimeSettings = map[string]any{"cpu_embeddings": false} + configured := *build + + t.Log("Reject the build without consuming its selected-chain marker") + err := resolveBuildSettings(&configured, nil) + require.ErrorContains(t, err, tt.wantErr) + require.Equal(t, tt.chains, configured.SelectedPropSyncChains) + }) + } +} + +func TestResolveBuildSettingsDeepMergesDefaultsAndOverrides(t *testing.T) { + t.Parallel() + + t.Log("Construct paired runtime defaults and model overrides") + defaults, overrides := map[string]any{}, map[string]any{} + for name, values := range map[string][2]any{ + "batch_size": {int64(1), int64(2)}, + "sequence_length": {int64(131072), int64(4096)}, + "input_size": {int64(1), int64(2)}, + "num_kv_caches": {int64(1), int64(2)}, + "cpu_embeddings": {true, false}, + "dkvc": {true, false}, + "num_dkvc_blocks": {int64(256), int64(64)}, + "prop_sync": {true, false}, + "prop_sync_back_pressure": {false, true}, + "prop_sync_multiple_load_sets": {false, true}, + "has_init_tx": {false, true}, + "extended_topology": {true, false}, + "c2c_error_scan": {false, true}, + "num_batch_split_divisions": {int64(2), int64(0)}, + "batch_folding": {true, false}, + "num_layers": {int64(24), int64(25)}, + "vocab_size": {int64(201088), int64(201089)}, + "tokenizer_path": {"/build/tokenizer", "/dgd/tokenizer"}, + "stop_tokens": {[]uint32{200002, 199999, 200012}, []any{int64(42)}}, + "swa": { + map[string]any{ + "chunked": false, "num_swa_dkvc_blocks": int64(1), "swa_ctx_len": int64(128), + "swa_num_users": int64(8), "swa_padding_len": int64(0), + }, + map[string]any{ + "chunked": true, "num_swa_dkvc_blocks": int64(2), "swa_ctx_len": int64(256), + "swa_num_users": int64(4), "swa_padding_len": int64(16), + }, + }, + } { + defaults[name], overrides[name] = values[0], values[1] + } + overrides["custom_setting"] = "preserved" + overrides["custom_array"] = []any{map[string]any{"nested": []any{int64(1)}}} + build := &Build{CompilationMode: BuildCompilationModeLPUOnly, runtimeSettings: defaults} + + t.Log("Resolve overrides and verify every explicit model value wins") + require.NoError(t, resolveBuildSettings(build, overrides)) + expectedSettings := maps.Clone(overrides) + delete(expectedSettings, "batch_folding") + delete(expectedSettings, "num_batch_split_divisions") + require.Equal(t, expectedSettings, build.runtimeSettings, "every supported explicit DGD value must win") + + t.Log("Verify merge ownership and copy-on-write behavior") + overrideTokens := overrides["stop_tokens"].([]any) + configuredTokens := build.runtimeSettings["stop_tokens"].([]any) + require.Same(t, &overrideTokens[0], &configuredTokens[0], "caller-owned override slices are consumed") + empty, err := mergeRuntimeSettingOverride(nil, []any(nil), "settings.empty") + require.NoError(t, err) + require.NotNil(t, empty.([]any), "typed-nil arrays retain normalized empty-slice shape") + build.runtimeSettings["custom_setting"] = "changed" + require.Equal(t, "preserved", overrides["custom_setting"], "the settings map header remains independent") + build.runtimeSettings["swa"].(map[string]any)["swa_ctx_len"] = int64(512) + require.Equal(t, int64(128), defaults["swa"].(map[string]any)["swa_ctx_len"]) + require.Equal(t, uint32(200002), defaults["stop_tokens"].([]uint32)[0]) + require.Equal(t, int64(256), overrides["swa"].(map[string]any)["swa_ctx_len"], "overridden maps remain copy-on-write") + + t.Log("Complete a partial nested override from immutable defaults") + sharedDefault := []uint32{1} + partialDefaults := map[string]any{ + "shared": sharedDefault, + "swa": map[string]any{ + "chunked": false, + "num_swa_dkvc_blocks": int64(1), + "swa_ctx_len": int64(128), + "swa_num_users": int64(8), + "swa_padding_len": int64(0), + }, + } + partial := &Build{CompilationMode: BuildCompilationModeLPUOnly, runtimeSettings: partialDefaults} + require.NoError(t, resolveBuildSettings(partial, map[string]any{"swa": map[string]any{"chunked": true}})) + require.Equal(t, map[string]any{ + "chunked": true, + "num_swa_dkvc_blocks": int64(1), + "swa_ctx_len": int64(128), + "swa_num_users": int64(8), + "swa_padding_len": int64(0), + }, partial.runtimeSettings["swa"]) + + t.Log("Verify untouched defaults remain shared while nested maps remain isolated") + configuredDefault := partial.runtimeSettings["shared"].([]uint32) + require.Same(t, &sharedDefault[0], &configuredDefault[0], "untouched defaults remain immutable shared evidence") + partial.runtimeSettings["swa"].(map[string]any)["swa_ctx_len"] = int64(256) + require.Equal(t, int64(128), partialDefaults["swa"].(map[string]any)["swa_ctx_len"]) +} + +func TestProjectModelLegacyNovaSettings(t *testing.T) { + t.Parallel() + + t.Log("Capture the merged HX and direct XT-hybrid settings paths without legacy overrides") + singleIntent := ModelProjectionInput{ + Models: []string{"default"}, Pipeline: PipelineSingle, + BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), + } + hybridFixture := newV2CompilerFixture() + hybridFixture.compilationMode = manifestcapnp.CompilationMode_lpx + hybridIntent := ModelProjectionInput{ + Models: []string{"default"}, Pipeline: PipelineLPX, + BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, hybridFixture))), + } + singleBaselineBatch, err := appendModelProjections(nil, singleIntent) + require.NoError(t, err) + singleBaseline := singleBaselineBatch[0] + hybridBaselineBatch, err := appendModelProjections(nil, hybridIntent) + require.NoError(t, err) + hybridBaseline := hybridBaselineBatch[0] + + t.Log("Define accepted legacy defaults and unsupported values") + tests := []struct { + settings, wantErr string + hybrid bool + }{ + {`{"batch_folding":false}`, "", false}, + {`{"num_batch_split_divisions":0}`, "", false}, + {`{"num_batch_split_divisions":1}`, "", false}, + {`{"num_batch_split_divisions":1.0}`, "", false}, + {`{"num_batch_split_divisions":10e-1}`, "", false}, + {`{"num_batch_split_divisions":-0.0}`, "", false}, + {`{"batch_folding":true}`, `iop.batch_folding=true is not supported with setup_ops_format "agent_v2"`, false}, + {`{"batch_folding":"false"}`, "iop.batch_folding must be a boolean", false}, + {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported with setup_ops_format "agent_v2"`, false}, + {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported with setup_ops_format "agent_v2"`, false}, + {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported with setup_ops_format "agent_v2"`, false}, + {`{"num_batch_split_divisions":0.5}`, "iop.num_batch_split_divisions must be an integer", false}, + {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer", false}, + {`{"num_batch_split_divisions":1.00000000000000000001}`, "iop.num_batch_split_divisions must be an integer", false}, + {`{"num_batch_split_divisions":1e-1000}`, "iop.num_batch_split_divisions must be an integer", false}, + {`{"num_batch_split_divisions":"1"}`, "iop.num_batch_split_divisions must be an integer", false}, + {`{"batch_folding":false,"num_batch_split_divisions":1.0}`, "", true}, + {`{"batch_folding":true}`, `iop.batch_folding=true is not supported with setup_ops_format "agent_v2"`, true}, + {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer", true}, + } + + for _, test := range tests { + intent, baseline := singleIntent, singleBaseline + if test.hybrid { + intent, baseline = hybridIntent, hybridBaseline + } + t.Run(string(intent.Pipeline)+"/"+test.settings, func(t *testing.T) { + t.Log("Validate model-provided JSON settings before rendering") + intent.ModelSettings = []byte(test.settings) + projectionBatch, err := appendModelProjections(nil, intent) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + projection := projectionBatch[0] + + t.Log("Preserve physical partitions, runtime partitions, and prop-sync connectors") + require.Equal(t, baseline.partitions, projection.partitions) + require.Equal(t, baseline.configuredBuild.Partitions, projection.configuredBuild.Partitions) + require.Equal(t, baseline.connectors, projection.connectors) + + t.Log("Omit both accepted legacy keys from normalized settings and rendered TOML") + require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") + require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") + configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", []*ModelProjection{projection}, nil) + require.NoError(t, err) + require.NotContains(t, configMap.Data["model_config.toml"], "batch_folding") + require.NotContains(t, configMap.Data["model_config.toml"], "num_batch_split_divisions") + }) + } +} + +func TestResolveBuildSettingsRejectsNestedNullOverride(t *testing.T) { + t.Parallel() + + t.Log("Reject a null value in a nested known runtime setting") + build := &Build{runtimeSettings: map[string]any{ + "swa": map[string]any{"chunked": false}, + }} + + err := resolveBuildSettings(build, map[string]any{ + "swa": map[string]any{"chunked": nil}, + }) + require.ErrorContains(t, err, "settings.swa.chunked must not be null") + + t.Log("Reject a null value nested inside a custom array") + build = &Build{} + err = resolveBuildSettings(build, map[string]any{ + "custom": []any{map[string]any{"value": nil}}, + }) + require.ErrorContains(t, err, "settings.custom[0].value must not be null") + + t.Log("Report deterministic field order when multiple settings are null") + build = &Build{} + err = resolveBuildSettings(build, map[string]any{"z": nil, "a": nil}) + require.ErrorContains(t, err, "settings.a must not be null") +} + +func TestValidateRuntimeTokenizerSettings(t *testing.T) { + t.Parallel() + + t.Log("Define incomplete and invalid tokenizer runtime settings") + tests := []struct { + name string + settings map[string]any + wantErr string + }{ + { + name: "tokenizer path", + settings: map[string]any{"stop_tokens": []uint32{1}}, + wantErr: "iop.tokenizer_path is required; set spec.components[].lpx.settings.tokenizer_path", + }, + { + name: "stop tokens", + settings: map[string]any{"tokenizer_path": "/models/tokenizer"}, + wantErr: "iop.stop_tokens is required; set spec.components[].lpx.settings.stop_tokens", + }, + { + name: "negative stop token", + settings: map[string]any{ + "tokenizer_path": "/models/tokenizer", + "stop_tokens": []any{int64(-1)}, + }, + wantErr: "iop.stop_tokens[0] must be an integer between 0 and 4294967295", + }, + { + name: "non-integer stop token representation", + settings: map[string]any{ + "tokenizer_path": "/models/tokenizer", + "stop_tokens": []any{float64(1)}, + }, + wantErr: "iop.stop_tokens[0] must be an integer between 0 and 4294967295", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Validate and verify rejection of the tokenizer settings") + err := validateRuntimeTokenizerSettings(tt.settings) + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func selectedPropSyncBuild(t *testing.T, chains [][]int) *Build { + t.Helper() + + topology := registryTestTopology + return selectedPropSyncBuildWithTopologies(t, []string{topology, topology, topology, topology}, chains) +} + +func selectedPropSyncBuildWithTopologies(t *testing.T, topologies []string, chains [][]int) *Build { + t.Helper() + + partitions := make([]BuildPartition, len(topologies)) + for i, raw := range topologies { + topology, err := parse(raw) + require.NoError(t, err) + partitions[i] = BuildPartition{SourcePartitionID: i, PartPath: fmt.Sprintf("part-%d", i), Topology: topology} + } + return &Build{ + Path: "file:///path/to/build-id", + CompilationMode: BuildCompilationModeLPUOnly, + Partitions: partitions, + StandaloneTokenEmbeddings: true, + SupportsCPUEmbeddings: true, + SelectedPropSyncChains: chains, + } +} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_shape.go b/deploy/operator/internal/dynamo/lpx/runtime_shape.go new file mode 100644 index 000000000000..b78011c6927a --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_shape.go @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +// Pipeline is the runtime pipeline shape derived while projecting a model build. +type Pipeline string + +const ( + PipelineSingle Pipeline = "single" + PipelineLPX Pipeline = "lpx" + PipelineSpecDecode Pipeline = "specDecode" +) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go new file mode 100644 index 000000000000..580a0bcab0b3 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +const ( + runtimeSSHSecretMountPath = "/ssh-pk" + runtimeSSHSecretPrivateKeyPath = runtimeSSHSecretMountPath + "/private.key" + conductorSSHKeyDir = runtimeTemporaryStorageMountPath + "/dynamo-lpu-ssh" + conductorSSHPrivateKeyPath = conductorSSHKeyDir + "/private.key" + conductorSSHKeyInitContainerName = "prepare-ssh-key" + conductorSSHKeyVolumeName = "single-v2-ssh-key" + conductorSSHKeyInitCommand = "set -euo pipefail; cp " + runtimeSSHSecretPrivateKeyPath + " " + + conductorSSHPrivateKeyPath + "; chmod 600 " + conductorSSHPrivateKeyPath +) + +func addSSHVolumeWithDefaultMode(podSpec *corev1.PodSpec, secretName, volumeName string, defaultMode int32) { + sshVolume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: ptr.To(defaultMode), + }, + }, + } + podSpec.Volumes = setVolumeByName(podSpec.Volumes, sshVolume) +} + +func sshVolumeMount(volumeName string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: volumeName, + MountPath: runtimeSSHSecretMountPath, + ReadOnly: true, + } +} + +func addConductorSSHKey(podSpec *corev1.PodSpec, secretName, volumeName string) { + // Mount the source Secret and the writable destination used by OpenMPI. + addSSHVolumeWithDefaultMode(podSpec, secretName, volumeName, 0600) + podSpec.Volumes = setVolumeByName(podSpec.Volumes, corev1.Volume{ + Name: conductorSSHKeyVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + // Copy the key with OpenSSH's required mode before the conductor starts. + mainContainer := podSpec.Containers[0] + initContainer := corev1.Container{ + Name: conductorSSHKeyInitContainerName, + Image: mainContainer.Image, + ImagePullPolicy: mainContainer.ImagePullPolicy, + Command: []string{"/bin/bash"}, + Args: []string{"-c", conductorSSHKeyInitCommand}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(0)), + RunAsGroup: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), + }, + VolumeMounts: []corev1.VolumeMount{conductorSSHKeyVolumeMount()}, + } + initContainer.VolumeMounts = setVolumeMount(initContainer.VolumeMounts, sshVolumeMount(volumeName)) + podSpec.InitContainers = setContainerByName(podSpec.InitContainers, initContainer) +} + +func conductorSSHKeyVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir} +} diff --git a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go new file mode 100644 index 000000000000..590e6f5d8324 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go @@ -0,0 +1,134 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +// validateSelectedPropSyncGraph returns validated forward-adjacent edge source positions in selected-chain order. +func validateSelectedPropSyncGraph( + partitions []BuildPartition, + chains [][]int, + subject string, + requireCompatibleTopology bool, +) ([]int, error) { + // Index every physical partition once for both reference validation and connector projection. + partitionPositions := make(map[int]int, len(partitions)) + for position, partition := range partitions { + partitionPositions[partition.SourcePartitionID] = position + } + + // Reject malformed references across every chain before evaluating relationships between valid members. + for chainIndex, chain := range chains { + if len(chain) < 2 { + return nil, fmt.Errorf("%s %d must contain at least two partition IDs", subject, chainIndex) + } + for _, partitionID := range chain { + if _, present := partitionPositions[partitionID]; !present { + return nil, fmt.Errorf("%s %d references missing partition ID %d", subject, chainIndex, partitionID) + } + } + } + + // Enforce disjoint forward-adjacent chains and record each ordered physical edge by source position. + edgePositions := make([]int, 0) + for chainIndex, chain := range chains { + rootPosition := partitionPositions[chain[0]] + previousPosition := rootPosition + for memberIndex, partitionID := range chain { + position, unused := partitionPositions[partitionID] + if !unused { + return nil, fmt.Errorf("%s %d overlaps partition ID %d", subject, chainIndex, partitionID) + } + delete(partitionPositions, partitionID) + if requireCompatibleTopology && + !partitions[rootPosition].Topology.compatibleWith(partitions[position].Topology) { + return nil, fmt.Errorf("%s %d has incompatible topology at partition ID %d", subject, chainIndex, partitionID) + } + if memberIndex == 0 { + continue + } + + if position != previousPosition+1 { + return nil, fmt.Errorf("%s %d is not forward-adjacent at partition ID %d", subject, chainIndex, partitionID) + } + edgePositions = append(edgePositions, previousPosition) + previousPosition = position + } + } + return edgePositions, nil +} + +func (b *Build) consumeRuntimeSelectedPropSyncChain() error { + if b == nil || len(b.SelectedPropSyncChains) == 0 { + return nil + } + if len(b.SelectedPropSyncChains) != 1 { + return fmt.Errorf("LPU-only runtime requires exactly one selected prop-sync chain, got %d", len(b.SelectedPropSyncChains)) + } + chain := b.SelectedPropSyncChains[0] + if len(chain) < 2 { + return fmt.Errorf("LPU-only selected prop-sync chain %s must contain at least two partition ids", formatPropSyncChain(chain)) + } + + // Locate the selected chain in normalized physical partition order. + firstSelected := sort.Search(len(b.Partitions), func(index int) bool { + return b.Partitions[index].SourcePartitionID >= chain[0] + }) + + // A reached member has a contiguous prefix, so modular distance below its index identifies a duplicate. + for memberIndex, partitionID := range chain { + if uint(partitionID)-uint(chain[0]) < uint(memberIndex) { + return fmt.Errorf("LPU-only selected prop-sync chain %s contains duplicate partition id %d", formatPropSyncChain(chain), partitionID) + } + if memberIndex > 0 && partitionID != chain[memberIndex-1]+1 { + return fmt.Errorf("LPU-only selected prop-sync chain %s is not contiguous at partition id %d", formatPropSyncChain(chain), partitionID) + } + + partitionIndex := firstSelected + memberIndex + if partitionIndex >= len(b.Partitions) || b.Partitions[partitionIndex].SourcePartitionID != partitionID { + return fmt.Errorf("LPU-only selected prop-sync chain %s references missing partition id %d", formatPropSyncChain(chain), partitionID) + } + } + + b.Partitions = b.Partitions[firstSelected : firstSelected+len(chain)] + b.SelectedPropSyncChains = nil + return nil +} + +// collapseSelectedPropSyncChain projects validated nonempty contiguous physical partitions onto their runtime root. +func collapseSelectedPropSyncChain(chain []int, partitions []BuildPartition) (BuildPartition, error) { + root := partitions[0] + totalChipCount := 0 + totalNodeCount := 0 + for _, partition := range partitions { + totalChipCount += partition.Topology.ChipCount + totalNodeCount += partition.effectiveNodeCount() + } + + topology, err := root.Topology.withChipCount(totalChipCount) + if err != nil { + return BuildPartition{}, fmt.Errorf("selected prop-sync chain %s cannot form combined topology: %w", formatPropSyncChain(chain), err) + } + return BuildPartition{ + SourcePartitionID: root.SourcePartitionID, + PartPath: root.PartPath, + Topology: topology, + runtimeNodeCount: totalNodeCount, + }, nil +} + +func formatPropSyncChain(chain []int) string { + ids := make([]string, 0, len(chain)) + for _, partitionID := range chain { + ids = append(ids, strconv.Itoa(partitionID)) + } + return "[" + strings.Join(ids, ",") + "]" +} diff --git a/deploy/operator/internal/dynamo/lpx/selected_workload.go b/deploy/operator/internal/dynamo/lpx/selected_workload.go new file mode 100644 index 000000000000..67b86e5d5ddd --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/selected_workload.go @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import "strings" + +// SelectedWorkload is the immutable result of resolving exactly one DGD for +// LPX scheduling and runtime materialization. +// Its methods require a successfully resolved, non-nil workload with projections. +type SelectedWorkload struct { + modelProjections []*ModelProjection + digest WorkloadDigest + engineReplicas int32 +} + +// ModelProjections returns a read-only view of the selected projection list. +// Callers must not modify the slice or its projections. +func (w *SelectedWorkload) ModelProjections() []*ModelProjection { + return w.modelProjections +} + +// Digest returns the selected workload digest. +func (w *SelectedWorkload) Digest() WorkloadDigest { + return w.digest +} + +// BuildFamily returns the selected build family. +func (w *SelectedWorkload) BuildFamily() BuildFamily { + return w.modelProjections[0].configuredBuild.Family +} + +// Pipeline returns the selected pipeline. +func (w *SelectedWorkload) Pipeline() Pipeline { + return w.modelProjections[0].pipeline +} + +// LPXComponentName returns the serving component name from the final projection. +func (w *SelectedWorkload) LPXComponentName() string { + return w.modelProjections[len(w.modelProjections)-1].stage +} + +// CyborgTemplateName derives the hybrid engine's GPU-role template name. +// A non-hybrid workload has no Cyborg template. +func (w *SelectedWorkload) CyborgTemplateName() string { + if w.Pipeline() != PipelineLPX { + return "" + } + return strings.ToLower(w.LPXComponentName()) + "-engine-gpu" +} diff --git a/deploy/operator/internal/dynamo/lpx/selection.go b/deploy/operator/internal/dynamo/lpx/selection.go new file mode 100644 index 000000000000..be208148f72a --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/selection.go @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "k8s.io/utils/ptr" +) + +const ( + draftStageName = "draft" + runtimeModelDraft = "draft" + runtimeModelTarget = "target" +) + +// Components returns LPX components. In a supported pair, the worker-only draft +// precedes the serving component. dgd is non-nil and is not mutated. +// Returned component pointers remain read-only. +func Components(dgd *dynamov1beta1.DynamoGraphDeployment) []*dynamov1beta1.DynamoComponentDeploymentSharedSpec { + components := make([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, 0, 2) + for index := range dgd.Spec.Components { + component := &dgd.Spec.Components[index] + if component.IsLPX() { + components = append(components, component) + } + } + if len(components) == 2 && components[0].ComponentRole(dynamov1beta1.ComponentRoleLeader) != nil { + components[0], components[1] = components[1], components[0] + } + return components +} + +// ServingComponent returns the sole LPX component or the component declaring +// the shared leader role. It returns nil when a pair has no leader owner. +// dgd is non-nil and the returned component is read-only. +func ServingComponent(dgd *dynamov1beta1.DynamoGraphDeployment) *dynamov1beta1.DynamoComponentDeploymentSharedSpec { + var soleComponent *dynamov1beta1.DynamoComponentDeploymentSharedSpec + componentCount := 0 + for index := range dgd.Spec.Components { + component := &dgd.Spec.Components[index] + if !component.IsLPX() { + continue + } + if component.ComponentRole(dynamov1beta1.ComponentRoleLeader) != nil { + return component + } + soleComponent = component + componentCount++ + } + if componentCount == 1 { + return soleComponent + } + return nil +} + +// SelectedModelNames returns build-backed runtime model identities, in canonical +// stage and draft-instance order, for a supported runtime. +// dgd must be non-nil and is not mutated. +func SelectedModelNames(dgd *dynamov1beta1.DynamoGraphDeployment) ([]string, error) { + components := Components(dgd) + if err := validateLPXComposition(dgd, len(components)).ToAggregate(); err != nil { + return nil, fmt.Errorf("%w: %w", ErrUnsupportedRuntime, err) + } + var names []string + for index, component := range components { + names = append(names, expandedSelectedModelNames(index, len(components), int(ptr.Deref(component.Replicas, 1)))...) + } + return names, nil +} + +func expandedSelectedModelNames(index, componentCount, draftCount int) []string { + // Component membership retains Nova's existing default/draft/target identities. + if componentCount == 1 { + return []string{"default"} + } + if index == 1 { + return []string{runtimeModelTarget} + } + names := make([]string, draftCount) + for index := range names { + names[index] = fmt.Sprintf("draft%d", index) + } + return names +} diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go new file mode 100644 index 000000000000..10810bf283f5 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/selection_test.go @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { + t.Log("Select components by their declared type, independently of build or templates") + canonicalLPX := []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentType: dynamov1beta1.ComponentTypeLPX}} + + t.Log("Define default-selection scenarios") + tests := []struct { + name string + annotations map[string]string + components []dynamov1beta1.DynamoComponentDeploymentSharedSpec + want bool + }{ + {name: "canonical LPX", components: canonicalLPX, want: true}, + { + name: "component provider", + annotations: map[string]string{ + commonconsts.KubeAnnotationWorkloadProvider: commonconsts.WorkloadProviderComponent, + }, + components: canonicalLPX, + want: true, + }, + { + name: "Grove opt-out", + annotations: map[string]string{ + commonconsts.KubeAnnotationEnableGrove: "FALSE", + }, + components: canonicalLPX, + want: true, + }, + {name: "non-LPX", components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: "worker"}}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Construct the DGD selection input") + dgd := &dynamov1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Annotations: test.annotations}, + Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ + Components: test.components, + Scheduling: &dynamov1beta1.SchedulingSpec{}, + }, + } + + t.Log("Resolve and verify scheduler selection") + selected := dgd.HasLPXComponent() + require.Equal(t, test.want, selected) + }) + } +} + +func TestSelectedModelNames(t *testing.T) { + t.Log("Construct a SpecDecode selection with draft fanout") + draftReplicas := int32(3) + dgd := &dynamov1beta1.DynamoGraphDeployment{Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ + Components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{ + testLPXComponent("small", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("draft")}), + testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: "leader"}, dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("target")}), + }, + }} + dgd.Spec.Components[0].Replicas = &draftReplicas + + t.Log("Expand the configured draft replica count into logical model names") + names, err := SelectedModelNames(dgd) + require.NoError(t, err) + require.Equal(t, []string{"draft0", "draft1", "draft2", "target"}, names) + + t.Log("Default an omitted draft replica count to one logical draft") + dgd.Spec.Components[0].Replicas = nil + names, err = SelectedModelNames(dgd) + require.NoError(t, err) + require.Equal(t, []string{"draft0", "target"}, names) + + for _, invalid := range []int32{0, 9} { + t.Logf("Reject invalid draft replica count %d", invalid) + dgd.Spec.Components[0].Replicas = &invalid + _, err = SelectedModelNames(dgd) + require.ErrorIs(t, err, ErrUnsupportedRuntime) + } + t.Log("Derive the existing default runtime identity with an implicit conductor") + dgd.Spec.Components = dgd.Spec.Components[1:] + dgd.Spec.Components[0].Roles = dgd.Spec.Components[0].Roles[1:] + names, err = SelectedModelNames(dgd) + require.NoError(t, err) + require.Equal(t, []string{"default"}, names) + require.Same(t, &dgd.Spec.Components[0], ServingComponent(dgd)) +} + +func TestSelectedModelNamesRequiresSharedConductorOwner(t *testing.T) { + t.Log("Read two agent-only components without a shared conductor owner") + dgd := newSelectedTestDGD(t, "missing-conductor", + testLPXComponent("small", "build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("agent")}), + testLPXComponent("large", "build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("agent")}), + ) + before := dgd.DeepCopy() + + t.Log("Reject the incomplete runtime without inventing a conductor") + names, err := SelectedModelNames(dgd) + require.ErrorIs(t, err, ErrUnsupportedRuntime) + require.Nil(t, names) + require.Nil(t, ServingComponent(dgd)) + require.Equal(t, before, dgd) +} diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go new file mode 100644 index 000000000000..42bc46fa89d5 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -0,0 +1,241 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "path" + "strings" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" +) + +// ValidateSelectedIntent validates the snapshot-independent selected-LPX +// requirements beyond admission's structural checks before controller downloads +// and selected-workload projection. +// dgd must be non-nil. +func ValidateSelectedIntent(dgd *dynamov1beta1.DynamoGraphDeployment) field.ErrorList { + _, allErrs := validateSelectedIntent(dgd) + return allErrs +} + +func validateSelectedIntent(dgd *dynamov1beta1.DynamoGraphDeployment) ([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, field.ErrorList) { + specPath := field.NewPath("spec") + if !dgd.HasLPXComponent() { + return nil, field.ErrorList{field.Forbidden( + specPath.Child("components"), + "DynamoGraphDeployment has no LPX component", + )} + } + allErrs := field.ErrorList{} + if dgd.Spec.TopologyConstraint != nil { + allErrs = append(allErrs, field.Forbidden( + specPath.Child("topologyConstraint"), + "selected LPX does not support Grove topologyConstraint", + )) + } + if dgd.Spec.ProviderOverride != nil { + allErrs = append(allErrs, field.Forbidden( + specPath.Child("providerOverride"), + "selected LPX does not support Grove topology overrides on the deployment", + )) + } + components := Components(dgd) + allErrs = append(allErrs, validateLPXComposition(dgd, len(components))...) + // Keep errors tied to authored component and role indices rather than runtime order. + for index := range dgd.Spec.Components { + component := &dgd.Spec.Components[index] + if !component.IsLPX() { + continue + } + componentPath := field.NewPath("spec", "components").Index(index) + allErrs = append(allErrs, validateSelectedLPXComponent(component, componentPath)...) + } + return components, allErrs +} + +// validateLPXComposition bounds the shared runtime before model expansion or acquisition. +func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentCount int) field.ErrorList { + componentsPath := field.NewPath("spec", "components") + if componentCount < 1 || componentCount > 2 { + return field.ErrorList{field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")} + } + allErrs := field.ErrorList{} + conductorCount := 0 + for index := range dgd.Spec.Components { + component := &dgd.Spec.Components[index] + if !component.IsLPX() { + continue + } + componentPath := componentsPath.Index(index) + conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLeader) + if conductor != nil { + conductorCount++ + } + if componentCount != 2 { + continue + } + // Draft fanout is per model instance; the target owns one shared serving runtime. + replicas := ptr.Deref(component.Replicas, 1) + if conductor == nil { + if replicas < 1 || replicas > maxSpecDecodeNumDrafts { + allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "draft replicas must be between 1 and 8")) + } + if component.ModelRef != nil { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("modelRef"), "the shared target owns the serving endpoint")) + } + } else if replicas != 1 { + allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "shared target replicas must be one")) + } + } + if componentCount == 2 && conductorCount != 1 { + allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one leader role")) + } + return allErrs +} + +// validateSelectedLPXComponent validates inputs required by the existing runtime. +// Shared role admission owns structure; composition validation bounds materialization. +func validateSelectedLPXComponent(component *dynamov1beta1.DynamoComponentDeploymentSharedSpec, componentPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if component.Experimental != nil && component.Experimental.Checkpoint != nil && component.Experimental.Checkpoint.Enabled { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("experimental", "checkpoint"), + fmt.Sprintf("selected LPX component %q does not support checkpointing", component.ComponentName))) + } + if component.TopologyConstraint != nil { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("topologyConstraint"), + fmt.Sprintf("selected LPX component %q does not support Grove topologyConstraint", component.ComponentName))) + } + if component.ProviderOverride != nil { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("providerOverride"), "LPX component does not support Grove topology overrides")) + } + if component.IsMultinode() { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("multinode"), "LPX component cannot use multinode")) + } + config := component.LPX + configPath := componentPath.Child("lpx") + if strings.TrimSpace(config.BuildID) == "" { + allErrs = append(allErrs, field.Required(configPath.Child("buildId"), "LPX component requires a buildId")) + } + for index, role := range component.Roles { + if role.PodTemplate == nil { + continue + } + rolePath := componentPath.Child("roles").Index(index) + main, mainPath, errs := validateSelectedRuntimeContainer(role.PodTemplate, "LPX "+role.Name, rolePath) + allErrs = append(allErrs, errs...) + if main != nil && role.Name == dynamov1beta1.ComponentRoleWorker { + allErrs = append(allErrs, validateAllocationInjectionTargetFields(main, mainPath)...) + } + allErrs = append(allErrs, validateLPXRolePlacement(&role.PodTemplate.Spec, rolePath.Child("podTemplate", "spec"), role.Name == dynamov1beta1.ComponentRoleWorker)...) + } + return allErrs +} + +// validateLPXRolePlacement protects controller-owned addressing and scheduling. +// Required node affinity remains a supported input to LPX Agent placement. +func validateLPXRolePlacement(spec *corev1.PodSpec, fldPath *field.Path, agent bool) field.ErrorList { + allErrs := field.ErrorList{} + if spec.SchedulerName != "" && spec.SchedulerName != corev1.DefaultSchedulerName { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulerName"), "LPX owns role scheduler selection")) + } + for _, entry := range []struct{ name, value string }{ + {"hostname", spec.Hostname}, {"subdomain", spec.Subdomain}, {"nodeName", spec.NodeName}, + } { + if entry.value != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child(entry.name), "LPX owns role addressing and placement")) + } + } + if len(spec.TopologySpreadConstraints) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("topologySpreadConstraints"), "LPX owns role placement")) + } + if !agent { + return allErrs + } + if len(spec.NodeSelector) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("nodeSelector"), "LPX exclusively owns Agent node selection")) + } + if a := spec.Affinity; a != nil && + ((a.NodeAffinity != nil && len(a.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0) || + (a.PodAffinity != nil && (len(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0)) || + (a.PodAntiAffinity != nil && (len(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0))) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("affinity"), "node-local LPX supports only required nodeAffinity")) + } + if len(spec.SchedulingGates) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulingGates"), "Grove and LPX own Agent scheduling gates")) + } + if len(spec.ResourceClaims) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("resourceClaims"), "node-local LPX Agents cannot use ResourceClaims")) + } + return allErrs +} + +// validateSelectedRuntimeContainer validates a present template at its authored role path. +func validateSelectedRuntimeContainer( + template *corev1.PodTemplateSpec, + role string, + rolePath *field.Path, +) (*corev1.Container, *field.Path, field.ErrorList) { + containersPath := rolePath.Child("podTemplate", "spec", "containers") + for index := range template.Spec.Containers { + container := &template.Spec.Containers[index] + if container.Name != commonconsts.MainContainerName { + continue + } + containerPath := containersPath.Index(index) + return container, containerPath, nil + } + return nil, nil, field.ErrorList{ + field.Required( + containersPath, + fmt.Sprintf("%s component requires a %q runtime container", role, commonconsts.MainContainerName), + ), + } +} + +// validateAllocationInjectionTargetFields requires nonnil container and containerPath. +func validateAllocationInjectionTargetFields(container *corev1.Container, containerPath *field.Path) field.ErrorList { + // Classify Command and Args once before reporting violations in contract order. + var hasAllocation, terminatesArguments, hasShell [2]bool + for vector, values := range [2][]string{container.Command, container.Args} { + for _, value := range values { + hasAllocation[vector] = hasAllocation[vector] || value == "--allocation" || + strings.HasPrefix(value, "--allocation=") + terminatesArguments[vector] = terminatesArguments[vector] || value == "--" + if !hasShell[vector] { + switch path.Base(value) { + case "ash", "bash", "dash", "ksh", "sh", "zsh": + hasShell[vector] = true + } + } + } + } + + allErrs := field.ErrorList{} + if hasAllocation[0] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list")) + } + if hasAllocation[1] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list")) + } + if terminatesArguments[0] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container must not terminate arguments before Dynamo appends --allocation")) + } + if terminatesArguments[1] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container must not terminate arguments before Dynamo appends --allocation")) + } + if hasShell[0] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container cannot use a shell because Dynamo appends --allocation")) + } + if hasShell[1] { + allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container cannot use a shell because Dynamo appends --allocation")) + } + return allErrs +} diff --git a/deploy/operator/internal/dynamo/lpx/topology.go b/deploy/operator/internal/dynamo/lpx/topology.go new file mode 100644 index 000000000000..ad9cf522ba0e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/topology.go @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +var topologyRe = regexp.MustCompile( + `^(.+?)__(.+?)__([0-9]+)C__G_([0-9_]+?)__(.+?)__GHZ_([0-9_]+?)__(.+?)` + + `(?:__G_([0-9_]+?))?(?:__(.+?))?$`, +) + +var legacyTopologyRe = regexp.MustCompile(`(^|_)(-?\d+)_CHIP(_|$)`) +var topologyChipCountRe = regexp.MustCompile(`__([0-9]+)C__`) + +// Topology is a parsed compiler topology with its scheduler-facing chip count. +type Topology struct { + // ChipCount is the number of LPU chips represented by the topology. + ChipCount int + // Raw is the original compiler topology string. + Raw string + compatibilityKey string +} + +// Replicas returns the number of replicas (ChipCount / 8, one per node, floor of 1). +// The receiver must be non-nil and is not mutated. +func (t *Topology) Replicas() int { + return max(1, t.ChipCount/lpuChipsPerNode) +} + +// compatibleWith reports whether two topologies share the V2 prop-sync compatibility identity. +func (t Topology) compatibleWith(other Topology) bool { + return t.compatibilityKey == other.compatibilityKey +} + +func (t Topology) withChipCount(chipCount int) (Topology, error) { + if chipCount <= 0 { + return Topology{}, fmt.Errorf("chip count must be positive") + } + + raw := t.Raw + switch { + case strings.Contains(raw, "__"): + raw = topologyChipCountRe.ReplaceAllString(raw, fmt.Sprintf("__%dC__", chipCount)) + case legacyTopologyRe.MatchString(raw): + raw = legacyTopologyRe.ReplaceAllString(raw, fmt.Sprintf("${1}%d_CHIP${3}", chipCount)) + default: + return Topology{}, fmt.Errorf("topology %q does not contain a replaceable chip count", raw) + } + + return parse(raw) +} + +func parse(raw string) (Topology, error) { + if strings.Contains(raw, "__") { + matches := topologyRe.FindStringSubmatchIndex(raw) + if matches == nil { + return Topology{}, fmt.Errorf("topology %q does not match expected pattern", raw) + } + + chipCount, err := strconv.Atoi(raw[matches[6]:matches[7]]) + if err != nil { + return Topology{}, fmt.Errorf("parsing chip count from %q: %w", raw, err) + } + + return Topology{ + ChipCount: chipCount, + Raw: raw, + compatibilityKey: raw[:matches[6]] + raw[matches[7]:], + }, nil + } + + match := strings.Trim(legacyTopologyRe.FindString(raw), "_") + if match == "" { + return Topology{}, fmt.Errorf("topology %q does not match expected pattern", raw) + } + + chipCount, err := strconv.Atoi(strings.TrimSuffix(match, "_CHIP")) + if err != nil { + return Topology{}, fmt.Errorf("parsing chip count from %q: %w", raw, err) + } + + return Topology{ + ChipCount: chipCount, + Raw: raw, + }, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/topology_test.go b/deploy/operator/internal/dynamo/lpx/topology_test.go new file mode 100644 index 000000000000..2dde1061da5b --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/topology_test.go @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import "testing" + +func TestParse(t *testing.T) { + t.Log("Define modern, legacy, and malformed topology strings") + tests := []struct { + name string + raw string + want Topology + wantErr bool + }{ + { + name: "without optional fields", + raw: "URSA_V2_1__INT8__32C__G_128_128__FEC_A__GHZ_1_7__FPGA_X", + want: Topology{ + ChipCount: 32, + Raw: "URSA_V2_1__INT8__32C__G_128_128__FEC_A__GHZ_1_7__FPGA_X", + compatibilityKey: "URSA_V2_1__INT8__C__G_128_128__FEC_A__GHZ_1_7__FPGA_X", + }, + }, + { + name: "with optional fields", + raw: "URSA_V2_1__FP8__64C__G_256_256__FEC_B__GHZ_2_0__FPGA_Y__G_512_512__BETA", + want: Topology{ + ChipCount: 64, + Raw: "URSA_V2_1__FP8__64C__G_256_256__FEC_B__GHZ_2_0__FPGA_Y__G_512_512__BETA", + compatibilityKey: "URSA_V2_1__FP8__C__G_256_256__FEC_B__GHZ_2_0__FPGA_Y__G_512_512__BETA", + }, + }, + { + name: "legacy", + raw: "RT_A14_8_CHIP_PS2_NOFEC", + want: Topology{ + ChipCount: 8, + Raw: "RT_A14_8_CHIP_PS2_NOFEC", + }, + }, + { + name: "invalid format", + raw: "not-a-topology", + wantErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Logf("Parse topology %q", tt.raw) + got, err := parse(tt.raw) + if tt.wantErr { + t.Log("Verify malformed topology input is rejected") + if err == nil { + t.Fatalf("parse(%q) expected error, got nil", tt.raw) + } + return + } + + t.Log("Verify valid topology input produces the expected normalized shape") + if err != nil { + t.Fatalf("parse(%q) unexpected error: %v", tt.raw, err) + } + + if got != tt.want { + t.Fatalf("parse(%q) = %#v, want %#v", tt.raw, got, tt.want) + } + }) + } +} + +func TestTopologyCompatibleWith(t *testing.T) { + t.Log("Define modern, legacy, and empty topology compatibility cases") + const modern = "A__Q__8C__G_1__F__GHZ_1__P" + tests := []struct { + name string + leftRaw string + rightRaw string + want bool + }{ + {name: "modern chip count differs", leftRaw: modern, rightRaw: "A__Q__16C__G_1__F__GHZ_1__P", want: true}, + {name: "modern non-count segment differs", leftRaw: modern, rightRaw: "A__R__8C__G_1__F__GHZ_1__P"}, + {name: "modern optional field differs", leftRaw: modern, rightRaw: modern + "__G_2__S"}, + {name: "legacy raw and chip count differ", leftRaw: "A_8_CHIP_X", rightRaw: "B_16_CHIP_Y", want: true}, + {name: "modern and legacy", leftRaw: modern, rightRaw: "A_8_CHIP_X"}, + {name: "legacy and zero-field topology", leftRaw: "A_8_CHIP_X", want: true}, + {name: "zero-field topology and legacy", rightRaw: "A_8_CHIP_X", want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Parse topology compatibility state and compare it with the requested peer") + left := Topology{} + var err error + if tt.leftRaw != "" { + left, err = parse(tt.leftRaw) + if err != nil { + t.Fatalf("parse(%q) unexpected error: %v", tt.leftRaw, err) + } + } + right := Topology{} + if tt.rightRaw != "" { + right, err = parse(tt.rightRaw) + if err != nil { + t.Fatalf("parse(%q) unexpected error: %v", tt.rightRaw, err) + } + } + + t.Log("Verify the compatibility result") + if got := left.compatibleWith(right); got != tt.want { + t.Fatalf("compatibleWith() = %t, want %t", got, tt.want) + } + }) + } + + t.Run("withChipCount preserves compatibility", func(t *testing.T) { + t.Log("Resize one modern topology and compare its retained compatibility state") + topology, err := parse(modern) + if err != nil { + t.Fatalf("parse(%q) unexpected error: %v", modern, err) + } + + t.Log("Resize the parsed topology") + resized, err := topology.withChipCount(64) + if err != nil { + t.Fatalf("withChipCount() unexpected error: %v", err) + } + + t.Log("Verify resizing preserves compatibility") + if !topology.compatibleWith(resized) { + t.Fatal("withChipCount() changed topology compatibility") + } + }) +} + +func TestTopologyReplicas(t *testing.T) { + t.Log("Define topology chip counts and expected host replica counts") + tests := []struct { + name string + topo Topology + want int + }{ + { + name: "exact multiple", + topo: Topology{ChipCount: 32}, + want: 4, + }, + { + name: "single node", + topo: Topology{ChipCount: 8}, + want: 1, + }, + { + name: "integer division", + topo: Topology{ChipCount: 30}, + want: 3, + }, + { + name: "single chip", + topo: Topology{ChipCount: 1}, + want: 1, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Log("Derive and verify the topology replica count") + got := tt.topo.Replicas() + if got != tt.want { + t.Fatalf("Replicas() = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go new file mode 100644 index 000000000000..e1c2da6cbeec --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -0,0 +1,219 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" +) + +const ( + // Preserve the versioned digest domains so this terminology refactor does not + // introduce an additional identity change beyond the base PR. + modelProjectionDigestVersion = "dynamo-node-local-compilation/v1" + workloadSetDigestVersion = "dynamo-lpx-compilation-set/v1" + maxLPXPartitions = 256 + // LPX reserves 192 KiB of the Kubernetes 1 MiB object ceiling for + // lifecycle status. Its pre-solve admission charges the status-free + // request plus a deterministic echo of every connector requirement. + maxLPXPlannedRequestObjectBytes = (1024 - 192) * 1024 +) + +// appendModelProjections appends one component's immutable model projections to +// the caller-owned destination, which may be nil. Existing elements are unchanged. +// intent.BuildSnapshot contains a normalized, non-nil build; Models is nonempty +// and Pipeline is selected by the validated resolver. ModelSettings is API-validated +// JSON: omitted, null or an object. Source inputs are not mutated; discard error results. +func appendModelProjections(dst []*ModelProjection, intent ModelProjectionInput) ([]*ModelProjection, error) { + modelSettings, modelSettingsObject := canonicalModelSettings(intent.ModelSettings) + intent.ModelSettings = modelSettings + + var projections []*ModelProjection + var err error + switch intent.BuildSnapshot.build.Family { + case BuildFamilyXT: + projections, err = appendV2ModelProjections(dst, intent, modelSettingsObject) + case BuildFamilyHX: + projections, err = appendV3ModelProjections(dst, intent, modelSettingsObject) + default: + return nil, fmt.Errorf("unsupported LPX target family %q", intent.BuildSnapshot.build.Family) + } + if err != nil { + return nil, err + } + // Bound the component's physical build before runtime expansion and request publication. + projection := projections[len(dst)] + if len(projection.partitions) < 1 || len(projection.partitions) > maxLPXPartitions { + return nil, fmt.Errorf("LPX projection has %d partitions, limit is 1..%d", len(projection.partitions), maxLPXPartitions) + } + return projections, nil +} + +func workloadSetDigest(projections []*ModelProjection) (WorkloadDigest, error) { + first := projections[0] + if len(projections) == 1 { + return first.digest, nil + } + transcript := &digestTranscript{hash: sha256.New()} + transcript.field("schema", []byte(workloadSetDigestVersion)) + for _, projection := range projections { + if projection.configuredBuild.Family != first.configuredBuild.Family { + return WorkloadDigest{}, fmt.Errorf( + "LPX model projections have mixed target families %q and %q", + first.configuredBuild.Family, + projection.configuredBuild.Family, + ) + } + transcript.field("model", []byte(projection.model)) + // Keep the versioned field tag stable so this terminology refactor does not + // introduce an additional aggregate identity change beyond the base PR. + transcript.field("compilation-digest", projection.digest[:]) + } + return transcript.sum(), nil +} + +// ValidateRequestSize applies the pinned LPX scheduler's pre-solve size +// accounting to a non-nil compiler-produced or API-decoded request with valid +// raw JSON fields. The request is not mutated. +func ValidateRequestSize(request *lpxv1alpha1.LPUPipelineRequest) error { + projectedBytes := requestWithRequirementEchoBytes(request) + if projectedBytes > maxLPXPlannedRequestObjectBytes { + return fmt.Errorf( + "status-free request plus connector-requirement echo is %d bytes; maximum is %d", + projectedBytes, + maxLPXPlannedRequestObjectBytes, + ) + } + return nil +} + +func requestWithRequirementEchoBytes(request *lpxv1alpha1.LPUPipelineRequest) int { + statusFree := *request + statusFree.Status = nil + if statusFree.ResourceVersion != "" { + statusFree.ResourceVersion = "0" + } + requestBytes, _ := json.Marshal(&statusFree) + requirements := make([]lpxv1alpha1.PropSyncConnectorRequirement, 0, len(request.Spec.PropSyncConnectors)) + for _, connector := range request.Spec.PropSyncConnectors { + requirements = append(requirements, connector.Requirement) + } + echoBytes, _ := json.Marshal(requirements) + return len(requestBytes) + len(echoBytes) +} + +// canonicalModelSettings canonicalizes an API-validated optional JSON object. +func canonicalModelSettings(raw json.RawMessage) (json.RawMessage, map[string]any) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil, make(map[string]any) + } + + // Decode the admitted object while preserving number spellings for identity. + decoder := json.NewDecoder(bytes.NewReader(trimmed)) + decoder.UseNumber() + var object map[string]any + _ = decoder.Decode(&object) + + // Decoder-owned values encode infallibly in memory; drop Encoder's documented trailing newline. + var output bytes.Buffer + encoder := json.NewEncoder(&output) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(object) + return output.Bytes()[:output.Len()-1], object +} + +func newModelProjectionTranscripts(intent ModelProjectionInput, projectionVersion string) []digestTranscript { + // Each logical model owns a fresh hash while sharing the component's canonical inputs. + transcripts := make([]digestTranscript, len(intent.Models)) + for index, model := range intent.Models { + transcript := &transcripts[index] + transcript.hash = sha256.New() + transcript.field("schema", []byte(modelProjectionDigestVersion)) + // Keep the versioned field tag stable so this terminology refactor does not + // introduce an additional identity change beyond the base PR. + transcript.field("lowerer", []byte(projectionVersion)) + // Ref is an acquisition locator, not build content. In particular, a + // file-backed snapshot's ref contains its absolute checkout path. + transcript.field("build-content-id", []byte(intent.BuildSnapshot.contentID)) + // Bind the family derived from immutable build metadata without retaining the + // removed API selector. + version := "v2" + if intent.BuildSnapshot.build.Family == BuildFamilyHX { + version = "v3" + } + transcript.field("device-version", []byte(version)) + transcript.field("pipeline", []byte(intent.Pipeline)) + + // Preserve the digest's wire values without storing a second runtime discriminator. + mode := "lpuOnly" + if intent.Pipeline == PipelineLPX { + mode = "strictHybrid" + } + transcript.field("workload-mode", []byte(mode)) + + transcript.field("model", []byte(model)) + transcript.field("model-settings", intent.ModelSettings) + } + return transcripts +} + +// bindHybridRuntimeIO records the shared Cyborg I/O contract in a model projection digest. +func bindHybridRuntimeIO( + transcript *digestTranscript, + pipeline Pipeline, + ioFPGACount int32, + ioFanoutFactor int32, +) { + if pipeline != PipelineLPX { + return + } + + transcript.intField("io-fpga-count", int64(ioFPGACount)) + if ioFanoutFactor > 1 { + transcript.intField("io-fanout-factor", int64(ioFanoutFactor)) + } +} + +type digestTranscript struct { + hash hash.Hash +} + +func (t *digestTranscript) field(tag string, value []byte) { + writeLengthDelimited(t.hash, []byte(tag)) + writeLengthDelimited(t.hash, value) +} + +func (t *digestTranscript) uint32Field(tag string, value uint32) { + var encoded [4]byte + binary.BigEndian.PutUint32(encoded[:], value) + t.field(tag, encoded[:]) +} + +func (t *digestTranscript) intField(tag string, value int64) { + var encoded [8]byte + binary.BigEndian.PutUint64(encoded[:], uint64(value)) + t.field(tag, encoded[:]) +} + +func (t *digestTranscript) sum() WorkloadDigest { + var digest WorkloadDigest + copy(digest[:], t.hash.Sum(nil)) + return digest +} + +func writeLengthDelimited(writer hash.Hash, value []byte) { + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(value))) + _, _ = writer.Write(length[:]) + _, _ = writer.Write(value) +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go new file mode 100644 index 000000000000..308369177833 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "strings" + "testing" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestValidateRequestSizeMatchesLPXBoundary(t *testing.T) { + t.Parallel() + + t.Log("Build a request close enough to the limit that valid annotation padding can reach the exact boundary") + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "request", + Namespace: "namespace", + Annotations: map[string]string{"scheduling.lpu.nvidia.com/size-proof": ""}, + }, + Spec: lpxv1alpha1.LPUPipelineRequestSpec{ + AllocationMetadata: runtime.RawExtension{Raw: []byte(`{}`)}, + ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, + NodeLocal: &lpxv1alpha1.NodeLocalRequest{Model: "default"}, + Partitions: []lpxv1alpha1.PartitionRequest{{ + ID: "partition-000", Ordinal: 0, CompilerPartitionID: 0, + Extent: &[]int64{16, 1, 1, 1}, + }}, + PodGangRef: lpxv1alpha1.NamespacedName{Name: "agents"}, + TargetFamily: lpxv1alpha1.TargetFamilyHx16x8x2x3, + WorkloadMode: lpxv1alpha1.WorkloadModeV3HxLPUOnly, + }, + } + connections := make([]lpxv1alpha1.HxLogicalConnection, 256) + for index := range connections { + connections[index] = lpxv1alpha1.HxLogicalConnection{ + FromLogicalDevice: int64(index / 16), + ToLogicalDevice: int64(index % 16), + } + } + lanes := []int64{4, 2, 1} + baseBytes := 0 + for len(request.Spec.PropSyncConnectors) < maxLPXPartitions-1 { + baseBytes = requestWithRequirementEchoBytes(request) + if baseBytes >= maxLPXPlannedRequestObjectBytes-200*1024 { + break + } + request.Spec.PropSyncConnectors = append( + request.Spec.PropSyncConnectors, + lpxv1alpha1.PropSyncConnectorRequest{ + FromPartitionID: "partition-000", + ToPartitionID: "partition-000", + Requirement: lpxv1alpha1.PropSyncConnectorRequirement{ + Kind: lpxv1alpha1.PropSyncConnectorKindHxPropSyncV1, + Connections: &connections, + AcceptableLaneMultiplicities: &lanes, + }, + }, + ) + } + + t.Log("Pad the request annotation to the exact inclusive scheduler budget") + baseBytes = requestWithRequirementEchoBytes(request) + require.Less(t, baseBytes, maxLPXPlannedRequestObjectBytes) + padding := maxLPXPlannedRequestObjectBytes - baseBytes + require.LessOrEqual(t, padding, 200*1024) + request.Annotations["scheduling.lpu.nvidia.com/size-proof"] = strings.Repeat("x", padding) + + t.Log("Accept equality and reject the first byte beyond LPX's inclusive limit") + require.NoError(t, ValidateRequestSize(request)) + request.Annotations["scheduling.lpu.nvidia.com/size-proof"] += "x" + require.ErrorContains(t, ValidateRequestSize(request), "maximum") +} + +func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { + t.Parallel() + + t.Log("Acquire equal compiler contents under distinct local build paths") + first := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + second := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + require.NotEqual(t, first.build.Path, second.build.Path) + require.Equal(t, first.contentID, second.contentID) + + t.Log("Project either immutable snapshot through the same intent") + firstProjection := projectTestBuild(t, first, PipelineSingle, `{"prop_sync":true}`) + secondProjection := projectTestBuild(t, second, PipelineSingle, `{"prop_sync":true}`) + + t.Log("Produce the same projection digest independent of build locator") + require.Equal(t, firstProjection.Digest(), secondProjection.Digest()) +} + +func TestCanonicalModelSettingsRetainRawAndNormalizeRuntimeNumbers(t *testing.T) { + t.Log("Decode canonical model settings while retaining their exact raw bytes") + raw, object := canonicalModelSettings(json.RawMessage( + `{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`, + )) + require.Equal(t, json.RawMessage(`{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`), raw) + require.Equal(t, json.Number("8192"), object["integer"]) + + t.Log("Normalize representable integers while retaining floating-point runtime values") + merged, err := mergeRuntimeSettingOverride(nil, object, "settings") + require.NoError(t, err) + normalized := merged.(map[string]any) + require.Equal(t, int64(8192), normalized["integer"]) + require.Equal(t, []any{int64(128001), float64(1.5), float64(9223372036854775808)}, normalized["nested"].(map[string]any)["array"]) + + t.Log("Normalize explicit null settings to a present empty object") + raw, object = canonicalModelSettings(json.RawMessage("null")) + require.Nil(t, raw) + require.NotNil(t, object) + require.Empty(t, object) +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go new file mode 100644 index 000000000000..33194e2175e6 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -0,0 +1,238 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" +) + +const ( + v2ProjectionVersion = "v2-xt-node-local/v1" + maxPropSyncMultipleLoadSetsOffset = int64(3) +) + +//nolint:gocyclo // V2 projection validates one complete transformation. +func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInput, modelSettings map[string]any) ([]*ModelProjection, error) { + configured := *intent.BuildSnapshot.build + usesResolvedRuntime := intent.Pipeline == PipelineSingle || + intent.Pipeline == PipelineSpecDecode + build := &configured + ioFPGACount, ioFanoutFactor := build.IOFPGACount, build.IOFanoutFactor + propSyncEnabled := build.CompilationMode == BuildCompilationModeLPUOnly && len(build.SelectedPropSyncChains) > 0 + connectorBuild := intent.BuildSnapshot.build + if usesResolvedRuntime { + if err := resolveBuildSettings(&configured, modelSettings); err != nil { + return nil, fmt.Errorf("resolving configured V2 build: %w", err) + } + } + if value, present := modelSettings["prop_sync"]; present { + enabled, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("model settings.prop_sync must be a boolean") + } + propSyncEnabled = enabled + } + maxInterPartitionOffset := int64(0) + if value, present := modelSettings["prop_sync_multiple_load_sets"]; present { + multiple, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("model settings.prop_sync_multiple_load_sets must be a boolean") + } + if multiple { + maxInterPartitionOffset = maxPropSyncMultipleLoadSetsOffset + } + } + + // Share legacy-key handling without consuming the LPX build's partitions or selected chains. + if !usesResolvedRuntime { + if err := normalizeLegacyNovaSettings(modelSettings); err != nil { + return nil, fmt.Errorf("resolving configured V2 build: %w", err) + } + configured.runtimeSettings = modelSettings + } + allocationMetadata := json.RawMessage(`{}`) + + // Hybrid embeddings always come from Cyborg, after validating authored settings. + if !usesResolvedRuntime { + if _, err := boolSetting(modelSettings, "cpu_embeddings"); err != nil { + return nil, fmt.Errorf("model settings.%w", err) + } + configured.omitStandaloneEmbeddingPartition() + } + + // Bind the V2 workload and runtime contract into projection identity before partition validation. + transcripts := newModelProjectionTranscripts(intent, v2ProjectionVersion) + for index := range transcripts { + transcript := &transcripts[index] + transcript.field("input-embeddings-on-gpu", []byte{1}) + if propSyncEnabled { + transcript.field("effective-prop-sync", []byte{1}) + } else { + transcript.field("effective-prop-sync", []byte{0}) + } + transcript.intField("max-inter-partition-offset", maxInterPartitionOffset) + bindHybridRuntimeIO(transcript, intent.Pipeline, ioFPGACount, ioFanoutFactor) + } + + // Bind validated physical partitions into projection identity while counting runtime endpoints. + partitions := configured.Partitions + agentReplicas := 0 + for index, partition := range partitions { + compilerID := uint32(partition.SourcePartitionID) + shape, endpoints, shapeErr := xtShape(partition.Topology.ChipCount) + if shapeErr != nil { + return nil, fmt.Errorf("V2 compiler partition %d: %w", compilerID, shapeErr) + } + agentReplicas += int(endpoints) + for modelIndex := range transcripts { + transcripts[modelIndex].uint32Field("compiler-partition-id", compilerID) + transcripts[modelIndex].uint32Field("model-partition-id", uint32(index)) + transcripts[modelIndex].intField("endpoint-count", endpoints) + transcripts[modelIndex].field("xt-shape", []byte(shape)) + } + } + connectors, err := v2Connectors(connectorBuild, partitions, propSyncEnabled, maxInterPartitionOffset) + if err != nil { + return nil, err + } + // Preserve physical scheduler partitions while collapsing selected chains only in LPU runtime state. + if intent.Pipeline == PipelineLPX && len(configured.SelectedPropSyncChains) != 0 { + omittedSourceZero := len(partitions) < len(connectorBuild.Partitions) + runtimeChainByRoot := make(map[int][]int, len(connectorBuild.SelectedPropSyncChains)) + for _, chain := range connectorBuild.SelectedPropSyncChains { + runtimeChain := chain + if omittedSourceZero && runtimeChain[0] == 0 { + runtimeChain = runtimeChain[1:] + } + if len(runtimeChain) >= 2 { + runtimeChainByRoot[runtimeChain[0]] = runtimeChain + } + } + collapsed := make([]BuildPartition, 0, len(partitions)) + for partitionIndex := 0; partitionIndex < len(partitions); { + partition := partitions[partitionIndex] + chain, selected := runtimeChainByRoot[partition.SourcePartitionID] + if !selected { + collapsed = append(collapsed, partition) + partitionIndex++ + continue + } + chainEnd := partitionIndex + len(chain) + chainPartition, collapseErr := collapseSelectedPropSyncChain( + chain, + partitions[partitionIndex:chainEnd], + ) + if collapseErr != nil { + return nil, fmt.Errorf("configuring V2 LPU runtime partitions: %w", collapseErr) + } + collapsed = append(collapsed, chainPartition) + partitionIndex = chainEnd + } + configured.Partitions = collapsed + configured.SelectedPropSyncChains = nil + } + + // Encode shared connectors once without changing any model's digest field order. + for _, connector := range connectors { + encoded, _ := json.Marshal(connector) + for index := range transcripts { + transcripts[index].field("connector", encoded) + } + } + + // Publish distinct logical identities backed by the component's immutable configuration. + for index := range transcripts { + transcript := &transcripts[index] + transcript.field("allocation-metadata", allocationMetadata) + + dst = append(dst, &ModelProjection{ + digest: transcript.sum(), + runtimeBuildRef: intent.RuntimeBuildRef, + model: intent.Models[index], + pipeline: intent.Pipeline, + configuredBuild: configured, + allocationMetadata: allocationMetadata, + partitions: partitions, + connectors: connectors, + agentReplicas: agentReplicas, + }) + } + return dst, nil +} + +func xtShape(chipCount int) (lpxv1alpha1.Xt8888PartitionShape, int64, error) { + // Reserve one whole physical host for compiler partitions that use fewer than eight chips. + if chipCount > 0 && chipCount < 8 { + return lpxv1alpha1.Xt8888PartitionShapeC8, 1, nil + } + + // Reject partial and unregistered whole-host shapes before deriving their LPX names. + if chipCount < 8 || chipCount%8 != 0 || (chipCount > 64 && chipCount != 96 && chipCount != 128) { + return "", 0, fmt.Errorf("chip count %d is not a registered XT8888 partition shape", chipCount) + } + return lpxv1alpha1.Xt8888PartitionShape(fmt.Sprintf("c%d", chipCount)), int64(chipCount / 8), nil +} + +// v2Connectors requires a normalized nonnil build and a nonempty contiguous +// interval of its physical partitions. Runtime chain collapse happens afterward. +func v2Connectors( + build *Build, + partitions []BuildPartition, + propSyncEnabled bool, + maxOffset int64, +) ([]lpxv1alpha1.PropSyncConnectorRequest, error) { + // Require compiler-authored relationships whenever prop-sync is enabled. + if len(build.SelectedPropSyncChains) == 0 { + if !propSyncEnabled { + return []lpxv1alpha1.PropSyncConnectorRequest{}, nil + } + return nil, fmt.Errorf("manifest v2 settings.prop_sync=true requires a selected prop-sync chain") + } + + // Validate explicit chains before ordering their scheduler edges. + edgePositions, err := validateSelectedPropSyncGraph( + build.Partitions, + build.SelectedPropSyncChains, + "selected prop-sync chain", + true, + ) + if err != nil { + return nil, err + } + + // Scheduler output follows physical order, not chain declaration order. + slices.Sort(edgePositions) + + // Rebase selected physical edges into the retained partition interval. + start := sort.Search(len(build.Partitions), func(index int) bool { + return build.Partitions[index].SourcePartitionID >= partitions[0].SourcePartitionID + }) + connectors := make([]lpxv1alpha1.PropSyncConnectorRequest, 0, len(edgePositions)) + for _, position := range edgePositions { + i := position - start + if i < 0 { + continue + } + if i+1 >= len(partitions) { + break + } + offset := maxOffset + connectors = append(connectors, lpxv1alpha1.PropSyncConnectorRequest{ + FromPartitionID: fmt.Sprintf("partition-%03d", i), + ToPartitionID: fmt.Sprintf("partition-%03d", i+1), + Requirement: lpxv1alpha1.PropSyncConnectorRequirement{ + Kind: lpxv1alpha1.PropSyncConnectorKindXt8888Gap, + MaxInterPartitionOffset: &offset, + }, + }) + } + return connectors, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go new file mode 100644 index 000000000000..94dc3630267b --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -0,0 +1,358 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "fmt" + "math" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "github.com/stretchr/testify/require" +) + +func TestProjectModelV2SubHostPartitionUsesWholeHostShape(t *testing.T) { + t.Log("Create an immutable V2 compiler build with one two-chip physical partition") + topology := "URSA_V2__Q8__2C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA" + fixture := newV2CompilerFixture() + fixture.numLPUNodes = 1 + fixture.selectedPropSyncChains = nil + fixture.partitions = []testV3CapnpPartition{{ + id: 1, deviceType: manifestcapnp.DeviceType_lpu, topology: topology, numChips: 2, devicesPerNode: 8, + }} + buildDir := writeCompilerFixture(t, fixture) + snapshot := acquireTestSnapshot(t, buildDir) + + t.Log("Project the sub-host build through the selected single-pipeline LPX path") + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, + Models: []string{"default"}, + BuildSnapshot: normalizeTestSnapshot(t, snapshot), + }) + require.NoError(t, err) + projection := projectionBatch[0] + + t.Log("Verify the two-chip compiler partition reserves one C8 host and produces one Agent replica") + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.Partitions, 1) + require.Equal(t, lpxv1alpha1.Xt8888PartitionShapeC8, *spec.Partitions[0].XtShape) + require.NotNil(t, spec.NodeLocal) + require.Len(t, spec.NodeLocal.PartitionMappings, 1) + require.Equal(t, 1, projection.agentReplicas) +} + +func TestProjectModelV2ValidatesPhysicalPartitionsInOrder(t *testing.T) { + t.Log("Create a V2 build whose first physical partition has an invalid shape") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.CompilationMode = BuildCompilationModeHybrid + build.SelectedPropSyncChains = [][]int{{100, 101}} + build.Partitions[0].Topology = Topology{ChipCount: 9} + build.Partitions[0].SourcePartitionID = 1 + + t.Log("Project the invalid physical partition") + _, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, Models: []string{"default"}, + BuildSnapshot: normalized, + }) + + t.Log("Reject the invalid shape at the first applicable validation boundary") + require.EqualError(t, err, "V2 compiler partition 1: chip count 9 is not a registered XT8888 partition shape") +} + +func TestXTShape(t *testing.T) { + t.Log("Check every registered shape and every integer gap through the largest shape") + registeredShapes := map[int]lpxv1alpha1.Xt8888PartitionShape{ + 8: lpxv1alpha1.Xt8888PartitionShapeC8, 16: lpxv1alpha1.Xt8888PartitionShapeC16, + 24: lpxv1alpha1.Xt8888PartitionShapeC24, 32: lpxv1alpha1.Xt8888PartitionShapeC32, + 40: lpxv1alpha1.Xt8888PartitionShapeC40, 48: lpxv1alpha1.Xt8888PartitionShapeC48, + 56: lpxv1alpha1.Xt8888PartitionShapeC56, 64: lpxv1alpha1.Xt8888PartitionShapeC64, + 96: lpxv1alpha1.Xt8888PartitionShapeC96, 128: lpxv1alpha1.Xt8888PartitionShapeC128, + } + for chipCount := -1; chipCount <= 129; chipCount++ { + roundedChipCount := max(chipCount, 8) + wantShape, registered := registeredShapes[roundedChipCount] + registered = registered && chipCount > 0 + shape, endpoints, err := xtShape(chipCount) + if !registered { + require.Error(t, err, chipCount) + continue + } + require.NoError(t, err, chipCount) + require.Equal(t, wantShape, shape, chipCount) + require.Equal(t, int64(roundedChipCount/8), endpoints, chipCount) + } + + t.Log("Reject integer extremes without indexing outside the registry") + for _, chipCount := range []int{math.MinInt, math.MaxInt} { + _, _, err := xtShape(chipCount) + require.Error(t, err, chipCount) + } +} + +func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { + t.Parallel() + + t.Log("Acquire a V2 build with compiler-selected prop-sync metadata") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) + + t.Log("Preserve both physical partitions and their selected connector without model settings") + defaultedBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalized, + }) + require.NoError(t, err) + defaulted := defaultedBatch[0] + defaultSpec := defaulted.RequestSpec("test", "agents", nil) + require.Len(t, defaultSpec.Partitions, 2) + require.Len(t, defaultSpec.PropSyncConnectors, 1) + require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) + + t.Log("Enable prop sync and project its effective connector offset") + projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":true,"prop_sync_multiple_load_sets":true}`) + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.PropSyncConnectors, 1) + require.Equal(t, int64(3), *spec.PropSyncConnectors[0].Requirement.MaxInterPartitionOffset) + + t.Log("Preserve the compiler-selected chain when model settings disable implicit prop sync") + intent := ModelProjectionInput{ + Pipeline: PipelineSingle, + Models: []string{"default"}, + BuildSnapshot: normalized, + ModelSettings: json.RawMessage(`{"prop_sync":false}`), + } + disabledBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + disabled := disabledBatch[0] + require.Len(t, disabled.RequestSpec("test", "agents", nil).PropSyncConnectors, 1) + + t.Log("Require manifest-selected evidence when revision 2 enables prop sync") + build.SelectedPropSyncChains = nil + _, err = appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, + BuildSnapshot: normalized, + ModelSettings: json.RawMessage(`{"prop_sync":true}`), + }) + require.ErrorContains(t, err, "manifest v2 settings.prop_sync=true requires a selected prop-sync chain") +} + +func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { + t.Log("Prepare a V2 build with a standalone embedding partition in its selected chain") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.Partitions[0].SourcePartitionID = 0 + build.Partitions[1].SourcePartitionID = 1 + build.StandaloneTokenEmbeddings = true + build.SupportsCPUEmbeddings = true + build.SelectedPropSyncChains = [][]int{{0, 1}} + + t.Log("Omit source partition zero when CPU embeddings are enabled") + offLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":true}`) + require.Equal(t, int64(1), offLPU.RequestSpec("test", "agents", nil).Partitions[0].CompilerPartitionID) + + t.Log("Retain source partition zero when CPU embeddings are disabled") + onLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":false}`) + require.Equal(t, 4, onLPU.agentReplicas) + require.Equal(t, int64(0), onLPU.RequestSpec("test", "agents", nil).Partitions[0].CompilerPartitionID) +} + +func TestProjectModelV2StrictHybridForcesCPUEmbeddingsAfterValidation(t *testing.T) { + t.Log("Prepare a hybrid V2 build with a standalone embedding partition") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.CompilationMode = BuildCompilationModeHybrid + build.Partitions[0].SourcePartitionID = 0 + build.Partitions[1].SourcePartitionID = 1 + build.StandaloneTokenEmbeddings = true + build.SupportsCPUEmbeddings = true + build.SelectedPropSyncChains = nil + + t.Log("Force strict-hybrid CPU embeddings after accepting an explicit false setting") + projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, + Models: []string{"default"}, + BuildSnapshot: normalized, + ModelSettings: json.RawMessage(`{"cpu_embeddings":false}`), + }) + require.NoError(t, err) + projection := projectionBatch[0] + spec := projection.RequestSpec("test", "agents", nil) + require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) + require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) + + t.Log("Reject a malformed setting before applying the strict-hybrid override") + _, err = appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, + Models: []string{"default"}, + BuildSnapshot: normalized, + ModelSettings: json.RawMessage(`{"cpu_embeddings":"false"}`), + }) + require.EqualError(t, err, "model settings.cpu_embeddings must be a boolean") +} + +func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { + t.Parallel() + + t.Log("Prepare a selected adjacent chain between unselected physical partitions") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + prefix, suffix := build.Partitions[0], build.Partitions[1] + prefix.SourcePartitionID, prefix.PartPath = 3, "part-3" + suffix.SourcePartitionID, suffix.PartPath = 11, "part-11" + build.Partitions = append([]BuildPartition{prefix}, append(build.Partitions, suffix)...) + build.SelectedPropSyncChains = [][]int{{7, 8}} + + t.Log("Project with global prop sync disabled") + projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":false}`) + + t.Log("Preserve only the source-selected connector and partition identities") + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.Partitions, 2) + require.Len(t, spec.PropSyncConnectors, 1) + require.Equal(t, int64(7), spec.Partitions[0].CompilerPartitionID) + require.Equal(t, int64(8), spec.Partitions[1].CompilerPartitionID) + require.Equal(t, spec.Partitions[0].ID, spec.PropSyncConnectors[0].FromPartitionID) + require.Equal(t, spec.Partitions[1].ID, spec.PropSyncConnectors[0].ToPartitionID) + + t.Log("Reject a selected XT edge between valid but incompatible physical topologies") + topology, err := parse("URSA_V2__Q8__16C__G_97_25__KP_FEC__GHZ_1_0__NO_FPGA") + require.NoError(t, err) + build.Partitions[2].Topology = topology + _, err = appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalized, + ModelSettings: json.RawMessage(`{"prop_sync":false}`), + }) + require.ErrorContains(t, err, "has incompatible topology at partition ID 8") +} + +func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing.T) { + t.Parallel() + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.CompilationMode = BuildCompilationModeHybrid + + t.Log("Use individually schedulable physical shapes whose collapsed runtime total is not a scheduler shape") + firstTopology, err := build.Partitions[0].Topology.withChipCount(8) + require.NoError(t, err) + secondTopology, err := build.Partitions[1].Topology.withChipCount(64) + require.NoError(t, err) + build.Partitions[0].Topology = firstTopology + build.Partitions[1].Topology = secondTopology + third := build.Partitions[1] + third.SourcePartitionID = 11 + third.PartPath = "part-11" + build.Partitions = append(build.Partitions, third) + build.SelectedPropSyncChains = [][]int{{7, 8}} + + t.Log("Project the selected chain beside an independent third partition") + projection := projectTestBuild(t, normalized, PipelineLPX, "") + + t.Log("Project all physical scheduler partitions and only the selected connector") + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.Partitions, 3) + require.Len(t, spec.PropSyncConnectors, 1) + require.Equal(t, spec.Partitions[0].ID, spec.PropSyncConnectors[0].FromPartitionID) + require.Equal(t, spec.Partitions[1].ID, spec.PropSyncConnectors[0].ToPartitionID) + + t.Log("Collapse only the Agent runtime projection of the selected chain") + data := resolvedPartitionData([]*ModelProjection{projection}) + require.Equal(t, "7\n11", data["partition_ids"]) + require.Equal(t, "9\n8", data["nodes_per_partition"]) + require.Equal(t, "0\n1", data["partition_indices"]) + require.Equal(t, "0\n9", data["partition_node_offsets"]) + require.Equal(t, "part-7\npart-11", data["partition_paths"]) + require.Equal(t, projection.configuredBuild.Partitions[0].Topology.Raw+"\n"+projection.configuredBuild.Partitions[1].Topology.Raw, data["topologies"]) + require.Empty(t, projection.configuredBuild.SelectedPropSyncChains) + require.Equal(t, 17, projection.agentReplicas) + require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) + + t.Log("Preserve physical output order when selected chains are declared in reverse") + fourth := build.Partitions[2] + fourth.SourcePartitionID = 13 + build.Partitions = append(build.Partitions, fourth) + build.SelectedPropSyncChains = [][]int{{11, 13}, {7, 8}} + connectors, err := v2Connectors(build, build.Partitions, true, maxPropSyncMultipleLoadSetsOffset) + require.NoError(t, err) + require.Len(t, connectors, 2) + require.Equal(t, "partition-000", connectors[0].FromPartitionID) + require.Equal(t, "partition-001", connectors[0].ToPartitionID) + require.Equal(t, "partition-002", connectors[1].FromPartitionID) + require.Equal(t, "partition-003", connectors[1].ToPartitionID) + require.NotSame(t, connectors[0].Requirement.MaxInterPartitionOffset, connectors[1].Requirement.MaxInterPartitionOffset) +} + +func TestProjectModelV2CollapsesSelectedChainAfterOmittingEmbeddingPrefix(t *testing.T) { + t.Parallel() + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.CompilationMode = BuildCompilationModeHybrid + + t.Log("Model the normalized adjacent 0/1/2 chain whose standalone embedding root is omitted") + topology, err := build.Partitions[0].Topology.withChipCount(8) + require.NoError(t, err) + partitions := make([]BuildPartition, 3) + for sourceID := range partitions { + partitions[sourceID] = build.Partitions[0] + partitions[sourceID].SourcePartitionID = sourceID + partitions[sourceID].PartPath = fmt.Sprintf("part-%d", sourceID) + partitions[sourceID].Topology = topology + } + build.Partitions = partitions + build.SelectedPropSyncChains = [][]int{{0, 1, 2}} + build.StandaloneTokenEmbeddings = true + build.SupportsCPUEmbeddings = true + + t.Log("Project after omitting the standalone embedding prefix") + projection := projectTestBuild(t, normalized, PipelineLPX, "") + + t.Log("Keep physical scheduler partitions 1/2 while collapsing their Agent runtime projection") + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.Partitions, 2) + require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) + require.Equal(t, int64(2), spec.Partitions[1].CompilerPartitionID) + require.Len(t, spec.PropSyncConnectors, 1) + require.Equal(t, 2, projection.agentReplicas) + data := resolvedPartitionData([]*ModelProjection{projection}) + require.Equal(t, "1", data["partition_ids"]) + require.Equal(t, "2", data["nodes_per_partition"]) + require.Equal(t, "part-1", data["partition_paths"]) + require.Equal(t, 16, projection.configuredBuild.Partitions[0].Topology.ChipCount) + require.Empty(t, projection.configuredBuild.SelectedPropSyncChains) + require.Equal(t, [][]int{{0, 1, 2}}, build.SelectedPropSyncChains) +} + +func TestProjectModelV2PreservesAgentReplicasWhenCollapsingSubHostPartitions(t *testing.T) { + t.Parallel() + + t.Log("Prepare two selected V2 sub-host partitions") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.CompilationMode = BuildCompilationModeHybrid + + for index := range build.Partitions { + topology, err := build.Partitions[index].Topology.withChipCount(2) + require.NoError(t, err) + build.Partitions[index].Topology = topology + } + build.SelectedPropSyncChains = [][]int{{7, 8}} + + t.Log("Project and collapse the selected runtime chain") + projection := projectTestBuild(t, normalized, PipelineLPX, "") + + t.Log("Preserve both physical scheduler partitions and Agent replicas") + require.Len(t, projection.RequestSpec("test", "agents", nil).Partitions, 2) + require.Equal(t, 2, projection.agentReplicas) + + t.Log("Keep both physical Agent endpoints in the collapsed C4 runtime partition") + data := resolvedPartitionData([]*ModelProjection{projection}) + require.Equal(t, "2", data["nodes_per_partition"]) + require.Equal(t, "0", data["partition_node_offsets"]) + require.Equal(t, 2, projection.configuredBuild.Partitions[0].effectiveNodeCount()) + require.Equal(t, 4, projection.configuredBuild.Partitions[0].Topology.ChipCount) +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go new file mode 100644 index 000000000000..0e5d22878085 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "fmt" + "strconv" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" +) + +const ( + v3CompilerEnvelopeSchema = "dynamo.lpx.v3-capnp/v1" + v3ProjectionVersion = "v3-hx-capnp/v1" + v3MaxSWADKVCBlocksDraft = "max_swa_dkvc_blocks_draft" + v3LPUDevice = "lpu" + v3HXLogicalDeviceCount = 16 +) + +func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInput, modelSettings map[string]any) ([]*ModelProjection, error) { + runtimeBuild := *intent.BuildSnapshot.build + manifestPartitions := runtimeBuild.Partitions + selectedPropSyncChains := runtimeBuild.SelectedPropSyncChains + + ioFPGACount, ioFanoutFactor := runtimeBuild.IOFPGACount, runtimeBuild.IOFanoutFactor + // V3 topology and selected prop-sync chains are projected below. Do not feed + // them through the runtime-settings consumer: the runtime projection + // intentionally contains no artifact partitions. + runtimeBuild.Partitions = nil + runtimeBuild.SelectedPropSyncChains = nil + if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { + return nil, fmt.Errorf("resolving configured V3 runtime build: %w", err) + } + propSyncEnabled, ok := runtimeBuild.runtimeSettings["prop_sync"].(bool) + if !ok { + return nil, fmt.Errorf("configured V3 runtime settings.prop_sync must be a boolean") + } + if runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly { + if len(selectedPropSyncChains) > 0 && !propSyncEnabled { + return nil, fmt.Errorf("V3 manifest-selected prop-sync chains require settings.prop_sync=true") + } + if len(selectedPropSyncChains) == 0 && propSyncEnabled { + return nil, fmt.Errorf("V3 settings.prop_sync=true requires a manifest-selected prop-sync chain") + } + } + + allocationMetadata, connectors, err := projectV3PropSync(manifestPartitions, selectedPropSyncChains, intent.Pipeline) + if err != nil { + return nil, err + } + + // Initialize independent model hashes after validating the shared component geometry. + transcripts := newModelProjectionTranscripts(intent, v3ProjectionVersion) + for index := range transcripts { + transcripts[index].field("v3-envelope-schema", []byte(v3CompilerEnvelopeSchema)) + } + + // Count runtime endpoints while binding ordered partitions into projection identity. + agentReplicas := 0 + for index, partition := range manifestPartitions { + agentReplicas += int(partition.HXExtent[1] * partition.HXExtent[2] * partition.HXExtent[3]) + for modelIndex := range transcripts { + transcripts[modelIndex].intField("ordered-compiler-id-index", int64(index)) + transcripts[modelIndex].uint32Field("ordered-compiler-id", uint32(partition.SourcePartitionID)) + } + } + + // Publish distinct logical identities backed by the component's immutable configuration. + for index := range transcripts { + transcript := &transcripts[index] + transcript.field("allocation-metadata", allocationMetadata) + // Bind the Cyborg runtime contract into hybrid projection identity. + bindHybridRuntimeIO(transcript, intent.Pipeline, ioFPGACount, ioFanoutFactor) + + dst = append(dst, &ModelProjection{ + digest: transcript.sum(), + runtimeBuildRef: intent.RuntimeBuildRef, + model: intent.Models[index], + pipeline: intent.Pipeline, + configuredBuild: runtimeBuild, + allocationMetadata: allocationMetadata, + partitions: manifestPartitions, + connectors: connectors, + agentReplicas: agentReplicas, + }) + } + return dst, nil +} + +func projectV3PropSync( + partitions []BuildPartition, + chains [][]int, + pipeline Pipeline, +) (json.RawMessage, []lpxv1alpha1.PropSyncConnectorRequest, error) { + edgePositions, err := validateSelectedPropSyncGraph(partitions, chains, "selected V3 prop-sync chain", false) + if err != nil { + return nil, nil, err + } + + // Project each physical partition into the V3 allocation metadata envelope. + partitionInfo := make(map[string]any, len(partitions)+1) + partitionInfo["num_partitions"] = len(partitions) + for _, partition := range partitions { + compilerID := uint32(partition.SourcePartitionID) + partitionInfo[strconv.FormatUint(uint64(compilerID), 10)] = map[string]any{ + "device": v3LPUDevice, + "allocation": partition.HXExtent, + } + } + + // Project validated edges into runtime metadata and scheduler connector order. + propSyncPairs := make([]any, 0) + connectors := make([]lpxv1alpha1.PropSyncConnectorRequest, 0, len(edgePositions)) + for _, fromPosition := range edgePositions { + source := partitions[fromPosition] + destinationID := partitions[fromPosition+1].SourcePartitionID + endpointCount := source.HXExtent[1] * source.HXExtent[2] * source.HXExtent[3] + logicalConnections := make([]lpxv1alpha1.HxLogicalConnection, v3HXLogicalDeviceCount) + connections := make([][2]int64, v3HXLogicalDeviceCount) + sourceOffset := source.HXExtent[0]*endpointCount - v3HXLogicalDeviceCount + for logicalDevice := range logicalConnections { + from := sourceOffset + int64(logicalDevice) + logicalConnections[logicalDevice] = lpxv1alpha1.HxLogicalConnection{ + FromLogicalDevice: from, + ToLogicalDevice: int64(logicalDevice), + } + connections[logicalDevice] = [2]int64{from, int64(logicalDevice)} + } + acceptableLaneMultiplicities := []int64{4, 2, 1} + propSyncPairs = append(propSyncPairs, map[string]any{ + "source_partition": source.SourcePartitionID, + "dest_partition": destinationID, + "connections": connections, + "num_supported_lanes": acceptableLaneMultiplicities, + }) + connectors = append(connectors, lpxv1alpha1.PropSyncConnectorRequest{ + FromPartitionID: fmt.Sprintf("partition-%03d", fromPosition), + ToPartitionID: fmt.Sprintf("partition-%03d", fromPosition+1), + Requirement: lpxv1alpha1.PropSyncConnectorRequirement{ + Kind: lpxv1alpha1.PropSyncConnectorKindHxPropSyncV1, + Connections: &logicalConnections, + AcceptableLaneMultiplicities: &acceptableLaneMultiplicities, + }, + }) + } + + // Encode the same validated graph for the LPU runtime allocation contract. + metadata, _ := json.Marshal(map[string]any{ + "arch": "lp30", + "topology": "lyra", + "metadata_version": 1, + "partition_info": partitionInfo, + "prop_sync_info": map[string]any{ + "version": 1, + "prop_sync_pairs": propSyncPairs, + }, + }) + if pipeline != PipelineLPX && len(connectors) != len(partitions)-1 { + return nil, nil, fmt.Errorf("V3 LPU-only workloads require a complete adjacent prop-sync connector chain") + } + return metadata, connectors, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go new file mode 100644 index 000000000000..3fa16187e8bb --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "encoding/json" + "testing" + + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "github.com/stretchr/testify/require" +) + +func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling(t *testing.T) { + t.Log("Create one LPX manifest with two selected LPU partitions and one CUDA artifact") + fixture := newV3CompilerFixture() + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + fixture.partitions[0].topology = "another-opaque-v3-topology" + second := fixture.partitions[0] + second.id = 2 + fixture.partitions = append(fixture.partitions, second, testV3CapnpPartition{ + id: 3, deviceType: manifestcapnp.DeviceType_cuda, + }) + fixture.selectedPropSyncChains = [][]uint32{{1, 2}} + buildDir := writeCompilerFixture(t, fixture) + + t.Log("Project hybrid while keeping manifest-selected links independent of global prop sync") + intent := ModelProjectionInput{ + Pipeline: PipelineLPX, + Models: []string{"default"}, + BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, buildDir)), + } + projectionBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + projection := projectionBatch[0] + require.Equal(t, BuildCompilationModeHybrid, projection.configuredBuild.CompilationMode) + require.Equal(t, int64(8192), projection.configuredBuild.runtimeSettings["sequence_length"]) + require.Equal(t, false, projection.configuredBuild.runtimeSettings["prop_sync"]) + digest, err := workloadSetDigest([]*ModelProjection{projection}) + require.NoError(t, err) + require.Equal(t, projection.Digest(), digest, "one model is the aggregate digest base case") + + t.Log("Project only the two LPU artifacts and their selected chain into the hybrid scheduler request") + cyborgRef := &lpxv1alpha1.CyborgPodCliqueReference{Name: "decode", UID: "decode-uid"} + spec := projection.RequestSpec("test", "agents", cyborgRef) + require.Equal(t, lpxv1alpha1.WorkloadModeV3HxStrictHybrid, spec.WorkloadMode) + require.Len(t, spec.Partitions, 2) + require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) + require.Equal(t, int64(2), spec.Partitions[1].CompilerPartitionID) + require.Len(t, spec.PropSyncConnectors, 1) + require.Equal(t, spec.Partitions[0].ID, spec.PropSyncConnectors[0].FromPartitionID) + require.Equal(t, spec.Partitions[1].ID, spec.PropSyncConnectors[0].ToPartitionID) + + t.Log("Bind workload references while mapping compiler IDs to model ordinals") + require.Equal(t, "agents", spec.PodGangRef.Name) + require.Equal(t, "test", *spec.PodGangRef.Namespace) + require.Equal(t, "default", spec.NodeLocal.Model) + require.Equal(t, cyborgRef, spec.CyborgPodCliqueRef) + require.Len(t, spec.NodeLocal.PartitionMappings, 2) + for index, partition := range spec.Partitions { + require.Equal(t, int64(index), partition.Ordinal) + require.Equal(t, int64(index), spec.NodeLocal.PartitionMappings[index].ModelPartitionID) + require.Equal(t, partition.ID, spec.NodeLocal.PartitionMappings[index].PartitionID) + } + + t.Log("Mutating returned fields must not change the projection or caller-owned reference") + before := spec.DeepCopy() + (*spec.Partitions[0].Extent)[0] = 0 + (*spec.PropSyncConnectors[0].Requirement.Connections)[0].FromLogicalDevice = 99 + (*spec.PropSyncConnectors[0].Requirement.AcceptableLaneMultiplicities)[0] = 99 + spec.AllocationMetadata.Raw[0] = ' ' + spec.CyborgPodCliqueRef.Name = "changed" + require.Equal(t, before.CyborgPodCliqueRef, cyborgRef) + require.Equal(t, *before, projection.RequestSpec("test", "agents", cyborgRef)) + require.Nil(t, projection.RequestSpec("test", "agents", nil).CyborgPodCliqueRef) + + t.Log("Remove the manifest's selected chain without synthesizing hybrid connectors") + fixture.selectedPropSyncChains = nil + writeTestV3CapnpManifest(t, buildDir, fixture) + intent.BuildSnapshot = normalizeTestSnapshot(t, acquireTestSnapshot(t, buildDir)) + projectionBatch, err = appendModelProjections(nil, intent) + require.NoError(t, err) + projection = projectionBatch[0] + require.Empty(t, projection.RequestSpec("test", "agents", nil).PropSyncConnectors) +} + +func TestProjectModelV3RejectsInvalidPropSyncChains(t *testing.T) { + t.Log("Define invalid selected chains on otherwise valid HX builds") + tests := []struct { + name string + partitions int + numLPUNodes uint32 + chains [][]uint32 + wantErr string + }{ + { + name: "selected prop-sync chain references missing partition", + partitions: 1, numLPUNodes: 2, chains: [][]uint32{{1, 2}}, + wantErr: "references missing partition ID 2", + }, + { + name: "selected prop-sync chain is not forward-adjacent", + partitions: 3, numLPUNodes: 6, chains: [][]uint32{{1, 3}}, + wantErr: "is not forward-adjacent at partition ID 3", + }, + { + name: "missing edge evidence precedes overlap validation", + partitions: 2, numLPUNodes: 4, chains: [][]uint32{{1, 1}, {1, 3}}, + wantErr: "selected V3 prop-sync chain 1 references missing partition ID 3", + }, + { + name: "selected prop-sync chains overlap", + partitions: 3, numLPUNodes: 6, chains: [][]uint32{{1, 2}, {2, 3}}, + wantErr: "overlaps partition ID 2", + }, + { + name: "selected prop-sync chain repeats a partition", + partitions: 1, numLPUNodes: 2, chains: [][]uint32{{1, 1}}, + wantErr: "overlaps partition ID 1", + }, + { + name: "selected prop-sync chain is incomplete", + partitions: 3, numLPUNodes: 6, chains: [][]uint32{{1, 2}}, + wantErr: "complete adjacent prop-sync connector chain", + }, + { + name: "LPU-only partitions require a complete prop-sync chain", + partitions: 2, numLPUNodes: 2, + wantErr: "complete adjacent prop-sync connector chain", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Acquire and normalize the real manifest before testing its selected chain") + fixture := newV3CompilerFixture() + for id := 2; id <= test.partitions; id++ { + partition := fixture.partitions[0] + partition.id = uint32(id) + fixture.partitions = append(fixture.partitions, partition) + } + fixture.numLPUNodes = test.numLPUNodes + fixture.selectedPropSyncChains = test.chains + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + + t.Log("Reject the selected chain at the projector boundary") + _, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalized, + }) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func TestProjectModelV3ProjectsSelectedPropSyncChain(t *testing.T) { + t.Log("Create a two-partition V3 build with one selected adjacent prop-sync chain") + fixture := newV3CompilerFixture() + second := fixture.partitions[0] + second.id = 2 + fixture.partitions = append(fixture.partitions, second) + fixture.numLPUNodes = 4 + fixture.selectedPropSyncChains = [][]uint32{{1, 2}} + buildDir := writeCompilerFixture(t, fixture) + snapshot := acquireTestSnapshot(t, buildDir) + intent := ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalizeTestSnapshot(t, snapshot), + } + + t.Log("Project the selected chain through the LPU-only runtime") + projectionBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + projection := projectionBatch[0] + + t.Log("Project both partitions and one HX prop-sync connector") + spec := projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.Partitions, 2) + require.Len(t, spec.PropSyncConnectors, 1) + connector := spec.PropSyncConnectors[0] + require.Equal(t, spec.Partitions[0].ID, connector.FromPartitionID) + require.Equal(t, spec.Partitions[1].ID, connector.ToPartitionID) + require.Equal(t, lpxv1alpha1.PropSyncConnectorKindHxPropSyncV1, connector.Requirement.Kind) + require.NotNil(t, connector.Requirement.AcceptableLaneMultiplicities) + require.Equal(t, []int64{4, 2, 1}, *connector.Requirement.AcceptableLaneMultiplicities) + require.NotNil(t, connector.Requirement.Connections) + require.Len(t, *connector.Requirement.Connections, 16) + for logicalDevice, connection := range *connector.Requirement.Connections { + require.Equal(t, int64(logicalDevice), connection.FromLogicalDevice) + require.Equal(t, int64(logicalDevice), connection.ToLogicalDevice) + } + + t.Log("Encode the same connector topology in allocation metadata") + require.JSONEq(t, `{ + "arch":"lp30", + "topology":"lyra", + "metadata_version":1, + "partition_info":{ + "num_partitions":2, + "1":{"device":"lpu","allocation":[16,1,1,1]}, + "2":{"device":"lpu","allocation":[16,1,1,1]} + }, + "prop_sync_info":{"version":1,"prop_sync_pairs":[{ + "source_partition":1, + "dest_partition":2, + "connections":[[0,0],[1,1],[2,2],[3,3],[4,4],[5,5],[6,6],[7,7],[8,8],[9,9],[10,10],[11,11],[12,12],[13,13],[14,14],[15,15]], + "num_supported_lanes":[4,2,1] + }]} + }`, string(spec.AllocationMetadata.Raw)) + + t.Log("Reject model settings that disable a manifest-selected chain") + intent.ModelSettings = json.RawMessage(`{"prop_sync":false}`) + _, err = appendModelProjections(nil, intent) + require.ErrorContains(t, err, "manifest-selected prop-sync chains require settings.prop_sync=true") + + t.Log("Reject global prop sync without manifest-selected evidence in an LPU-only build") + fixture.selectedPropSyncChains = nil + intent.BuildSnapshot = normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + intent.ModelSettings = json.RawMessage(`{"prop_sync":true}`) + _, err = appendModelProjections(nil, intent) + require.ErrorContains(t, err, "settings.prop_sync=true requires a manifest-selected prop-sync chain") + + t.Log("Extend the native selected chain across three HX artifacts with distinct opaque topology names") + third := fixture.partitions[0] + third.id, third.topology = 3, "other-opaque-hx-topology" + fixture.partitions = append(fixture.partitions, third) + fixture.numLPUNodes = 6 + fixture.selectedPropSyncChains = [][]uint32{{1, 2, 3}} + writeTestV3CapnpManifest(t, buildDir, fixture) + intent.BuildSnapshot = normalizeTestSnapshot(t, acquireTestSnapshot(t, buildDir)) + intent.ModelSettings = nil + projectionBatch, err = appendModelProjections(nil, intent) + require.NoError(t, err) + projection = projectionBatch[0] + + t.Log("Advance each native scheduler connector to the next physical partition") + spec = projection.RequestSpec("test", "agents", nil) + require.Len(t, spec.PropSyncConnectors, 2) + require.Equal(t, "partition-000", spec.PropSyncConnectors[0].FromPartitionID) + require.Equal(t, "partition-001", spec.PropSyncConnectors[0].ToPartitionID) + require.Equal(t, "partition-001", spec.PropSyncConnectors[1].FromPartitionID) + require.Equal(t, "partition-002", spec.PropSyncConnectors[1].ToPartitionID) +} + +func TestProjectModelV3UsesMultiNodePropSyncBoundary(t *testing.T) { + t.Log("Create adjacent V3 partitions whose source boundary begins at logical device 16") + fixture := newV3CompilerFixture() + second := fixture.partitions[0] + second.id = 2 + fixture.partitions[0].topology = v3HXTopologyFamily + fixture.partitions[0].topologyFamily = v3HXTopologyFamily + fixture.partitions[0].partitionShape = []uint32{16, 2, 1, 1} + fixture.partitions[0].numChips = 32 + fixture.partitions = append(fixture.partitions, second) + fixture.numLPUNodes = 3 + fixture.selectedPropSyncChains = [][]uint32{{1, 2}} + buildDir := writeCompilerFixture(t, fixture) + snapshot := acquireTestSnapshot(t, buildDir) + + t.Log("Project the multi-node prop-sync chain") + projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineSingle, "") + + t.Log("Project logical connections from the source partition's final node") + spec := projection.RequestSpec("test", "agents", nil) + connections := *spec.PropSyncConnectors[0].Requirement.Connections + require.Equal(t, lpxv1alpha1.HxLogicalConnection{FromLogicalDevice: 16, ToLogicalDevice: 0}, connections[0]) + require.Equal(t, lpxv1alpha1.HxLogicalConnection{FromLogicalDevice: 31, ToLogicalDevice: 15}, connections[15]) + require.Contains(t, string(spec.AllocationMetadata.Raw), `"connections":[[16,0]`) +} + +func TestProjectModelV3UsesTopologyMetadataAndTracksManifestDigest(t *testing.T) { + t.Log("Create and project a one-node V3 topology-metadata manifest") + fixture := newV3CompilerFixture() + fixture.partitions[0].topology = v3HXTopologyFamily + fixture.partitions[0].topologyFamily = v3HXTopologyFamily + fixture.partitions[0].partitionShape = []uint32{16, 1, 1, 1} + fixture.numLPUNodes = 1 + buildDir := writeCompilerFixture(t, fixture) + firstSnapshot := acquireTestSnapshot(t, buildDir) + intent := ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalizeTestSnapshot(t, firstSnapshot), + } + firstBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + first := firstBatch[0] + + t.Log("Retain the acquired locator and exact single-partition allocation metadata") + require.Equal(t, "file://"+buildDir, intent.BuildSnapshot.build.Path) + require.Equal(t, intent.BuildSnapshot.build.Path, first.configuredBuild.Path) + firstSpec := first.RequestSpec("test", "agents", nil) + require.JSONEq(t, `{ + "arch":"lp30", + "topology":"lyra", + "metadata_version":1, + "partition_info":{ + "num_partitions":1, + "1":{"device":"lpu","allocation":[16,1,1,1]} + }, + "prop_sync_info":{"version":1,"prop_sync_pairs":[]} + }`, string(firstSpec.AllocationMetadata.Raw)) + + t.Log("Change the immutable manifest to a two-node topology and project again") + fixture.partitions[0].partitionShape = []uint32{16, 2, 1, 1} + fixture.partitions[0].numChips = 32 + fixture.numLPUNodes = 2 + writeTestV3CapnpManifest(t, buildDir, fixture) + secondSnapshot := acquireTestSnapshot(t, buildDir) + intent.BuildSnapshot = normalizeTestSnapshot(t, secondSnapshot) + secondBatch, err := appendModelProjections(nil, intent) + require.NoError(t, err) + second := secondBatch[0] + + t.Log("Track the immutable manifest change in snapshot and projection digests") + require.NotEqual(t, firstSnapshot.contentID, secondSnapshot.contentID) + require.NotEqual(t, first.Digest(), second.Digest()) + secondSpec := second.RequestSpec("test", "agents", nil) + + t.Log("Project each manifest extent and the updated runtime partition data") + require.Equal(t, []int64{16, 1, 1, 1}, *firstSpec.Partitions[0].Extent) + require.Equal(t, []int64{16, 2, 1, 1}, *secondSpec.Partitions[0].Extent) + data := resolvedPartitionData([]*ModelProjection{second}) + require.Equal(t, "part-1", data["partition_paths"]) + require.Equal(t, v3HXTopologyFamily, data["topologies"]) + require.Empty(t, firstSpec.PropSyncConnectors) + require.Empty(t, secondSpec.PropSyncConnectors) +} diff --git a/deploy/operator/internal/dynamo/patch.go b/deploy/operator/internal/dynamo/patch.go new file mode 100644 index 000000000000..e07e18b64b67 --- /dev/null +++ b/deploy/operator/internal/dynamo/patch.go @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package dynamo + +import ( + "fmt" + "slices" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/imdario/mergo" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/strategicpatch" +) + +func mergeObjectOverride(obj any, patch any) error { + if err := mergo.Merge(obj, patch, mergo.WithOverride); err != nil { + return fmt.Errorf("failed to apply override merge: %w", err) + } + return nil +} + +// Strategic merge field-merges matching list items, which breaks union-style +// objects when a user needs to replace an existing entry. +func removeNamed[T any](items, overrides []T, nameOf func(T) string) []T { + if len(items) == 0 || len(overrides) == 0 { + return items + } + + // Index every non-empty override name. + overrideNames := make(map[string]struct{}, len(overrides)) + for _, override := range overrides { + if name := nameOf(override); name != "" { + overrideNames[name] = struct{}{} + } + } + if len(overrideNames) == 0 { + return items + } + + // Remove every overridden base item while preserving survivor order. + return slices.DeleteFunc(items, func(item T) bool { + _, exists := overrideNames[nameOf(item)] + return exists + }) +} + +// mergeContainerByName merges override into base; both must be non-nil. +// mergeStrategy must be resolved by ResolveExtraPodSpecMergeStrategy. +// It preserves override by copying it once, then transferring owned fields to base. +func mergeContainerByName(base *corev1.Container, override *corev1.Container, mergeStrategy v1alpha1.ExtraPodSpecMergeStrategy) error { + user := override.DeepCopy() + user.Name = commonconsts.MainContainerName + + if mergeStrategy == v1alpha1.ExtraPodSpecMergeStrategyOverride { + baseEnv := base.Env + if err := mergeObjectOverride(base, *user); err != nil { + return err + } + base.Env = MergeEnvs(baseEnv, user.Env) + if user.Ports != nil { + base.Ports = user.Ports + } + } else { + clearPorts := user.Ports != nil && len(user.Ports) == 0 + base.Env = removeNamed(base.Env, user.Env, func(env corev1.EnvVar) string { return env.Name }) + if !clearPorts { + base.Ports = removeNamed(base.Ports, user.Ports, func(port corev1.ContainerPort) string { return port.Name }) + } + if err := patchObjectStrategic(base, user); err != nil { + return err + } + if clearPorts { + base.Ports = user.Ports + } + } + if user.LivenessProbe != nil { + base.LivenessProbe = user.LivenessProbe + } + if user.ReadinessProbe != nil { + base.ReadinessProbe = user.ReadinessProbe + } + if user.StartupProbe != nil { + base.StartupProbe = user.StartupProbe + } + base.Name = commonconsts.MainContainerName + return nil +} + +// mergePodSpecOverride applies the caller-owned override to podSpec, which must be non-nil. +// mergeStrategy must be resolved by ResolveExtraPodSpecMergeStrategy. +func mergePodSpecOverride(podSpec *corev1.PodSpec, podSpecOverride corev1.PodSpec, mergeStrategy v1alpha1.ExtraPodSpecMergeStrategy) error { + if mergeStrategy == v1alpha1.ExtraPodSpecMergeStrategyOverride { + return mergeObjectOverride(podSpec, &podSpecOverride) + } + + // Replace named volume unions before strategic field merging. + podSpec.Volumes = removeNamed(podSpec.Volumes, podSpecOverride.Volumes, func(volume corev1.Volume) string { return volume.Name }) + return patchObjectStrategic(podSpec, &podSpecOverride) +} + +func patchObjectStrategic[T any](obj T, patch T) error { + objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return fmt.Errorf("failed to convert object to unstructured: %w", err) + } + + patchMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(patch) + if err != nil { + return fmt.Errorf("failed to convert patch to unstructured: %w", err) + } + + var dataStruct T + patchedObj, err := strategicpatch.StrategicMergeMapPatch(objMap, patchMap, dataStruct) + if err != nil { + return fmt.Errorf("failed to apply strategic merge patch: %w", err) + } + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(patchedObj, obj) + if err != nil { + return fmt.Errorf("failed to deserialize patched object: %w", err) + } + + return nil +} diff --git a/deploy/operator/internal/dynamo/patch_test.go b/deploy/operator/internal/dynamo/patch_test.go new file mode 100644 index 000000000000..07a6e63d434f --- /dev/null +++ b/deploy/operator/internal/dynamo/patch_test.go @@ -0,0 +1,294 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package dynamo + +import ( + "testing" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestGenerateBasePodSpec_StrategicMergeReplacesExistingUnionStyleEntries(t *testing.T) { + secretsRetriever := &mockSecretsRetriever{} + controllerConfig := &configv1alpha1.OperatorConfiguration{} + controllerConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy = v1alpha1.ExtraPodSpecMergeStrategyStrategic + + t.Run("container ports with same name", func(t *testing.T) { + component := &v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "decode", + ComponentType: v1beta1.ComponentTypeDecode, + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: commonconsts.MainContainerName, + Ports: []corev1.ContainerPort{ + { + Name: commonconsts.DynamoSystemPortName, + ContainerPort: 28090, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + } + + podSpec, err := GenerateBasePodSpec( + component, + BackendFrameworkVLLM, + secretsRetriever, + "test-deployment", + "default", + RoleMain, + 1, + controllerConfig, + commonconsts.MultinodeDeploymentTypeGrove, + "decode", + nil, + staticContainerGPUCount(0), + ) + + require.NoError(t, err) + require.Len(t, podSpec.Containers, 1) + + systemPorts := make([]int32, 0) + for _, port := range podSpec.Containers[0].Ports { + if port.Name == commonconsts.DynamoSystemPortName { + systemPorts = append(systemPorts, port.ContainerPort) + } + } + assert.Equal(t, []int32{28090}, systemPorts) + assert.NotContains(t, podSpec.Containers[0].Ports, corev1.ContainerPort{ + Name: commonconsts.DynamoSystemPortName, + ContainerPort: int32(commonconsts.DynamoSystemPort), + Protocol: corev1.ProtocolTCP, + }) + }) + + t.Run("env vars", func(t *testing.T) { + component := &v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", + ComponentType: v1beta1.ComponentTypeFrontend, + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: commonconsts.MainContainerName, + Env: []corev1.EnvVar{ + {Name: "POD_NAME", Value: "static-pod-name"}, + }, + }, + }, + }, + }, + } + + podSpec, err := GenerateBasePodSpec( + component, + BackendFrameworkVLLM, + secretsRetriever, + "test-deployment", + "default", + RoleMain, + 1, + controllerConfig, + commonconsts.MultinodeDeploymentTypeGrove, + "frontend", + nil, + staticContainerGPUCount(0), + ) + + require.NoError(t, err) + require.Len(t, podSpec.Containers, 1) + + matches := 0 + var podNameEnv *corev1.EnvVar + for i := range podSpec.Containers[0].Env { + if podSpec.Containers[0].Env[i].Name == "POD_NAME" { + matches++ + podNameEnv = &podSpec.Containers[0].Env[i] + } + } + + require.Equal(t, 1, matches) + require.NotNil(t, podNameEnv) + assert.Equal(t, "static-pod-name", podNameEnv.Value) + assert.Nil(t, podNameEnv.ValueFrom) + }) +} + +func TestGenerateDynamoComponentsDeployments_ExtraPodSpecMergeStrategyPropagation(t *testing.T) { + tests := []struct { + name string + strategy v1beta1.ExtraPodSpecMergeStrategy + wantStrategy v1beta1.ExtraPodSpecMergeStrategy + }{ + { + name: "explicit strategy propagates to generated DCD spec", + strategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, + wantStrategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, + }, + { + name: "unset strategy remains unset on generated DCD spec", + strategy: "", + wantStrategy: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dgd := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dgd", + Namespace: "default", + }, + Spec: v1beta1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "frontend", + ComponentType: v1beta1.ComponentTypeFrontend, + Replicas: ptr.To(int32(1)), + ExtraPodSpecMergeStrategy: tt.strategy, + }, + }, + }, + } + + dcds, err := GenerateDynamoComponentsDeployments(dgd, nil, nil, RollingUpdateContext{}) + require.NoError(t, err) + + dcd := dcds["frontend"] + require.NotNil(t, dcd) + assert.Equal(t, tt.wantStrategy, dcd.Spec.ExtraPodSpecMergeStrategy) + }) + } +} + +func TestGenerateBasePodSpec_InvalidExtraPodSpecMergeStrategy(t *testing.T) { + secretsRetriever := &mockSecretsRetriever{} + controllerConfig := &configv1alpha1.OperatorConfiguration{} + component := &v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", + ComponentType: v1beta1.ComponentTypeFrontend, + ExtraPodSpecMergeStrategy: v1beta1.ExtraPodSpecMergeStrategy("invalid"), + } + + _, err := GenerateBasePodSpec( + component, + BackendFrameworkVLLM, + secretsRetriever, + "test-deployment", + "default", + RoleMain, + 1, + controllerConfig, + commonconsts.MultinodeDeploymentTypeGrove, + "frontend", + nil, + staticContainerGPUCount(0), + ) + + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid extraPodSpec merge strategy") +} + +func TestGenerateBasePodSpecForController_UsesOperatorConfigDefaultExtraPodSpecMergeStrategy(t *testing.T) { + tests := []struct { + name string + strategy v1beta1.ExtraPodSpecMergeStrategy + defaultStrategy v1alpha1.ExtraPodSpecMergeStrategy + wantPortNames []string + }{ + { + name: "falls back to override when field is absent", + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, + wantPortNames: []string{"metrics"}, + }, + { + name: "falls back to runtime default when field is absent", + strategy: "", + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, + wantPortNames: []string{commonconsts.DynamoContainerPortName, "metrics"}, + }, + { + name: "explicit field overrides runtime default", + strategy: v1beta1.ExtraPodSpecMergeStrategyOverride, + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, + wantPortNames: []string{"metrics"}, + }, + { + name: "explicit strategic field overrides runtime override", + strategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, + defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, + wantPortNames: []string{commonconsts.DynamoContainerPortName, "metrics"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controllerConfig := &configv1alpha1.OperatorConfiguration{} + controllerConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy = tt.defaultStrategy + + podSpec, err := GenerateBasePodSpecForController( + &v1beta1.DynamoComponentDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "frontend", + Namespace: "default", + }, + Spec: v1beta1.DynamoComponentDeploymentSpec{ + BackendFramework: "vllm", + DynamoComponentDeploymentSharedSpec: v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", + ComponentType: v1beta1.ComponentTypeFrontend, + ExtraPodSpecMergeStrategy: tt.strategy, + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: commonconsts.MainContainerName, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9001, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &mockSecretsRetriever{}, + controllerConfig, + RoleMain, + commonconsts.MultinodeDeploymentTypeGrove, + staticContainerGPUCount(0), + GenerateBasePodSpecForControllerOptions{}, + ) + require.NoError(t, err) + require.Len(t, podSpec.Containers, 1) + + var gotPortNames []string + for _, port := range podSpec.Containers[0].Ports { + gotPortNames = append(gotPortNames, port.Name) + } + assert.ElementsMatch(t, tt.wantPortNames, gotPortNames) + }) + } +} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml new file mode 100644 index 000000000000..5833abbf8a45 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -0,0 +1,230 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: dgd-lpx-v2-vllm + namespace: inference-engine +spec: + components: + - name: frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - args: + - --model-name + - meta-llama/Llama-3.2-1B-Instruct + - --router-mode + - kv + - --router-reset-states + - --no-decode-fallback + command: + - python3 + - -m + - dynamo.frontend + env: + - name: HF_HOME + value: /nfs/huggingface + - name: HF_HUB_CACHE + value: /nfs/huggingface/hub + - name: HF_HUB_OFFLINE + value: "1" + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.1 + name: main + volumeMounts: + - mountPath: /nfs + name: model-storage + - name: lpu + type: lpx + extraPodSpecMergeStrategy: strategic + replicas: 2 + lpx: + buildId: llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c + roles: + - name: leader + replicas: 1 + podTemplate: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1v2-b300n1-nic1 + containers: + - args: + - --dynamo-model-name + - meta-llama/Llama-3.2-1B-Instruct + - --model-card + - /etc/cyborg/model-card.json + - --use-disagg-prefill + - --nixl-kv-layout + - HND + - --num-layers + - '16' + env: + - name: GLUE_RDMA_PATH_MTU + value: '1024' + image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 + name: main + resources: + limits: + nvidia.com/gpu: '1' + volumeMounts: + - mountPath: /etc/cyborg/model-card.json + name: cyborg-model-card + readOnly: true + subPath: model-card.json + - mountPath: /nfs + name: model-storage + tolerations: + - key: nvidia.com/gpu + operator: Exists + volumes: + - configMap: + name: dgd-lpx-v2-vllm-cyborg-model-card + name: cyborg-model-card + - name: worker + podTemplate: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1r1-lpu1 + - c1r1-lpu2 + - c1r1-lpu3 + - c1r1-lpu4 + - c1r1-lpu5 + - c1r1-lpu6 + - c1r1-lpu7 + - c1r1-lpu8 + - c1r2-lpu1 + - c1r2-lpu2 + - c1r2-lpu3 + - c1r2-lpu4 + - c1r2-lpu5 + - c1r2-lpu6 + - c1r2-lpu7 + - c1r2-lpu8 + - c1r7-lpu1 + - c1r7-lpu2 + - c1r7-lpu3 + - c1r7-lpu4 + - c1r7-lpu5 + - c1r7-lpu6 + - c1r7-lpu7 + - c1r7-lpu8 + containers: + - args: + - -c + - "readarray -t partition_node_counts < /configs/nodes_per_partition\nreadarray -t partition_node_offsets < /configs/partition_node_offsets\n\ + part=\"\"\nfor i in \"${!partition_node_counts[@]}\"; do\n if (( GROVE_PCLQ_POD_INDEX >= partition_node_offsets[i]\ + \ &&\n GROVE_PCLQ_POD_INDEX < partition_node_offsets[i] + partition_node_counts[i] )); then\n part=\"\ + ${i}\"\n break\n fi\ndone\n: \"${part:?no partition row for Grove pod index ${GROVE_PCLQ_POD_INDEX}}\"\nexec\ + \ agent-hybrid-v2 \\\n --assemble-dir \"${GAS_DIR}\" \\\n --part \"${part}\" \\\n --port \"${RDMA_PORT}\" \\\ + \n --nic \"${NIC_NAME}\" \\\n --metadata-file \"$(mktemp)\" \\\n --readiness-port \"${READINESS_PORT}\"" + image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f + name: main + volumeMounts: + - mountPath: /nfs + name: model-storage + - name: prefill + type: prefill + replicas: 1 + sharedMemorySize: 16Gi + podTemplate: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1v2-b300n1-nic1 + containers: + - args: + - --model + - meta-llama/Llama-3.2-1B-Instruct + - --served-model-name + - meta-llama/Llama-3.2-1B-Instruct + - --tensor-parallel-size + - "1" + - --disaggregation-mode + - prefill + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + - --attention-backend + - FLASH_ATTN + - --max-num-seqs + - "1" + - --gpu-memory-utilization + - "0.8" + - --max-model-len + - "8192" + - --enforce-eager + - --block-size + - "32" + command: + - python3 + - -m + - dynamo.vllm + env: + - name: VLLM_NIXL_SIDE_CHANNEL_PORT + value: "5600" + - name: VLLM_KV_CACHE_LAYOUT + value: HND + - name: UCX_NET_DEVICES + value: all + - name: HF_HOME + value: /nfs/huggingface + - name: HF_HUB_CACHE + value: /nfs/huggingface/hub + - name: HF_HUB_OFFLINE + value: "1" + - name: TRANSFORMERS_OFFLINE + value: "1" + - name: HF_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hf-token-secret + optional: true + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.1 + name: main + resources: + limits: + nvidia.com/gpu: "1" + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RESOURCE + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /nfs + name: model-storage + workingDir: /workspace/examples/backends/vllm + hostIPC: true + tolerations: + - key: nvidia.com/gpu + operator: Exists + volumes: + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml new file mode 100644 index 000000000000..f1d6b1ca4d1f --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -0,0 +1,975 @@ +data: + lpu_servers-0: |- + lpu-0-lpu-wkr-m-0-0 + lpu-0-lpu-wkr-m-0-1 + lpu-0-lpu-wkr-m-0-2 + lpu-0-lpu-wkr-m-0-3 + lpu-0-lpu-wkr-m-0-4 + lpu-0-lpu-wkr-m-0-5 + lpu-0-lpu-wkr-m-0-6 + lpu-0-lpu-wkr-m-0-7 + lpu-0-lpu-wkr-m-0-8 + lpu-0-lpu-wkr-m-0-9 + lpu-0-lpu-wkr-m-0-10 + lpu-0-lpu-wkr-m-0-11 + lpu-0-lpu-wkr-m-0-12 + lpu-0-lpu-wkr-m-0-13 + lpu-0-lpu-wkr-m-0-14 + lpu-0-lpu-wkr-m-0-15 + lpu-0-lpu-wkr-m-0-16 + lpu_servers-1: |- + lpu-1-lpu-wkr-m-0-0 + lpu-1-lpu-wkr-m-0-1 + lpu-1-lpu-wkr-m-0-2 + lpu-1-lpu-wkr-m-0-3 + lpu-1-lpu-wkr-m-0-4 + lpu-1-lpu-wkr-m-0-5 + lpu-1-lpu-wkr-m-0-6 + lpu-1-lpu-wkr-m-0-7 + lpu-1-lpu-wkr-m-0-8 + lpu-1-lpu-wkr-m-0-9 + lpu-1-lpu-wkr-m-0-10 + lpu-1-lpu-wkr-m-0-11 + lpu-1-lpu-wkr-m-0-12 + lpu-1-lpu-wkr-m-0-13 + lpu-1-lpu-wkr-m-0-14 + lpu-1-lpu-wkr-m-0-15 + lpu-1-lpu-wkr-m-0-16 + tokenizer_dir: /testdata/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + name: dgd-lpx-v2-vllm-decode + namespace: inference-engine +--- +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + gas_dir: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c + model_config.toml: | + type = 'Single' + + [iop] + model_path = '/nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c' + tokenizer_path = '/nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer' + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: |- + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + partition_ids: |- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + partition_indices: |- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + partition_models: |- + default + default + default + default + default + default + default + default + default + default + default + default + default + default + default + default + default + partition_node_offsets: |- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + partition_paths: |- + part-0 + part-1 + part-2 + part-3 + part-4 + part-5 + part-6 + part-7 + part-8 + part-9 + part-10 + part-11 + part-12 + part-13 + part-14 + part-15 + part-16 + topologies: |- + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + name: dgd-lpx-v2-vllm-lpu + namespace: inference-engine +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + labels: + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + name: dgd-lpx-v2-2c5c68ef-lpx + namespace: inference-engine +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + nvidia.com/extra-resources-hash: a9c40c6452f37713460176f888205de0185c5478ab3a4aa2a06936caa86ff0c4 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-lpx-v2-vllm-lpu + name: lpu-engine-gpu + spec: + minAvailable: 1 + podSpec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1v2-b300n1-nic1 + containers: + - args: + - --dynamo-model-name + - meta-llama/Llama-3.2-1B-Instruct + - --model-card + - /etc/cyborg/model-card.json + - --use-disagg-prefill + - --nixl-kv-layout + - HND + - --num-layers + - "16" + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin + - name: GLUE_RDMA_PATH_MTU + value: "1024" + - name: LPX_ENGINE_REPLICA + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podcliquescalinggroup-replica-index'] + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: SERVER_HOSTS_FILE + value: /configs/lpu_servers-$(LPX_ENGINE_REPLICA) + - name: TOKENIZER_DIR + valueFrom: + configMapKeyRef: + key: tokenizer_dir + name: dgd-lpx-v2-vllm-decode + - name: TOTAL_REPLICAS + value: "1" + image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: "1" + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /etc/cyborg/model-card.json + name: cyborg-model-card + readOnly: true + subPath: model-card.json + - mountPath: /nfs + name: model-storage + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /configs + name: config + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: nvidia.com/gpu + operator: Exists + volumes: + - configMap: + name: dgd-lpx-v2-vllm-cyborg-model-card + name: cyborg-model-card + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband + - configMap: + name: dgd-lpx-v2-vllm-decode + name: config + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 1 + roleName: lpu-engine-gpu + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + nvidia.com/extra-resources-hash: d782ea49a4ae530a65fb915ae77c6d4b603d6bee3f5b336a52d026439b8f2d41 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-lpx-v2-vllm-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 17 + podSpec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1r1-lpu1 + - c1r1-lpu2 + - c1r1-lpu3 + - c1r1-lpu4 + - c1r1-lpu5 + - c1r1-lpu6 + - c1r1-lpu7 + - c1r1-lpu8 + - c1r2-lpu1 + - c1r2-lpu2 + - c1r2-lpu3 + - c1r2-lpu4 + - c1r2-lpu5 + - c1r2-lpu6 + - c1r2-lpu7 + - c1r2-lpu8 + - c1r7-lpu1 + - c1r7-lpu2 + - c1r7-lpu3 + - c1r7-lpu4 + - c1r7-lpu5 + - c1r7-lpu6 + - c1r7-lpu7 + - c1r7-lpu8 + containers: + - args: + - -c + - |- + readarray -t partition_node_counts < /configs/nodes_per_partition + readarray -t partition_node_offsets < /configs/partition_node_offsets + part="" + for i in "${!partition_node_counts[@]}"; do + if (( GROVE_PCLQ_POD_INDEX >= partition_node_offsets[i] && + GROVE_PCLQ_POD_INDEX < partition_node_offsets[i] + partition_node_counts[i] )); then + part="${i}" + break + fi + done + : "${part:?no partition row for Grove pod index ${GROVE_PCLQ_POD_INDEX}}" + exec agent-hybrid-v2 \ + --assemble-dir "${GAS_DIR}" \ + --part "${part}" \ + --port "${RDMA_PORT}" \ + --nic "${NIC_NAME}" \ + --metadata-file "$(mktemp)" \ + --readiness-port "${READINESS_PORT}" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GAS_DIR + valueFrom: + configMapKeyRef: + key: gas_dir + name: dgd-lpx-v2-vllm-lpu + - name: GLUE_RDMA_PATH_MTU + value: "1024" + - name: LPU_MODEL_NAME + value: default + - name: NATS_SERVER + value: nats-address + - name: NIC_NAME + value: mlx5_0 + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: READINESS_PORT + value: "19878" + - name: TOPOLOGIES + valueFrom: + configMapKeyRef: + key: topologies + name: dgd-lpx-v2-vllm-lpu + - name: UCX_TLS + value: tcp + image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f + name: agent + readinessProbe: + failureThreshold: 6 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsUser: 0 + startupProbe: + failureThreshold: 60 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + runAsGroup: 0 + runAsUser: 0 + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: dgd-lpx-v2-vllm-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - emptyDir: + medium: HugePages + name: hugepages + replicas: 17 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + cliqueNames: + - lpu-wkr-m-0 + - lpu-engine-gpu + minAvailable: 1 + name: lpu + replicas: 2 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 +--- +metadata: + labels: + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + name: dgd-lpx-v2-vllm + namespace: inference-engine +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeAnyOrder + cliques: + - labels: + nvidia.com/dynamo-component: frontend + nvidia.com/dynamo-component-type: frontend + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-lpx-v2-vllm-frontend + name: frontend + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - --model-name + - meta-llama/Llama-3.2-1B-Instruct + - --router-mode + - kv + - --router-reset-states + - --no-decode-fallback + command: + - python3 + - -m + - dynamo.frontend + env: + - name: DYNAMO_PORT + value: "8000" + - name: DYN_COMPONENT + value: frontend + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_HTTP_PORT + value: "8000" + - name: DYN_NAMESPACE + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_NAMESPACE_PREFIX + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: ETCD_ENDPOINTS + value: etcd-address + - name: HF_HOME + value: /nfs/huggingface + - name: HF_HUB_CACHE + value: /nfs/huggingface/hub + - name: HF_HUB_OFFLINE + value: "1" + - name: NATS_SERVER + value: nats-address + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.1 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /live + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + name: main + ports: + - containerPort: 8000 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: {} + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 1 + roleName: frontend + - labels: + nvidia.com/dynamo-component: prefill + nvidia.com/dynamo-component-type: prefill + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm + nvidia.com/dynamo-sub-component-type: prefill + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-lpx-v2-vllm-prefill + name: prefill + spec: + minAvailable: 1 + podSpec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1v2-b300n1-nic1 + containers: + - args: + - --model + - meta-llama/Llama-3.2-1B-Instruct + - --served-model-name + - meta-llama/Llama-3.2-1B-Instruct + - --tensor-parallel-size + - "1" + - --disaggregation-mode + - prefill + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + - --attention-backend + - FLASH_ATTN + - --max-num-seqs + - "1" + - --gpu-memory-utilization + - "0.8" + - --max-model-len + - "8192" + - --enforce-eager + - --block-size + - "32" + command: + - python3 + - -m + - dynamo.vllm + env: + - name: DYN_COMPONENT + value: prefill + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: HF_HOME + value: /nfs/huggingface + - name: HF_HUB_CACHE + value: /nfs/huggingface/hub + - name: HF_HUB_OFFLINE + value: "1" + - name: HF_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hf-token-secret + optional: true + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: TRANSFORMERS_OFFLINE + value: "1" + - name: UCX_NET_DEVICES + value: all + - name: VLLM_KV_CACHE_LAYOUT + value: HND + - name: VLLM_NIXL_SIDE_CHANNEL_PORT + value: "5600" + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.1 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: "1" + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RESOURCE + privileged: true + runAsUser: 0 + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + workingDir: /workspace/examples/backends/vllm + hostIPC: true + restartPolicy: Always + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: nvidia.com/gpu + operator: Exists + volumes: + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 16Gi + name: shared-memory + replicas: 1 + roleName: prefill + headlessServiceConfig: + publishNotReadyAddresses: true + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml new file mode 100644 index 000000000000..044507b99025 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -0,0 +1,59 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v2-hybrid + namespace: test + uid: 22222222-2222-4222-8222-222222222222 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected-lpx + roles: + - name: leader + replicas: 1 + podTemplate: + spec: + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + containers: + - name: main + image: example.invalid/gpu-runtime:test + command: + - /opt/gpu-runtime + args: + - serve + volumeMounts: + - name: config + mountPath: /configs + - name: infiniband + mountPath: /dev/infiniband + resources: + claims: + - name: gpu + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + command: + - /opt/dynamo-lpu + args: + - serve + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml new file mode 100644 index 000000000000..3d18af7b9e91 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -0,0 +1,436 @@ +data: + lpu_servers: lpu-0-lpu-wkr-m-0-0 + tokenizer_dir: /testdata/node-local-v2-connected-lpx/tokenizer +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + name: node-local-v2-hybrid-decode + namespace: test +--- +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0, 1, 2, 3] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + gas_dir: /nfs/node-local-v2-connected-lpx + model_config.toml: | + type = 'Single' + + [iop] + model_path = '/nfs/node-local-v2-connected-lpx' + tokenizer_path = '/nfs/node-local-v2-connected-lpx/tokenizer' + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: "4" + partition_ids: "0" + partition_indices: "0" + partition_models: default + partition_node_offsets: "0" + partition_paths: part-0 + topologies: URSA_V2__Q8__32C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + name: node-local-v2-hybrid-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid + name: node-local-9a49489a-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + nvidia.com/extra-resources-hash: d410b2a3efc122a1a232c8d3d209b74eda592811c79dad628c5d46d8fac52264 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid + nvidia.com/dynamo-namespace: test-node-local-v2-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-hybrid-lpu + name: lpu-engine-gpu + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + command: + - /opt/gpu-runtime + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v2-connected-lpx/manifest.v2.capnp.bin + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: SERVER_HOSTS_FILE + value: /configs/lpu_servers + - name: TOKENIZER_DIR + valueFrom: + configMapKeyRef: + key: tokenizer_dir + name: node-local-v2-hybrid-decode + - name: TOTAL_REPLICAS + value: "1" + image: example.invalid/gpu-runtime:test + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + claims: + - name: gpu + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /configs + name: config + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /dev/shm + name: shared-memory + - mountPath: /nfs + name: model-storage + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - configMap: + name: node-local-v2-hybrid-decode + name: config + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + replicas: 1 + roleName: lpu-engine-gpu + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + nvidia.com/extra-resources-hash: 24aefcdb8d31a8d6357564441af523737eca563784d5fbe2d770e94f5c2b9c05 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid + nvidia.com/dynamo-namespace: test-node-local-v2-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-hybrid-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GAS_DIR + valueFrom: + configMapKeyRef: + key: gas_dir + name: node-local-v2-hybrid-lpu + - name: GLUE_RDMA_PATH_MTU + value: "1024" + - name: LPU_MODEL_NAME + value: default + - name: NATS_SERVER + value: nats-address + - name: NIC_NAME + value: mlx5_0 + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: READINESS_PORT + value: "19878" + - name: TOPOLOGIES + valueFrom: + configMapKeyRef: + key: topologies + name: node-local-v2-hybrid-lpu + - name: UCX_TLS + value: tcp + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 6 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsUser: 0 + startupProbe: + failureThreshold: 60 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + runAsGroup: 0 + runAsUser: 0 + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-hybrid-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - emptyDir: + medium: HugePages + name: hugepages + replicas: 4 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + cliqueNames: + - lpu-wkr-m-0 + - lpu-engine-gpu + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml new file mode 100644 index 000000000000..2182ba77c75d --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -0,0 +1,37 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v2-lpu-only + namespace: test + uid: 11111111-1111-4111-8111-111111111111 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected + roles: + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + command: + - /opt/dynamo-lpu + args: + - serve + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml new file mode 100644 index 000000000000..89a7bc892d34 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -0,0 +1,432 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0, 1, 2, 3] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + model_config.toml: | + type = 'Single' + + [iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/testdata/node-local-v2-connected' + num_kv_caches = 1 + output_size = 1 + prop_sync = true + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: |- + 2 + 2 + partition_ids: |- + 0 + 1 + partition_node_offsets: |- + 0 + 2 + partition_paths: |- + part-0 + part-1 + topologies: |- + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + name: node-local-v2-lpu-only-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only + name: node-local-1f9267e3-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v2-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-lpu-only-lpu + name: lpu-ldr + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - lpu-wkr-m-0 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + initContainers: + - args: + - -c + - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; + chmod 600 /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/bash + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-lpu-only-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 384 + secretName: ssh-secret + - emptyDir: {} + name: single-v2-ssh-key + replicas: 1 + roleName: lpu-ldr + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v2-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-lpu-only-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-lpu-only-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + replicas: 4 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + cliqueNames: + - lpu-ldr + - lpu-wkr-m-0 + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml new file mode 100644 index 000000000000..b6d20b1ebaf2 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -0,0 +1,56 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v2-specdecode + namespace: test + uid: 33333333-3333-4333-8333-333333333333 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: draft + type: lpx + replicas: 2 + lpx: + buildId: node-local-v2-connected + roles: + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected + roles: + - name: leader + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml new file mode 100644 index 000000000000..3b88d7e41336 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -0,0 +1,1061 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0, 1, 2, 3] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + + [datacenters.racks.lpu-wkr-m-1] + available_nodes = [0, 1, 2, 3] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-1-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + + [datacenters.racks.lpu-wkr-m-2] + available_nodes = [0, 1, 2, 3] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-2-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + model_config.toml: | + draft_to_target_port = 23456 + head_to_head_port = 23458 + num_drafts = 2 + target_to_draft_port = 23457 + type = 'SpecDecode' + + [draft] + [draft.iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/testdata/node-local-v2-connected' + num_kv_caches = 1 + output_size = 1 + prop_sync = true + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' + + [draft.scheduler] + + [draft.setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + + [target] + [target.iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/testdata/node-local-v2-connected' + num_kv_caches = 1 + output_size = 1 + prop_sync = true + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' + + [target.scheduler] + + [target.setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: |- + 2 + 2 + 2 + 2 + 2 + 2 + partition_ids: |- + 0 + 1 + 0 + 1 + 0 + 1 + partition_indices: |- + 0 + 1 + 0 + 1 + 0 + 1 + partition_models: |- + draft0 + draft0 + draft1 + draft1 + target + target + partition_node_offsets: |- + 0 + 2 + 0 + 2 + 0 + 2 + partition_paths: |- + part-0 + part-1 + part-0 + part-1 + part-0 + part-1 + topologies: |- + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + name: node-local-v2-specdecode-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode + name: node-local-3bd5df51-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode + nvidia.com/dynamo-namespace: test-node-local-v2-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-specdecode-lpu + name: lpu-ldr + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" + "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing + GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" + -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" + /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - -- + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /tmp/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - lpu-wkr-m-0:lpu-wkr-m-1:lpu-wkr-m-2 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + initContainers: + - args: + - -c + - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; + chmod 600 /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/bash + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 384 + secretName: ssh-secret + - emptyDir: {} + name: single-v2-ssh-key + replicas: 1 + roleName: lpu-ldr + startsAfter: + - lpu-wkr-m-0 + - lpu-wkr-m-1 + - lpu-wkr-m-2 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 + scheduling.lpu.nvidia.com/model: draft0 + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode + nvidia.com/dynamo-namespace: test-node-local-v2-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-specdecode-draft + name: lpu-wkr-m-0 + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit + 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number of + rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is + required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} + != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models + and partition_indices must match the partition config row count\" >&2\n\t\texit + 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count + ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove + pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition + rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif + ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid + logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" + >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime + partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport + RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif + [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" + ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_MODEL_NAME + value: draft0 + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + replicas: 4 + roleName: lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b + scheduling.lpu.nvidia.com/model: draft1 + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode + nvidia.com/dynamo-namespace: test-node-local-v2-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-specdecode-draft + name: lpu-wkr-m-1 + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit + 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number of + rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is + required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} + != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models + and partition_indices must match the partition config row count\" >&2\n\t\texit + 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count + ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove + pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition + rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif + ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid + logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" + >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime + partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport + RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif + [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" + ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_MODEL_NAME + value: draft1 + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + replicas: 4 + roleName: lpu-wkr-m-1 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e + scheduling.lpu.nvidia.com/model: target + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode + nvidia.com/dynamo-namespace: test-node-local-v2-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-specdecode-lpu + name: lpu-wkr-m-2 + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit + 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number of + rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is + required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} + != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models + and partition_indices must match the partition config row count\" >&2\n\t\texit + 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count + ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove + pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition + rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif + ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid + logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" + >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime + partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport + RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif + [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" + ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_MODEL_NAME + value: target + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + replicas: 4 + roleName: lpu-wkr-m-2 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + cliqueNames: + - lpu-ldr + - lpu-wkr-m-0 + - lpu-wkr-m-1 + - lpu-wkr-m-2 + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml new file mode 100644 index 000000000000..9b9d57e4b397 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -0,0 +1,53 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v3-hx-hybrid + namespace: test + uid: 44444444-4444-4444-8444-444444444444 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v3-hx-connected-lpx + roles: + - name: leader + replicas: 1 + podTemplate: + spec: + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + containers: + - name: main + image: example.invalid/gpu-runtime:test + command: + - /opt/gpu-runtime + args: + - serve + resources: + claims: + - name: gpu + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/v3-hybrid + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml new file mode 100644 index 000000000000..0e2b973df923 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -0,0 +1,603 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + gas_dir: /nfs/node-local-v3-hx-connected-lpx + model_config.toml: | + type = 'Single' + + [iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/nfs/node-local-v3-hx-connected-lpx' + num_kv_caches = 1 + output_size = 1 + prop_sync = false + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/nfs/node-local-v3-hx-connected-lpx/tokenizer' + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: "1" + partition_ids: "1" + partition_indices: "0" + partition_models: default + partition_node_offsets: "0" + partition_paths: part-1 + topologies: opaque-v3-topology +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + name: node-local-v3-hx-hybrid-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid + name: node-local-b112da53-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid + nvidia.com/dynamo-namespace: test-node-local-v3-hx-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-hybrid-lpu + name: lpu-engine-gpu + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + command: + - /opt/gpu-runtime + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v3-hx-connected-lpx/manifest.v2.capnp.bin + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/gpu-runtime:test + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + claims: + - name: gpu + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + - mountPath: /nfs + name: model-storage + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + replicas: 1 + roleName: lpu-engine-gpu + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + nvidia.com/extra-resources-hash: 8803dd921b1a97775b917767487276baa0462067b92142d4456e4409df78ee42 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxStrictHybrid + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid + nvidia.com/dynamo-namespace: test-node-local-v3-hx-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-hybrid-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit + 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number of + rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is + required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} + != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models + and partition_indices must match the partition config row count\" >&2\n\t\texit + 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count + ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove + pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition + rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif + ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid + logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" + >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime + partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport + RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nAGENT_BINARY=\"${AGENT_BINARY:-agent-hybrid-v2-roce}\"\nHOME=\"${HOME:-/root}\"\nMPI_SSH_PORT=\"${MPI_SSH_PORT:-2222}\"\nREADINESS_PORT=\"${READINESS_PORT:-19878}\"\nRDMA_PORT=\"${RDMA_PORT:-19877}\"\n\nreadarray + -t SOURCE_PARTITIONS < \"${LPU_CONFIG_DIR}/partition_ids\"\nreadarray + -t PARTITION_PATHS < \"${LPU_CONFIG_DIR}/partition_paths\"\nreadarray + -t TOPOLOGIES < \"${LPU_CONFIG_DIR}/topologies\"\n\nSOURCE_PARTITION=\"${SOURCE_PARTITIONS[$CONFIG_PARTITION_INDEX]}\"\nPARTITION_PATH=\"${PARTITION_PATHS[$CONFIG_PARTITION_INDEX]}\"\nTOPOLOGY=\"${TOPOLOGIES[$CONFIG_PARTITION_INDEX]}\"\n\nagent_part_from_partition_path() + {\n\tlocal part_path=\"$1\"\n\tlocal part_name=\"${part_path##*/}\"\n\tif + [[ \"${part_name}\" =~ ^part-?([0-9]+)$ ]]; then\n\t\tprintf \"%s\\n\" + \"${BASH_REMATCH[1]}\"\n\t\treturn\n\tfi\n\n\tprintf \"%s\\n\" \"${SOURCE_PARTITION}\"\n}\n\nagent_assemble_dir_from_partition_path() + {\n\tlocal part_path=\"$1\"\n\tlocal agent_part=\"$2\"\n\tlocal part_name=\"${part_path##*/}\"\n\tlocal + parent=\"${part_path%/*}\"\n\tif [[ \"${part_name}\" != \"part-${agent_part}\" + ]]; then\n\t\tlocal assemble_dir=\"/tmp/dynamo-agent-assemble/row-${CONFIG_PARTITION_INDEX}\"\n\t\tmkdir + -p \"${assemble_dir}\"\n\t\tln -sfn \"${GAS_DIR}/${part_path}\" \"${assemble_dir}/part-${agent_part}\"\n\t\tprintf + \"%s\\n\" \"${assemble_dir}\"\n\t\treturn\n\tfi\n\tif [[ \"${parent}\" + == \"${part_path}\" ]]; then\n\t\tprintf \"%s\\n\" \"${GAS_DIR}\"\n\t\treturn\n\tfi\n\tprintf + \"%s/%s\\n\" \"${GAS_DIR}\" \"${parent}\"\n}\n\nAGENT_PART=\"$(agent_part_from_partition_path + \"${PARTITION_PATH}\")\"\nAGENT_ASSEMBLE_DIR=\"$(agent_assemble_dir_from_partition_path + \"${PARTITION_PATH}\" \"${AGENT_PART}\")\"\n\nsetup_ssh() {\n\tmkdir + -p \"${HOME}/.ssh\" \"${HOME}/.ssh/host_keys\"\n\tcp /ssh-pk/private.key + \"${HOME}/.ssh/id_rsa\"\n\tcp /ssh-pk/private.key.pub \"${HOME}/.ssh/id_rsa.pub\"\n\tcp + /ssh-pk/private.key.pub \"${HOME}/.ssh/authorized_keys\"\n\tchmod 600 + \"${HOME}/.ssh/id_rsa\" \"${HOME}/.ssh/authorized_keys\"\n\tchmod 644 + \"${HOME}/.ssh/id_rsa.pub\"\n\tprintf \"Host *\\nIdentityFile %s/.ssh/id_rsa\\nStrictHostKeyChecking + no\\nPort %s\\n\" \"${HOME}\" \"${MPI_SSH_PORT}\" > \"${HOME}/.ssh/config\"\n\n\tssh-keygen + -t rsa -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" -N \"\" -q\n\tssh-keygen + -t ecdsa -f \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" -N \"\" -q\n\tssh-keygen + -t ed25519 -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" -N \"\" + -q\n}\n\nrun_sshd() {\n\tsetup_ssh\n\techo \"Starting sshd on port ${MPI_SSH_PORT} + for partition ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}\"\n\texec + /bin/sshd -D -e \\\n\t\t-o \"PidFile none\" \\\n\t\t-o \"Port ${MPI_SSH_PORT}\" + \\\n\t\t-o \"AuthorizedKeysFile ${HOME}/.ssh/authorized_keys\" \\\n\t\t-o + \"StrictModes no\" \\\n\t\t-o \"PermitRootLogin yes\" \\\n\t\t-o \"PubkeyAuthentication + yes\" \\\n\t\t-o \"SetEnv PARTITION_ID=\\\"${PARTITION_ID}\\\" RANK_IN_PARTITION=\\\"${RANK_IN_PARTITION}\\\"\" + \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\"\n}\n\npartition_hostnames() + {\n\tfor ((rank = 0; rank < NODE_COUNT; rank++)); do\n\t\tlocal node_index=$((NODE_OFFSET + + rank))\n\t\tprintf \"%s-%d.%s\\n\" \"${GROVE_PCLQ_NAME}\" \"${node_index}\" + \"${GROVE_HEADLESS_SERVICE}\"\n\tdone\n}\n\nwait_for_peer_ssh() {\n\tlocal + host=\"$1\"\n\tfor i in $(seq 1 120); do\n\t\tif ssh -o ConnectTimeout=2 + \"root@${host}\" echo ok >/dev/null 2>&1; then\n\t\t\techo \"${host}:${MPI_SSH_PORT} + reachable\"\n\t\t\treturn\n\t\tfi\n\t\tif [[ \"${i}\" == \"120\" ]]; + then\n\t\t\techo \"${host}:${MPI_SSH_PORT} unreachable after 120 attempts\" + >&2\n\t\t\texit 1\n\t\tfi\n\t\tsleep 1\n\tdone\n}\n\nrun_lpu_system_init\ncommand + -v \"${AGENT_BINARY}\"\n\nif (( NODE_COUNT == 1 )); then\n\techo \"Starting + ${AGENT_BINARY} for logical partition ${LOGICAL_PARTITION_INDEX}, source + partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir + ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}\"\n\texec \"${AGENT_BINARY}\" + \\\n\t\t--assemble-dir \"${AGENT_ASSEMBLE_DIR}\" \\\n\t\t--part \"${AGENT_PART}\" + \\\n\t\t--topology \"${TOPOLOGY}\" \\\n\t\t--port \"${RDMA_PORT}\" \\\n\t\t--readiness-port + \"${READINESS_PORT}\" \\\n\t\t--stage-name continue\nfi\n\nif [[ \"${PARTITION_RANK}\" + != \"0\" ]]; then\n\trun_sshd\nfi\n\nsetup_ssh\nPARTITION_HOSTS=\"$(partition_hostnames)\"\nreadarray + -t NODE_HOSTS <<< \"${PARTITION_HOSTS}\"\n\nMPI_HOSTS=()\nMPI_HOSTS+=(\"${POD_IP}:1\")\nfor + host in \"${NODE_HOSTS[@]:1}\"; do\n\tMPI_HOSTS+=(\"${host}:1\")\ndone\nMPI_HOST_LIST=$(IFS=,; + echo \"${MPI_HOSTS[*]}\")\n\nfor host in \"${NODE_HOSTS[@]:1}\"; do\n\twait_for_peer_ssh + \"${host}\"\ndone\n\nMPI_ARGS=(\n\t--allow-run-as-root\n\t--host \"${MPI_HOST_LIST}\"\n\t-np + \"${NODE_COUNT}\"\n\t--map-by ppr:1:node:OVERSUBSCRIBE\n\t--mca pml + ob1\n\t--mca btl tcp,self\n\t--mca mtl ^ofi\n\t--mca plm_rsh_args \"-p + ${MPI_SSH_PORT}\"\n\t--bind-to none\n\t-x GROQ_DRACO_NETWORK_UP_CHECK_TIMEOUT\n\t-x + DRACO_FORCE_RECONFIGURE\n\t-x DRACO_MAXFILE_OVERRIDE\n\t-x LPU_VSAP_DIB_CREDIT_AUTO_RESTORE_US\n\t--output + TAG\n)\nif [[ -n \"${MPI_BTL_TCP_IF_INCLUDE:-}\" ]]; then\n\tMPI_ARGS+=(--mca + btl_tcp_if_include \"${MPI_BTL_TCP_IF_INCLUDE}\")\nfi\n\necho \"Starting + mpirun for logical partition ${LOGICAL_PARTITION_INDEX}, source partition + ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, + topology ${TOPOLOGY}\"\nexec numactl --cpunodebind=0 --membind=0 mpirun + \"${MPI_ARGS[@]}\" -- \\\n\t\"${AGENT_BINARY}\" \\\n\t\t--assemble-dir + \"${AGENT_ASSEMBLE_DIR}\" \\\n\t\t--part \"${AGENT_PART}\" \\\n\t\t--topology + \"${TOPOLOGY}\" \\\n\t\t--multinode \\\n\t\t--port \"${RDMA_PORT}\" + \\\n\t\t--readiness-port \"${READINESS_PORT}\" \\\n\t\t--stage-name + continue\n" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GAS_DIR + valueFrom: + configMapKeyRef: + key: gas_dir + name: node-local-v3-hx-hybrid-lpu + - name: GLUE_RDMA_PATH_MTU + value: "1024" + - name: LPU_MODEL_NAME + value: default + - name: NATS_SERVER + value: nats-address + - name: NIC_NAME + value: mlx5_0 + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: READINESS_PORT + value: "19878" + - name: TOPOLOGIES + valueFrom: + configMapKeyRef: + key: topologies + name: node-local-v3-hx-hybrid-lpu + - name: UCX_TLS + value: tcp + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + exec: + command: + - /bin/bash + - -c + - "set -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" + >&2\n\texit 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number + of rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME + is required when partition_models is configured\" >&2\n\t\texit + 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} + || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); + then\n\t\techo \"partition_models and partition_indices must match + the partition config row count\" >&2\n\t\texit 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node + count ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit + 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo + \"Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} + partition rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit + 1\nfi\nif ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo + \"Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for + config row ${CONFIG_PARTITION_INDEX}\" >&2\n\texit 1\nfi\n\nif (( + NODE_COUNT > 1 )) && [[ \"${PARTITION_RANK}\" != \"0\" ]]; then\n\tcat + < /dev/null > \"/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-2222}\"\nelse\n\texec + 3<>\"/dev/tcp/127.0.0.1/${READINESS_PORT:-19878}\"\n\tprintf \"GET + /readyz HTTP/1.0\\r\\n\\r\\n\" >&3\n\tread -r STATUS <&3\n\t[[ \"${STATUS}\" + == *\" 200 \"* ]]\nfi" + failureThreshold: 6 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + nvidia.com/lpu: "16" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsUser: 0 + startupProbe: + exec: + command: + - /bin/bash + - -c + - "set -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" + >&2\n\texit 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number + of rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME + is required when partition_models is configured\" >&2\n\t\texit + 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} + || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); + then\n\t\techo \"partition_models and partition_indices must match + the partition config row count\" >&2\n\t\texit 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node + count ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit + 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo + \"Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} + partition rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit + 1\nfi\nif ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo + \"Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for + config row ${CONFIG_PARTITION_INDEX}\" >&2\n\texit 1\nfi\n\nif (( + NODE_COUNT > 1 )) && [[ \"${PARTITION_RANK}\" != \"0\" ]]; then\n\tcat + < /dev/null > \"/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-2222}\"\nelse\n\texec + 3<>\"/dev/tcp/127.0.0.1/${READINESS_PORT:-19878}\"\n\tprintf \"GET + /readyz HTTP/1.0\\r\\n\\r\\n\" >&3\n\tread -r STATUS <&3\n\t[[ \"${STATUS}\" + == *\" 200 \"* ]]\nfi" + failureThreshold: 240 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + runAsGroup: 0 + runAsUser: 0 + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-hybrid-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - emptyDir: + medium: HugePages + name: hugepages + replicas: 1 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + cliqueNames: + - lpu-wkr-m-0 + - lpu-engine-gpu + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml new file mode 100644 index 000000000000..d403e3f6dc37 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -0,0 +1,36 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v3-hx-lpu-only + namespace: test + uid: 33333333-3333-4333-8333-333333333333 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v3-hx-connected + roles: + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/llama32-1b-lp30 + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - mountPath: /nfs + name: model-storage + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml new file mode 100644 index 000000000000..893b9295dd89 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -0,0 +1,551 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + model_config.toml: | + type = 'Single' + + [iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/nfs/node-local-v3-hx-connected' + num_kv_caches = 1 + output_size = 1 + prop_sync = false + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/nfs/node-local-v3-hx-connected/tokenizer' + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: "1" + partition_ids: "1" + partition_indices: "0" + partition_models: default + partition_node_offsets: "0" + partition_paths: part-1 + topologies: opaque-v3-topology +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + name: node-local-v3-hx-lpu-only-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only + name: node-local-071e1c6d-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v3-hx-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-lpu-only-lpu + name: lpu-ldr + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" + "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing + GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" + -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" + /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - -- + - --instance-model-name + - test/llama32-1b-lp30 + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /tmp/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - lpu-wkr-m-0 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + - --agent-env-vars + - BMC_SSH_ENABLED=false + - --agent-env-vars + - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 + - --agent-env-vars + - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false + - --agent-env-vars + - GROQ_HAL_V2_BACKGROUND_METRICS=false + - --agent-env-vars + - GROQ_PG613_PORT_CONFIG=EB3 + - --agent-env-vars + - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false + - --agent-env-vars + - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 + - --agent-env-vars + - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 + - --agent-env-vars + - GROQ_V2_RESET_ON_OPEN=0 + - --agent-env-vars + - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false + - --agent-env-vars + - LPU_FPGA_RECONFIGURE_METHOD=cvp + - --agent-env-vars + - LPU_PARTITION_RUNNER_DATAPLANE=pcie + - --agent-env-vars + - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 + - --agent-env-vars + - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 + - --agent-env-vars + - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true + - --agent-env-vars + - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 + - --agent-env-vars + - REDFISH_BMC_ENABLED=false + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + initContainers: + - args: + - -c + - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; + chmod 600 /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/bash + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-lpu-only-lpu + name: config + - emptyDir: {} + name: tmp + - name: agent-v2-ssh + secret: + defaultMode: 384 + secretName: ssh-secret + - emptyDir: {} + name: single-v2-ssh-key + replicas: 1 + roleName: lpu-ldr + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v3-hx-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-lpu-only-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ + \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; + then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: BMC_SSH_ENABLED + value: "false" + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: "false" + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: "false" + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: "0" + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: "0" + - name: GROQ_V2_RESET_ON_OPEN + value: "0" + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: "false" + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: REDFISH_BMC_ENABLED + value: "false" + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + nvidia.com/lpu: "16" + requests: + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + resources: {} + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-lpu-only-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: agent-v2-ssh + secret: + defaultMode: 420 + secretName: ssh-secret + replicas: 1 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + cliqueNames: + - lpu-ldr + - lpu-wkr-m-0 + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml new file mode 100644 index 000000000000..5424c62339a3 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -0,0 +1,62 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: node-local-v3-hx-specdecode + namespace: test + uid: 44444444-4444-4444-8444-444444444444 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: draft + type: lpx + replicas: 2 + lpx: + buildId: node-local-v3-hx-sd-draft + roles: + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/v3-specdecode + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - name: model-storage + mountPath: /nfs + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: lpu + type: lpx + replicas: 1 + lpx: + buildId: node-local-v3-hx-sd-target + roles: + - name: leader + - name: worker + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/v3-specdecode + resources: + limits: + lpu.nvidia.com/devices: '1' + volumeMounts: + - name: model-storage + mountPath: /nfs + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml new file mode 100644 index 000000000000..bf9e5c092df0 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -0,0 +1,1083 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + + [datacenters.racks.lpu-wkr-m-1] + available_nodes = [0] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-1-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + + [datacenters.racks.lpu-wkr-m-2] + available_nodes = [0] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-2-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + model_config.toml: | + draft_to_target_port = 23456 + head_to_head_port = 23458 + num_drafts = 2 + target_to_draft_port = 23457 + type = 'SpecDecode' + + [draft] + [draft.iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/nfs/node-local-v3-hx-sd-draft' + num_kv_caches = 1 + output_size = 1 + prop_sync = false + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/nfs/node-local-v3-hx-sd-draft/tokenizer' + + [draft.scheduler] + + [draft.setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + + [target] + [target.iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/nfs/node-local-v3-hx-sd-target' + num_kv_caches = 1 + output_size = 1 + prop_sync = false + sequence_length = 8192 + stop_tokens = [1] + tokenizer_path = '/nfs/node-local-v3-hx-sd-target/tokenizer' + + [target.scheduler] + + [target.setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: |- + 1 + 1 + 1 + partition_ids: |- + 1 + 1 + 1 + partition_indices: |- + 0 + 0 + 0 + partition_models: |- + draft0 + draft1 + target + partition_node_offsets: |- + 0 + 0 + 0 + partition_paths: |- + part-1 + part-1 + part-1 + topologies: |- + opaque-v3-topology + opaque-v3-topology + opaque-v3-topology +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + name: node-local-v3-hx-specdecode-lpu + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + labels: + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + name: node-local-75563e58-lpx + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-specdecode-lpu + name: lpu-ldr + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" + "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing + GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" + -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" + /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - -- + - --instance-model-name + - test/v3-specdecode + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /tmp/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - lpu-wkr-m-0:lpu-wkr-m-1:lpu-wkr-m-2 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + - --agent-env-vars + - BMC_SSH_ENABLED=false + - --agent-env-vars + - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 + - --agent-env-vars + - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false + - --agent-env-vars + - GROQ_HAL_V2_BACKGROUND_METRICS=false + - --agent-env-vars + - GROQ_PG613_PORT_CONFIG=EB3 + - --agent-env-vars + - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false + - --agent-env-vars + - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 + - --agent-env-vars + - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 + - --agent-env-vars + - GROQ_V2_RESET_ON_OPEN=0 + - --agent-env-vars + - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false + - --agent-env-vars + - LPU_FPGA_RECONFIGURE_METHOD=cvp + - --agent-env-vars + - LPU_PARTITION_RUNNER_DATAPLANE=pcie + - --agent-env-vars + - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 + - --agent-env-vars + - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 + - --agent-env-vars + - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true + - --agent-env-vars + - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 + - --agent-env-vars + - REDFISH_BMC_ENABLED=false + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + initContainers: + - args: + - -c + - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; + chmod 600 /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/bash + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - name: agent-v2-ssh + secret: + defaultMode: 384 + secretName: ssh-secret + - emptyDir: {} + name: single-v2-ssh-key + replicas: 1 + roleName: lpu-ldr + startsAfter: + - lpu-wkr-m-0 + - lpu-wkr-m-1 + - lpu-wkr-m-2 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea + scheduling.lpu.nvidia.com/model: draft0 + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-specdecode-draft + name: lpu-wkr-m-0 + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ + \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; + then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: BMC_SSH_ENABLED + value: "false" + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: "false" + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: "false" + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: "0" + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: "0" + - name: GROQ_V2_RESET_ON_OPEN + value: "0" + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: "false" + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: LPU_MODEL_NAME + value: draft0 + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: REDFISH_BMC_ENABLED + value: "false" + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + nvidia.com/lpu: "16" + requests: + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + resources: {} + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: agent-v2-ssh + secret: + defaultMode: 420 + secretName: ssh-secret + replicas: 1 + roleName: lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed + scheduling.lpu.nvidia.com/model: draft1 + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-specdecode-draft + name: lpu-wkr-m-1 + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ + \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; + then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: BMC_SSH_ENABLED + value: "false" + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: "false" + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: "false" + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: "0" + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: "0" + - name: GROQ_V2_RESET_ON_OPEN + value: "0" + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: "false" + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: LPU_MODEL_NAME + value: draft1 + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: REDFISH_BMC_ENABLED + value: "false" + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + nvidia.com/lpu: "16" + requests: + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + resources: {} + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: agent-v2-ssh + secret: + defaultMode: 420 + secretName: ssh-secret + replicas: 1 + roleName: lpu-wkr-m-1 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a + scheduling.lpu.nvidia.com/model: target + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-specdecode-lpu + name: lpu-wkr-m-2 + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ + \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; + then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: BMC_SSH_ENABLED + value: "false" + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: "false" + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: "false" + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: "0" + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: "0" + - name: GROQ_V2_RESET_ON_OPEN + value: "0" + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: "false" + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: LPU_MODEL_NAME + value: target + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: REDFISH_BMC_ENABLED + value: "false" + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + nvidia.com/lpu: "16" + requests: + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: agent-v2-ssh + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + resources: {} + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-specdecode-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: agent-v2-ssh + secret: + defaultMode: 420 + secretName: ssh-secret + replicas: 1 + roleName: lpu-wkr-m-2 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + cliqueNames: + - lpu-ldr + - lpu-wkr-m-0 + - lpu-wkr-m-1 + - lpu-wkr-m-2 + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml new file mode 100644 index 000000000000..48b8070c5a8f --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml @@ -0,0 +1,47 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler + generation: 1 + name: dgd-single-v2 + namespace: inference-engine + uid: 11111111-1111-4111-8111-111111111111 +spec: + components: + - name: lpu + type: lpx + extraPodSpecMergeStrategy: strategic + replicas: 1 + lpx: + buildId: llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c + settings: + batch_size: 1 + input_size: 1 + num_kv_caches: 1 + sequence_length: 8192 + stop_tokens: + - 128001 + - 128008 + - 128009 + tokenizer_path: llama3.1 + vocab_size: 128256 + roles: + - name: leader + - name: worker + podTemplate: + spec: + containers: + - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: main + args: + - --agent-env-vars + - EXTRA_RUNTIME_KNOB + env: + - name: MPI_BTL_TCP_IF_INCLUDE + value: 7.241.144.0/24 + - name: OMPI_MCA_btl_tcp_if_include + value: 7.241.144.0/24 + volumeMounts: + - mountPath: /nfs + name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml new file mode 100644 index 000000000000..e91708c3d9e1 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -0,0 +1,604 @@ +data: + datacenter.toml: | + model_base_paths = ['/nfs'] + + [datacenters] + [datacenters.racks] + [datacenters.racks.lpu-wkr-m-0] + available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + preferred_start = 0 + model_config.toml: | + type = 'Single' + + [iop] + batch_size = 1 + cpu_embeddings = false + dkvc = false + input_size = 1 + model_path = '/testdata/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c' + num_kv_caches = 1 + output_size = 1 + prop_sync = false + sequence_length = 8192 + stop_tokens = [128001, 128008, 128009] + tokenizer_path = 'llama3.1' + vocab_size = 128256 + + [scheduler] + + [setup] + resolved_partitions_dir = '/configs' + setup_ops_format = 'agent_v2' + nodes_per_partition: |- + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + partition_ids: |- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + partition_node_offsets: |- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + partition_paths: |- + part-0 + part-1 + part-2 + part-3 + part-4 + part-5 + part-6 + part-7 + part-8 + part-9 + part-10 + part-11 + part-12 + part-13 + part-14 + part-15 + part-16 + topologies: |- + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 + URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + name: dgd-single-v2-lpu + namespace: inference-engine +--- +metadata: + annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + labels: + nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 + name: dgd-single-c76fa0f7-lpx + namespace: inference-engine +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 + nvidia.com/dynamo-namespace: inference-engine-dgd-single-v2 + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-single-v2-lpu + name: lpu-ldr + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - -c + - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" + "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing + GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" + -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" + /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - -- + - --agent-env-vars + - EXTRA_RUNTIME_KNOB + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /tmp/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - lpu-wkr-m-0 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: inference-engine-dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: ETCD_ENDPOINTS + value: etcd-address + - name: MPI_BTL_TCP_IF_INCLUDE + value: 7.241.144.0/24 + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_btl_tcp_if_include + value: 7.241.144.0/24 + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + initContainers: + - args: + - -c + - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; + chmod 600 /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/bash + image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: dgd-single-v2-lpu + name: config + - emptyDir: {} + name: tmp + - name: ssh-secret + secret: + defaultMode: 384 + secretName: ssh-secret + - emptyDir: {} + name: single-v2-ssh-key + replicas: 1 + roleName: lpu-ldr + startsAfter: + - lpu-wkr-m-0 + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 + nvidia.com/dynamo-namespace: inference-engine-dgd-single-v2 + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-single-v2-lpu + name: lpu-wkr-m-0 + spec: + minAvailable: 17 + podSpec: + containers: + - args: + - -c + - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd + ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p + /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte + || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init + >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor + attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt + ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif + (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset + +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset + -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" + | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo + \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" + != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt + == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo + \"lpu-system-init exited with status ${init_status}; retrying with remediation\" + >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; + retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif + (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo + \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit + 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing + without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif + ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX + must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit + 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray + -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif + (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} + != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition + and partition_node_offsets must contain the same non-zero number of + rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" + ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray + -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif + [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is + required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} + != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models + and partition_indices must match the partition config row count\" >&2\n\t\texit + 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor + i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif + ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" + =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count + ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif + (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" + != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX + < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count + )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX + - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove + pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition + rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif + ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid + logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" + >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime + partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport + RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif + [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit + -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls + -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub + $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod + 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf + 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort + 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key + -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key + -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary + in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped + /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" + ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap + cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep + \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down + information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} + RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" + ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec + /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o + \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes + no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no + \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h + \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" + \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + command: + - /bin/bash + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: inference-engine-dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: MPI_BTL_TCP_IF_INCLUDE + value: 7.241.144.0/24 + - name: NATS_SERVER + value: nats-address + - name: OMPI_MCA_btl_tcp_if_include + value: 7.241.144.0/24 + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + stdin: true + tty: true + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + - mountPath: /configs + name: config + - mountPath: /tmp + name: tmp + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: dgd-single-v2-lpu + name: config + - emptyDir: {} + name: tmp + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + replicas: 17 + roleName: lpu-wkr-m-0 + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-uid: lpx-render-uid + lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + scheduling.lpu.nvidia.com/dgd-generation: "1" + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + cliqueNames: + - lpu-ldr + - lpu-wkr-m-0 + minAvailable: 1 + name: lpu + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml new file mode 100644 index 000000000000..819f9c7e2dc5 --- /dev/null +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml @@ -0,0 +1,28 @@ +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler + generation: 1 + name: dgd-single-v2-manifest-defaults + namespace: inference-engine + uid: 22222222-2222-4222-8222-222222222222 +spec: + components: + - name: lpx + type: lpx + extraPodSpecMergeStrategy: strategic + replicas: 1 + lpx: + buildId: single-v2-manifest-defaults/build_manifest_defaults + roles: + - name: leader + - name: worker + podTemplate: + spec: + containers: + - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: main + volumeMounts: + - mountPath: /nfs + name: model-storage diff --git a/deploy/operator/internal/features/gates.go b/deploy/operator/internal/features/gates.go index 0e2b3ff24eef..021782def401 100644 --- a/deploy/operator/internal/features/gates.go +++ b/deploy/operator/internal/features/gates.go @@ -130,6 +130,7 @@ const ( var allNames = [...]Name{ Checkpoint, Grove, + LPX, LWS, KaiScheduler, VolcanoScheduler, @@ -147,6 +148,7 @@ type Gate interface { type Gates struct { Checkpoint bool `json:"checkpoint"` Grove bool `json:"grove"` + LPX bool `json:"lpx"` LWS bool `json:"lws"` KaiScheduler bool `json:"kaiScheduler"` VolcanoScheduler bool `json:"volcanoScheduler"` @@ -196,6 +198,13 @@ func New(ctx context.Context, mgr ctrl.Manager, config *configv1alpha1.OperatorC return Gates{}, err } + // Enable LPX only when explicitly configured and its external API dependency is available. + if config.LPX.Enabled { + if gates.LPX, err = resolveLPX(ctx, mgr.GetConfig()); err != nil { + return Gates{}, err + } + } + lwsAvailable, err := detectAPIAvailability(ctx, mgr.GetConfig(), "leaderworkerset.x-k8s.io", "", "") if err != nil { return Gates{}, err diff --git a/deploy/operator/internal/features/gates_test.go b/deploy/operator/internal/features/gates_test.go index 300e20b1a696..51c66bbf1bf2 100644 --- a/deploy/operator/internal/features/gates_test.go +++ b/deploy/operator/internal/features/gates_test.go @@ -58,6 +58,7 @@ func allEnabledGates() Gates { return Gates{ Checkpoint: true, Grove: true, + LPX: true, LWS: true, KaiScheduler: true, VolcanoScheduler: true, diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go new file mode 100644 index 000000000000..755ccbd8b886 --- /dev/null +++ b/deploy/operator/internal/features/lpx.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package features + +import ( + "context" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" +) + +// LPX enables LPX scheduler integration. +// +// Owner: @andrewpaprotsky +// Experimental since: N/A +// Beta since: N/A +// GA since: N/A +// Configuration: lpx.enabled +// Auto-detection: N/A; API availability is verified when explicitly enabled +// Requires: LPX Scheduler serving scheduling.lpu.nvidia.com/v1alpha1 +// LpuPipelineRequest resources +// Default: false +const LPX Name = "lpx" + +// resolveLPX verifies API availability for explicitly enabled LPX integration. +// A nil config retains the native discovery-unavailable error. +func resolveLPX(ctx context.Context, config *rest.Config) (bool, error) { + // Discover the exact request resource before resolving the enabled gate. + resource := lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests") + available, err := detectAPIAvailability(ctx, config, resource.Group, resource.Version, resource.Resource) + if err != nil { + return false, err + } + + return resolve(ptr.To(true), available, + "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster") +} diff --git a/deploy/operator/internal/observability/resource_counter.go b/deploy/operator/internal/observability/resource_counter.go index 92fcc5f68131..1a40e7a6dee5 100644 --- a/deploy/operator/internal/observability/resource_counter.go +++ b/deploy/operator/internal/observability/resource_counter.go @@ -142,7 +142,7 @@ func countResourcesByState[T any, PT StateProvider]( } func updateDynamoGraphDeploymentCounts(ctx context.Context, c client.Client, excludedNamespaces ExcludedNamespaces, logger logr.Logger) { - dgdList := &v1alpha1.DynamoGraphDeploymentList{} + dgdList := &v1beta1.DynamoGraphDeploymentList{} if err := c.List(ctx, dgdList); err != nil { logger.Error(err, "failed to list DynamoGraphDeployments") return @@ -151,12 +151,12 @@ func updateDynamoGraphDeploymentCounts(ctx context.Context, c client.Client, exc dgdList.Items, excludedNamespaces, consts.ResourceTypeDynamoGraphDeployment, - func(d *v1alpha1.DynamoGraphDeployment) *v1alpha1.DynamoGraphDeployment { return d }, + func(d *v1beta1.DynamoGraphDeployment) *v1beta1.DynamoGraphDeployment { return d }, ) } func updateDynamoComponentDeploymentCounts(ctx context.Context, c client.Client, excludedNamespaces ExcludedNamespaces, logger logr.Logger) { - dcdList := &v1alpha1.DynamoComponentDeploymentList{} + dcdList := &v1beta1.DynamoComponentDeploymentList{} if err := c.List(ctx, dcdList); err != nil { logger.Error(err, "failed to list DynamoComponentDeployments") return @@ -165,7 +165,7 @@ func updateDynamoComponentDeploymentCounts(ctx context.Context, c client.Client, dcdList.Items, excludedNamespaces, consts.ResourceTypeDynamoComponentDeployment, - func(d *v1alpha1.DynamoComponentDeployment) *v1alpha1.DynamoComponentDeployment { return d }, + func(d *v1beta1.DynamoComponentDeployment) *v1beta1.DynamoComponentDeployment { return d }, ) } diff --git a/deploy/operator/internal/observability/resource_counter_test.go b/deploy/operator/internal/observability/resource_counter_test.go index 77d157f0ca99..346bdf85bcd5 100644 --- a/deploy/operator/internal/observability/resource_counter_test.go +++ b/deploy/operator/internal/observability/resource_counter_test.go @@ -18,11 +18,27 @@ package observability import ( + "context" + "reflect" "testing" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" ) +type listRecordingClient struct { + client.Client + listed client.ObjectList +} + +func (c *listRecordingClient) List(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + c.listed = list + return nil +} + func TestResourceCounterDoesNotNeedLeaderElection(t *testing.T) { counter := NewResourceCounter(fake.NewClientBuilder().Build(), nil) @@ -30,3 +46,33 @@ func TestResourceCounterDoesNotNeedLeaderElection(t *testing.T) { t.Fatal("ResourceCounter must run on every replica") } } + +func TestDeploymentResourceCountersListV1Beta1(t *testing.T) { + tests := []struct { + name string + update func(context.Context, client.Client, ExcludedNamespaces, logr.Logger) + wantList client.ObjectList + }{ + { + name: "DynamoGraphDeployment", + update: updateDynamoGraphDeploymentCounts, + wantList: &v1beta1.DynamoGraphDeploymentList{}, + }, + { + name: "DynamoComponentDeployment", + update: updateDynamoComponentDeploymentCounts, + wantList: &v1beta1.DynamoComponentDeploymentList{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c := &listRecordingClient{Client: fake.NewClientBuilder().Build()} + test.update(context.Background(), c, nil, logr.Discard()) + + if got, want := reflect.TypeOf(c.listed), reflect.TypeOf(test.wantList); got != want { + t.Fatalf("listed %v, want %v", got, want) + } + }) + } +} diff --git a/deploy/operator/internal/podcache/transform.go b/deploy/operator/internal/podcache/transform.go index d9aa5154ddea..daf6295b10eb 100644 --- a/deploy/operator/internal/podcache/transform.go +++ b/deploy/operator/internal/podcache/transform.go @@ -83,6 +83,9 @@ func projectSpec(in corev1.PodSpec) corev1.PodSpec { return corev1.PodSpec{ // Topology and snapshot controllers require the assigned node. NodeName: in.NodeName, + // Node-local LPX disruption recovery distinguishes Agent Pods scheduled + // through LPX from unrelated Pods. + SchedulerName: in.SchedulerName, // Model endpoint classification requires the main container's command // and arguments. Containers: projectContainers(in.Containers), @@ -165,8 +168,9 @@ func projectStatus(in corev1.PodStatus) corev1.PodStatus { // Failover, DGDR diagnostics, and Recreate drain barriers require the // lifecycle phase. Phase: in.Phase, - // Model endpoint classification requires Ready; Snapshot-aware GMS - // replacement requires the public restore outcome. + // Model endpoint classification requires Ready; Snapshot-aware GMS replacement + // requires the public restore outcome; node-local LPX disruption recovery + // requires DisruptionTarget and its Kubernetes reason. Conditions: projectConditions(in.Conditions), // DGDR diagnostics inspect container failures. ContainerStatuses: projectContainerStatuses(in.ContainerStatuses), @@ -178,16 +182,16 @@ func projectStatus(in corev1.PodStatus) corev1.PodStatus { func projectConditions(in []corev1.PodCondition) []corev1.PodCondition { out := make([]corev1.PodCondition, 0, len(in)) for i := range in { - if in[i].Type != corev1.PodReady && - in[i].Type != corev1.PodConditionType(podcontract.RestoredCondition) { - continue - } condition := corev1.PodCondition{ Type: in[i].Type, Status: in[i].Status, } - if in[i].Type == corev1.PodConditionType(podcontract.RestoredCondition) { + switch in[i].Type { + case corev1.PodReady: + case corev1.PodConditionType(podcontract.RestoredCondition), corev1.DisruptionTarget: condition.Reason = in[i].Reason + default: + continue } out = append(out, condition) } diff --git a/deploy/operator/internal/podcache/transform_test.go b/deploy/operator/internal/podcache/transform_test.go index ce7600a8fa71..7afc9fae1ae9 100644 --- a/deploy/operator/internal/podcache/transform_test.go +++ b/deploy/operator/internal/podcache/transform_test.go @@ -41,7 +41,12 @@ func TestProjectConsumerContract(t *testing.T) { ManagedFields: []metav1.ManagedFieldsEntry{{Manager: "large-manager"}}, }, Spec: corev1.PodSpec{ - NodeName: "node-a", + NodeName: "node-a", + SchedulerName: "lpx-scheduler", + ResourceClaims: []corev1.PodResourceClaim{{ + Name: "lpu-partition", + ResourceClaimName: ptr.To("lpu-partition-0"), + }}, Containers: []corev1.Container{{ Name: "main", Image: "large-image", @@ -102,6 +107,7 @@ func TestProjectConsumerContract(t *testing.T) { Reason: podcontract.RestoreReasonFailed, Message: "discard-me", }, + {Type: corev1.DisruptionTarget, Status: corev1.ConditionTrue, Reason: "EvictionByEvictionAPI", Message: "discard-me"}, }, ContainerStatuses: []corev1.ContainerStatus{ { @@ -155,6 +161,9 @@ func TestProjectConsumerContract(t *testing.T) { t.Run("model retains Ready identity command and arguments", func(t *testing.T) { require.Len(t, got.Spec.Containers, 1) assert.Equal(t, corev1.Container{Name: "main", Command: []string{"python"}, Args: []string{"-m", "dynamo"}}, got.Spec.Containers[0]) + }) + t.Run("snapshot and LPX retain restore and disruption state", func(t *testing.T) { + assert.Equal(t, "lpx-scheduler", got.Spec.SchedulerName) assert.Equal(t, []corev1.PodCondition{ {Type: corev1.PodReady, Status: corev1.ConditionTrue}, { @@ -162,6 +171,7 @@ func TestProjectConsumerContract(t *testing.T) { Status: corev1.ConditionFalse, Reason: podcontract.RestoreReasonFailed, }, + {Type: corev1.DisruptionTarget, Status: corev1.ConditionTrue, Reason: "EvictionByEvictionAPI"}, }, got.Status.Conditions) }) t.Run("failover DGDR GMS replacement and Recreate retain status state", func(t *testing.T) { diff --git a/deploy/operator/internal/thirdparty/capnp/LICENSE b/deploy/operator/internal/thirdparty/capnp/LICENSE new file mode 100644 index 000000000000..261eeb9e9f8b --- /dev/null +++ b/deploy/operator/internal/thirdparty/capnp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go b/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go new file mode 100644 index 000000000000..38d89a015f62 --- /dev/null +++ b/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go @@ -0,0 +1,2656 @@ +// Code generated by capnpc-go. DO NOT EDIT. + +package gbuildmanifestcapnpv2 + +import ( + capnp "capnproto.org/go/capnp/v3" + math "math" + strconv "strconv" +) + +// Constants defined in gbuild_manifest.capnp. +const ( + CurrentContractRevision = uint32(2) +) + +type Manifest capnp.Struct + +// Manifest_TypeID is the unique identifier for the type Manifest. +const Manifest_TypeID = 0xdfeff145e86a5a8f + +func NewManifest(s *capnp.Segment) (Manifest, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return Manifest(st), err +} + +func NewRootManifest(s *capnp.Segment) (Manifest, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return Manifest(st), err +} + +func ReadRootManifest(msg *capnp.Message) (Manifest, error) { + root, err := msg.Root() + return Manifest(root.Struct()), err +} + +func (s Manifest) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (Manifest) DecodeFromPtr(p capnp.Ptr) Manifest { + return Manifest(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s Manifest) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s Manifest) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s Manifest) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s Manifest) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s Manifest) Model() (ModelInfo, error) { + p, err := capnp.Struct(s).Ptr(0) + return ModelInfo(p.Struct()), err +} + +func (s Manifest) HasModel() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s Manifest) SetModel(v ModelInfo) error { + return capnp.Struct(s).SetPtr(0, capnp.Struct(v).ToPtr()) +} + +// NewModel sets the model field to a newly +// allocated ModelInfo struct, preferring placement in s's segment. +func (s Manifest) NewModel() (ModelInfo, error) { + ss, err := NewModelInfo(capnp.Struct(s).Segment()) + if err != nil { + return ModelInfo{}, err + } + err = capnp.Struct(s).SetPtr(0, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s Manifest) Deployment() (DeploymentInfo, error) { + p, err := capnp.Struct(s).Ptr(1) + return DeploymentInfo(p.Struct()), err +} + +func (s Manifest) HasDeployment() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s Manifest) SetDeployment(v DeploymentInfo) error { + return capnp.Struct(s).SetPtr(1, capnp.Struct(v).ToPtr()) +} + +// NewDeployment sets the deployment field to a newly +// allocated DeploymentInfo struct, preferring placement in s's segment. +func (s Manifest) NewDeployment() (DeploymentInfo, error) { + ss, err := NewDeploymentInfo(capnp.Struct(s).Segment()) + if err != nil { + return DeploymentInfo{}, err + } + err = capnp.Struct(s).SetPtr(1, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s Manifest) Artifacts() (ArtifactInfo, error) { + p, err := capnp.Struct(s).Ptr(2) + return ArtifactInfo(p.Struct()), err +} + +func (s Manifest) HasArtifacts() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s Manifest) SetArtifacts(v ArtifactInfo) error { + return capnp.Struct(s).SetPtr(2, capnp.Struct(v).ToPtr()) +} + +// NewArtifacts sets the artifacts field to a newly +// allocated ArtifactInfo struct, preferring placement in s's segment. +func (s Manifest) NewArtifacts() (ArtifactInfo, error) { + ss, err := NewArtifactInfo(capnp.Struct(s).Segment()) + if err != nil { + return ArtifactInfo{}, err + } + err = capnp.Struct(s).SetPtr(2, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s Manifest) Reserved3() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(3) +} + +func (s Manifest) HasReserved3() bool { + return capnp.Struct(s).HasPtr(3) +} + +func (s Manifest) SetReserved3(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(3, v) +} +func (s Manifest) ContractRevision() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s Manifest) SetContractRevision(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +// Manifest_List is a list of Manifest. +type Manifest_List = capnp.StructList[Manifest] + +// NewManifest creates a new list of Manifest. +func NewManifest_List(s *capnp.Segment, sz int32) (Manifest_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}, sz) + return capnp.StructList[Manifest](l), err +} + +// Manifest_Future is a wrapper for a Manifest promised by a client call. +type Manifest_Future struct{ *capnp.Future } + +func (f Manifest_Future) Struct() (Manifest, error) { + p, err := f.Future.Ptr() + return Manifest(p.Struct()), err +} +func (p Manifest_Future) Model() ModelInfo_Future { + return ModelInfo_Future{Future: p.Future.Field(0, nil)} +} +func (p Manifest_Future) Deployment() DeploymentInfo_Future { + return DeploymentInfo_Future{Future: p.Future.Field(1, nil)} +} +func (p Manifest_Future) Artifacts() ArtifactInfo_Future { + return ArtifactInfo_Future{Future: p.Future.Field(2, nil)} +} +func (p Manifest_Future) Reserved3() *capnp.Future { + return p.Future.Field(3, nil) +} + +type ModelInfo capnp.Struct + +// ModelInfo_TypeID is the unique identifier for the type ModelInfo. +const ModelInfo_TypeID = 0xf647409b6c50c2b2 + +func NewModelInfo(s *capnp.Segment) (ModelInfo, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return ModelInfo(st), err +} + +func NewRootModelInfo(s *capnp.Segment) (ModelInfo, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return ModelInfo(st), err +} + +func ReadRootModelInfo(msg *capnp.Message) (ModelInfo, error) { + root, err := msg.Root() + return ModelInfo(root.Struct()), err +} + +func (s ModelInfo) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (ModelInfo) DecodeFromPtr(p capnp.Ptr) ModelInfo { + return ModelInfo(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s ModelInfo) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s ModelInfo) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s ModelInfo) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s ModelInfo) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s ModelInfo) Arch() (ArchConfig, error) { + p, err := capnp.Struct(s).Ptr(0) + return ArchConfig(p.Struct()), err +} + +func (s ModelInfo) HasArch() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s ModelInfo) SetArch(v ArchConfig) error { + return capnp.Struct(s).SetPtr(0, capnp.Struct(v).ToPtr()) +} + +// NewArch sets the arch field to a newly +// allocated ArchConfig struct, preferring placement in s's segment. +func (s ModelInfo) NewArch() (ArchConfig, error) { + ss, err := NewArchConfig(capnp.Struct(s).Segment()) + if err != nil { + return ArchConfig{}, err + } + err = capnp.Struct(s).SetPtr(0, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ModelInfo) Tokenizer() (TokenizerInfo, error) { + p, err := capnp.Struct(s).Ptr(1) + return TokenizerInfo(p.Struct()), err +} + +func (s ModelInfo) HasTokenizer() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s ModelInfo) SetTokenizer(v TokenizerInfo) error { + return capnp.Struct(s).SetPtr(1, capnp.Struct(v).ToPtr()) +} + +// NewTokenizer sets the tokenizer field to a newly +// allocated TokenizerInfo struct, preferring placement in s's segment. +func (s ModelInfo) NewTokenizer() (TokenizerInfo, error) { + ss, err := NewTokenizerInfo(capnp.Struct(s).Segment()) + if err != nil { + return TokenizerInfo{}, err + } + err = capnp.Struct(s).SetPtr(1, capnp.Struct(ss).ToPtr()) + return ss, err +} + +// ModelInfo_List is a list of ModelInfo. +type ModelInfo_List = capnp.StructList[ModelInfo] + +// NewModelInfo creates a new list of ModelInfo. +func NewModelInfo_List(s *capnp.Segment, sz int32) (ModelInfo_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz) + return capnp.StructList[ModelInfo](l), err +} + +// ModelInfo_Future is a wrapper for a ModelInfo promised by a client call. +type ModelInfo_Future struct{ *capnp.Future } + +func (f ModelInfo_Future) Struct() (ModelInfo, error) { + p, err := f.Future.Ptr() + return ModelInfo(p.Struct()), err +} +func (p ModelInfo_Future) Arch() ArchConfig_Future { + return ArchConfig_Future{Future: p.Future.Field(0, nil)} +} +func (p ModelInfo_Future) Tokenizer() TokenizerInfo_Future { + return TokenizerInfo_Future{Future: p.Future.Field(1, nil)} +} + +type TokenizerInfo capnp.Struct + +// TokenizerInfo_TypeID is the unique identifier for the type TokenizerInfo. +const TokenizerInfo_TypeID = 0xa26e80e8f5cd8871 + +func NewTokenizerInfo(s *capnp.Segment) (TokenizerInfo, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 3}) + return TokenizerInfo(st), err +} + +func NewRootTokenizerInfo(s *capnp.Segment) (TokenizerInfo, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 3}) + return TokenizerInfo(st), err +} + +func ReadRootTokenizerInfo(msg *capnp.Message) (TokenizerInfo, error) { + root, err := msg.Root() + return TokenizerInfo(root.Struct()), err +} + +func (s TokenizerInfo) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (TokenizerInfo) DecodeFromPtr(p capnp.Ptr) TokenizerInfo { + return TokenizerInfo(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s TokenizerInfo) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s TokenizerInfo) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s TokenizerInfo) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s TokenizerInfo) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s TokenizerInfo) Path() (string, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.Text(), err +} + +func (s TokenizerInfo) HasPath() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s TokenizerInfo) PathBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.TextBytes(), err +} + +func (s TokenizerInfo) SetPath(v string) error { + return capnp.Struct(s).SetText(0, v) +} + +func (s TokenizerInfo) StopTokens() (capnp.UInt32List, error) { + p, err := capnp.Struct(s).Ptr(1) + return capnp.UInt32List(p.List()), err +} + +func (s TokenizerInfo) HasStopTokens() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s TokenizerInfo) SetStopTokens(v capnp.UInt32List) error { + return capnp.Struct(s).SetPtr(1, v.ToPtr()) +} + +// NewStopTokens sets the stopTokens field to a newly +// allocated capnp.UInt32List, preferring placement in s's segment. +func (s TokenizerInfo) NewStopTokens(n int32) (capnp.UInt32List, error) { + l, err := capnp.NewUInt32List(capnp.Struct(s).Segment(), n) + if err != nil { + return capnp.UInt32List{}, err + } + err = capnp.Struct(s).SetPtr(1, l.ToPtr()) + return l, err +} +func (s TokenizerInfo) Reserved2() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(2) +} + +func (s TokenizerInfo) HasReserved2() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s TokenizerInfo) SetReserved2(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(2, v) +} +func (s TokenizerInfo) VocabSize() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s TokenizerInfo) SetVocabSize(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +func (s TokenizerInfo) Reserved4() uint16 { + return capnp.Struct(s).Uint16(4) +} + +func (s TokenizerInfo) SetReserved4(v uint16) { + capnp.Struct(s).SetUint16(4, v) +} + +// TokenizerInfo_List is a list of TokenizerInfo. +type TokenizerInfo_List = capnp.StructList[TokenizerInfo] + +// NewTokenizerInfo creates a new list of TokenizerInfo. +func NewTokenizerInfo_List(s *capnp.Segment, sz int32) (TokenizerInfo_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 3}, sz) + return capnp.StructList[TokenizerInfo](l), err +} + +// TokenizerInfo_Future is a wrapper for a TokenizerInfo promised by a client call. +type TokenizerInfo_Future struct{ *capnp.Future } + +func (f TokenizerInfo_Future) Struct() (TokenizerInfo, error) { + p, err := f.Future.Ptr() + return TokenizerInfo(p.Struct()), err +} +func (p TokenizerInfo_Future) Reserved2() *capnp.Future { + return p.Future.Field(2, nil) +} + +type ArchConfig capnp.Struct + +// ArchConfig_TypeID is the unique identifier for the type ArchConfig. +const ArchConfig_TypeID = 0xa59716d6849bbf90 + +func NewArchConfig(s *capnp.Segment) (ArchConfig, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 40, PointerCount: 6}) + return ArchConfig(st), err +} + +func NewRootArchConfig(s *capnp.Segment) (ArchConfig, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 40, PointerCount: 6}) + return ArchConfig(st), err +} + +func ReadRootArchConfig(msg *capnp.Message) (ArchConfig, error) { + root, err := msg.Root() + return ArchConfig(root.Struct()), err +} + +func (s ArchConfig) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (ArchConfig) DecodeFromPtr(p capnp.Ptr) ArchConfig { + return ArchConfig(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s ArchConfig) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s ArchConfig) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s ArchConfig) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s ArchConfig) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s ArchConfig) Reserved0() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s ArchConfig) SetReserved0(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +func (s ArchConfig) NLayers() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s ArchConfig) SetNLayers(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +func (s ArchConfig) Reserved2() uint32 { + return capnp.Struct(s).Uint32(8) +} + +func (s ArchConfig) SetReserved2(v uint32) { + capnp.Struct(s).SetUint32(8, v) +} + +func (s ArchConfig) Reserved3() uint32 { + return capnp.Struct(s).Uint32(12) +} + +func (s ArchConfig) SetReserved3(v uint32) { + capnp.Struct(s).SetUint32(12, v) +} + +func (s ArchConfig) Reserved4() uint32 { + return capnp.Struct(s).Uint32(16) +} + +func (s ArchConfig) SetReserved4(v uint32) { + capnp.Struct(s).SetUint32(16, v) +} + +func (s ArchConfig) Reserved5() uint32 { + return capnp.Struct(s).Uint32(20) +} + +func (s ArchConfig) SetReserved5(v uint32) { + capnp.Struct(s).SetUint32(20, v) +} + +func (s ArchConfig) VocabSize() uint32 { + return capnp.Struct(s).Uint32(24) +} + +func (s ArchConfig) SetVocabSize(v uint32) { + capnp.Struct(s).SetUint32(24, v) +} + +func (s ArchConfig) Reserved7() uint32 { + return capnp.Struct(s).Uint32(28) +} + +func (s ArchConfig) SetReserved7(v uint32) { + capnp.Struct(s).SetUint32(28, v) +} + +func (s ArchConfig) Reserved8() float32 { + return math.Float32frombits(capnp.Struct(s).Uint32(32)) +} + +func (s ArchConfig) SetReserved8(v float32) { + capnp.Struct(s).SetUint32(32, math.Float32bits(v)) +} + +func (s ArchConfig) Reserved9() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(0) +} + +func (s ArchConfig) HasReserved9() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s ArchConfig) SetReserved9(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(0, v) +} +func (s ArchConfig) Swa() (SwaConfig, error) { + p, err := capnp.Struct(s).Ptr(1) + return SwaConfig(p.Struct()), err +} + +func (s ArchConfig) HasSwa() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s ArchConfig) SetSwa(v SwaConfig) error { + return capnp.Struct(s).SetPtr(1, capnp.Struct(v).ToPtr()) +} + +// NewSwa sets the swa field to a newly +// allocated SwaConfig struct, preferring placement in s's segment. +func (s ArchConfig) NewSwa() (SwaConfig, error) { + ss, err := NewSwaConfig(capnp.Struct(s).Segment()) + if err != nil { + return SwaConfig{}, err + } + err = capnp.Struct(s).SetPtr(1, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ArchConfig) Reserved11() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(2) +} + +func (s ArchConfig) HasReserved11() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s ArchConfig) SetReserved11(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(2, v) +} +func (s ArchConfig) Reserved12() bool { + return capnp.Struct(s).Bit(288) +} + +func (s ArchConfig) SetReserved12(v bool) { + capnp.Struct(s).SetBit(288, v) +} + +func (s ArchConfig) Reserved13() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(3) +} + +func (s ArchConfig) HasReserved13() bool { + return capnp.Struct(s).HasPtr(3) +} + +func (s ArchConfig) SetReserved13(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(3, v) +} +func (s ArchConfig) Reserved14() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(4) +} + +func (s ArchConfig) HasReserved14() bool { + return capnp.Struct(s).HasPtr(4) +} + +func (s ArchConfig) SetReserved14(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(4, v) +} +func (s ArchConfig) Reserved15() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(5) +} + +func (s ArchConfig) HasReserved15() bool { + return capnp.Struct(s).HasPtr(5) +} + +func (s ArchConfig) SetReserved15(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(5, v) +} + +// ArchConfig_List is a list of ArchConfig. +type ArchConfig_List = capnp.StructList[ArchConfig] + +// NewArchConfig creates a new list of ArchConfig. +func NewArchConfig_List(s *capnp.Segment, sz int32) (ArchConfig_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 40, PointerCount: 6}, sz) + return capnp.StructList[ArchConfig](l), err +} + +// ArchConfig_Future is a wrapper for a ArchConfig promised by a client call. +type ArchConfig_Future struct{ *capnp.Future } + +func (f ArchConfig_Future) Struct() (ArchConfig, error) { + p, err := f.Future.Ptr() + return ArchConfig(p.Struct()), err +} +func (p ArchConfig_Future) Reserved9() *capnp.Future { + return p.Future.Field(0, nil) +} +func (p ArchConfig_Future) Swa() SwaConfig_Future { + return SwaConfig_Future{Future: p.Future.Field(1, nil)} +} +func (p ArchConfig_Future) Reserved11() *capnp.Future { + return p.Future.Field(2, nil) +} +func (p ArchConfig_Future) Reserved13() *capnp.Future { + return p.Future.Field(3, nil) +} +func (p ArchConfig_Future) Reserved14() *capnp.Future { + return p.Future.Field(4, nil) +} +func (p ArchConfig_Future) Reserved15() *capnp.Future { + return p.Future.Field(5, nil) +} + +type SwaConfig capnp.Struct + +// SwaConfig_TypeID is the unique identifier for the type SwaConfig. +const SwaConfig_TypeID = 0xe67869945d5b5a16 + +func NewSwaConfig(s *capnp.Segment) (SwaConfig, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1}) + return SwaConfig(st), err +} + +func NewRootSwaConfig(s *capnp.Segment) (SwaConfig, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1}) + return SwaConfig(st), err +} + +func ReadRootSwaConfig(msg *capnp.Message) (SwaConfig, error) { + root, err := msg.Root() + return SwaConfig(root.Struct()), err +} + +func (s SwaConfig) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (SwaConfig) DecodeFromPtr(p capnp.Ptr) SwaConfig { + return SwaConfig(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s SwaConfig) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s SwaConfig) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s SwaConfig) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s SwaConfig) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s SwaConfig) MaxSeqLen() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s SwaConfig) SetMaxSeqLen(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +func (s SwaConfig) Padding() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s SwaConfig) SetPadding(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +func (s SwaConfig) NumUsers() (OptionalUInt32, error) { + p, err := capnp.Struct(s).Ptr(0) + return OptionalUInt32(p.Struct()), err +} + +func (s SwaConfig) HasNumUsers() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s SwaConfig) SetNumUsers(v OptionalUInt32) error { + return capnp.Struct(s).SetPtr(0, capnp.Struct(v).ToPtr()) +} + +// NewNumUsers sets the numUsers field to a newly +// allocated OptionalUInt32 struct, preferring placement in s's segment. +func (s SwaConfig) NewNumUsers() (OptionalUInt32, error) { + ss, err := NewOptionalUInt32(capnp.Struct(s).Segment()) + if err != nil { + return OptionalUInt32{}, err + } + err = capnp.Struct(s).SetPtr(0, capnp.Struct(ss).ToPtr()) + return ss, err +} + +// SwaConfig_List is a list of SwaConfig. +type SwaConfig_List = capnp.StructList[SwaConfig] + +// NewSwaConfig creates a new list of SwaConfig. +func NewSwaConfig_List(s *capnp.Segment, sz int32) (SwaConfig_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1}, sz) + return capnp.StructList[SwaConfig](l), err +} + +// SwaConfig_Future is a wrapper for a SwaConfig promised by a client call. +type SwaConfig_Future struct{ *capnp.Future } + +func (f SwaConfig_Future) Struct() (SwaConfig, error) { + p, err := f.Future.Ptr() + return SwaConfig(p.Struct()), err +} +func (p SwaConfig_Future) NumUsers() OptionalUInt32_Future { + return OptionalUInt32_Future{Future: p.Future.Field(0, nil)} +} + +type DeploymentInfo capnp.Struct + +// DeploymentInfo_TypeID is the unique identifier for the type DeploymentInfo. +const DeploymentInfo_TypeID = 0xa151d4fb5c342435 + +func NewDeploymentInfo(s *capnp.Segment) (DeploymentInfo, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 3}) + return DeploymentInfo(st), err +} + +func NewRootDeploymentInfo(s *capnp.Segment) (DeploymentInfo, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 3}) + return DeploymentInfo(st), err +} + +func ReadRootDeploymentInfo(msg *capnp.Message) (DeploymentInfo, error) { + root, err := msg.Root() + return DeploymentInfo(root.Struct()), err +} + +func (s DeploymentInfo) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (DeploymentInfo) DecodeFromPtr(p capnp.Ptr) DeploymentInfo { + return DeploymentInfo(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s DeploymentInfo) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s DeploymentInfo) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s DeploymentInfo) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s DeploymentInfo) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s DeploymentInfo) CompilationMode() CompilationMode { + return CompilationMode(capnp.Struct(s).Uint16(0)) +} + +func (s DeploymentInfo) SetCompilationMode(v CompilationMode) { + capnp.Struct(s).SetUint16(0, uint16(v)) +} + +func (s DeploymentInfo) NumLpuNodes() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s DeploymentInfo) SetNumLpuNodes(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +func (s DeploymentInfo) Reserved2() uint32 { + return capnp.Struct(s).Uint32(8) +} + +func (s DeploymentInfo) SetReserved2(v uint32) { + capnp.Struct(s).SetUint32(8, v) +} + +func (s DeploymentInfo) Program() (ProgramConfig, error) { + p, err := capnp.Struct(s).Ptr(0) + return ProgramConfig(p.Struct()), err +} + +func (s DeploymentInfo) HasProgram() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s DeploymentInfo) SetProgram(v ProgramConfig) error { + return capnp.Struct(s).SetPtr(0, capnp.Struct(v).ToPtr()) +} + +// NewProgram sets the program field to a newly +// allocated ProgramConfig struct, preferring placement in s's segment. +func (s DeploymentInfo) NewProgram() (ProgramConfig, error) { + ss, err := NewProgramConfig(capnp.Struct(s).Segment()) + if err != nil { + return ProgramConfig{}, err + } + err = capnp.Struct(s).SetPtr(0, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s DeploymentInfo) SelectedPropSyncChains() (PropSyncChain_List, error) { + p, err := capnp.Struct(s).Ptr(1) + return PropSyncChain_List(p.List()), err +} + +func (s DeploymentInfo) HasSelectedPropSyncChains() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s DeploymentInfo) SetSelectedPropSyncChains(v PropSyncChain_List) error { + return capnp.Struct(s).SetPtr(1, v.ToPtr()) +} + +// NewSelectedPropSyncChains sets the selectedPropSyncChains field to a newly +// allocated PropSyncChain_List, preferring placement in s's segment. +func (s DeploymentInfo) NewSelectedPropSyncChains(n int32) (PropSyncChain_List, error) { + l, err := NewPropSyncChain_List(capnp.Struct(s).Segment(), n) + if err != nil { + return PropSyncChain_List{}, err + } + err = capnp.Struct(s).SetPtr(1, l.ToPtr()) + return l, err +} +func (s DeploymentInfo) RuntimeIo() (RuntimeIoConfig, error) { + p, err := capnp.Struct(s).Ptr(2) + return RuntimeIoConfig(p.Struct()), err +} + +func (s DeploymentInfo) HasRuntimeIo() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s DeploymentInfo) SetRuntimeIo(v RuntimeIoConfig) error { + return capnp.Struct(s).SetPtr(2, capnp.Struct(v).ToPtr()) +} + +// NewRuntimeIo sets the runtimeIo field to a newly +// allocated RuntimeIoConfig struct, preferring placement in s's segment. +func (s DeploymentInfo) NewRuntimeIo() (RuntimeIoConfig, error) { + ss, err := NewRuntimeIoConfig(capnp.Struct(s).Segment()) + if err != nil { + return RuntimeIoConfig{}, err + } + err = capnp.Struct(s).SetPtr(2, capnp.Struct(ss).ToPtr()) + return ss, err +} + +// DeploymentInfo_List is a list of DeploymentInfo. +type DeploymentInfo_List = capnp.StructList[DeploymentInfo] + +// NewDeploymentInfo creates a new list of DeploymentInfo. +func NewDeploymentInfo_List(s *capnp.Segment, sz int32) (DeploymentInfo_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 16, PointerCount: 3}, sz) + return capnp.StructList[DeploymentInfo](l), err +} + +// DeploymentInfo_Future is a wrapper for a DeploymentInfo promised by a client call. +type DeploymentInfo_Future struct{ *capnp.Future } + +func (f DeploymentInfo_Future) Struct() (DeploymentInfo, error) { + p, err := f.Future.Ptr() + return DeploymentInfo(p.Struct()), err +} +func (p DeploymentInfo_Future) Program() ProgramConfig_Future { + return ProgramConfig_Future{Future: p.Future.Field(0, nil)} +} +func (p DeploymentInfo_Future) RuntimeIo() RuntimeIoConfig_Future { + return RuntimeIoConfig_Future{Future: p.Future.Field(2, nil)} +} + +type CompilationMode uint16 + +// CompilationMode_TypeID is the unique identifier for the type CompilationMode. +const CompilationMode_TypeID = 0xf66fba7eaeef9fd8 + +// Values of CompilationMode. +const ( + CompilationMode_lpuOnly CompilationMode = 0 + CompilationMode_lpx CompilationMode = 1 +) + +// String returns the enum's constant name. +func (c CompilationMode) String() string { + switch c { + case CompilationMode_lpuOnly: + return "lpuOnly" + case CompilationMode_lpx: + return "lpx" + + default: + return "" + } +} + +// CompilationModeFromString returns the enum value with a name, +// or the zero value if there's no such value. +func CompilationModeFromString(c string) CompilationMode { + switch c { + case "lpuOnly": + return CompilationMode_lpuOnly + case "lpx": + return CompilationMode_lpx + + default: + return 0 + } +} + +type CompilationMode_List = capnp.EnumList[CompilationMode] + +func NewCompilationMode_List(s *capnp.Segment, sz int32) (CompilationMode_List, error) { + return capnp.NewEnumList[CompilationMode](s, sz) +} + +type ProgramConfig capnp.Struct + +// ProgramConfig_TypeID is the unique identifier for the type ProgramConfig. +const ProgramConfig_TypeID = 0xc22680a74d7bbfe2 + +func NewProgramConfig(s *capnp.Segment) (ProgramConfig, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 32, PointerCount: 4}) + return ProgramConfig(st), err +} + +func NewRootProgramConfig(s *capnp.Segment) (ProgramConfig, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 32, PointerCount: 4}) + return ProgramConfig(st), err +} + +func ReadRootProgramConfig(msg *capnp.Message) (ProgramConfig, error) { + root, err := msg.Root() + return ProgramConfig(root.Struct()), err +} + +func (s ProgramConfig) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (ProgramConfig) DecodeFromPtr(p capnp.Ptr) ProgramConfig { + return ProgramConfig(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s ProgramConfig) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s ProgramConfig) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s ProgramConfig) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s ProgramConfig) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s ProgramConfig) ProgramName() (string, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.Text(), err +} + +func (s ProgramConfig) HasProgramName() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s ProgramConfig) ProgramNameBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.TextBytes(), err +} + +func (s ProgramConfig) SetProgramName(v string) error { + return capnp.Struct(s).SetText(0, v) +} + +func (s ProgramConfig) PackageName() (string, error) { + p, err := capnp.Struct(s).Ptr(1) + return p.Text(), err +} + +func (s ProgramConfig) HasPackageName() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s ProgramConfig) PackageNameBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(1) + return p.TextBytes(), err +} + +func (s ProgramConfig) SetPackageName(v string) error { + return capnp.Struct(s).SetText(1, v) +} + +func (s ProgramConfig) BatchSize() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s ProgramConfig) SetBatchSize(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +func (s ProgramConfig) SequenceLength() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s ProgramConfig) SetSequenceLength(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +func (s ProgramConfig) InputSize() uint32 { + return capnp.Struct(s).Uint32(8) +} + +func (s ProgramConfig) SetInputSize(v uint32) { + capnp.Struct(s).SetUint32(8, v) +} + +func (s ProgramConfig) OutputSize() uint32 { + return capnp.Struct(s).Uint32(12) +} + +func (s ProgramConfig) SetOutputSize(v uint32) { + capnp.Struct(s).SetUint32(12, v) +} + +func (s ProgramConfig) Dkvc() (DkvcConfig, error) { + p, err := capnp.Struct(s).Ptr(2) + return DkvcConfig(p.Struct()), err +} + +func (s ProgramConfig) HasDkvc() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s ProgramConfig) SetDkvc(v DkvcConfig) error { + return capnp.Struct(s).SetPtr(2, capnp.Struct(v).ToPtr()) +} + +// NewDkvc sets the dkvc field to a newly +// allocated DkvcConfig struct, preferring placement in s's segment. +func (s ProgramConfig) NewDkvc() (DkvcConfig, error) { + ss, err := NewDkvcConfig(capnp.Struct(s).Segment()) + if err != nil { + return DkvcConfig{}, err + } + err = capnp.Struct(s).SetPtr(2, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ProgramConfig) NumKvCaches() uint32 { + return capnp.Struct(s).Uint32(16) +} + +func (s ProgramConfig) SetNumKvCaches(v uint32) { + capnp.Struct(s).SetUint32(16, v) +} + +func (s ProgramConfig) NumBatchSplitDivisions() uint32 { + return capnp.Struct(s).Uint32(20) +} + +func (s ProgramConfig) SetNumBatchSplitDivisions(v uint32) { + capnp.Struct(s).SetUint32(20, v) +} + +func (s ProgramConfig) BatchFolding() bool { + return capnp.Struct(s).Bit(192) +} + +func (s ProgramConfig) SetBatchFolding(v bool) { + capnp.Struct(s).SetBit(192, v) +} + +func (s ProgramConfig) SupportsCpuEmbeddings() bool { + return capnp.Struct(s).Bit(193) +} + +func (s ProgramConfig) SetSupportsCpuEmbeddings(v bool) { + capnp.Struct(s).SetBit(193, v) +} + +func (s ProgramConfig) Reserved11() bool { + return capnp.Struct(s).Bit(194) +} + +func (s ProgramConfig) SetReserved11(v bool) { + capnp.Struct(s).SetBit(194, v) +} + +func (s ProgramConfig) StandaloneTokenEmbeddings() bool { + return capnp.Struct(s).Bit(195) +} + +func (s ProgramConfig) SetStandaloneTokenEmbeddings(v bool) { + capnp.Struct(s).SetBit(195, v) +} + +func (s ProgramConfig) Topk() (OptionalUInt32, error) { + p, err := capnp.Struct(s).Ptr(3) + return OptionalUInt32(p.Struct()), err +} + +func (s ProgramConfig) HasTopk() bool { + return capnp.Struct(s).HasPtr(3) +} + +func (s ProgramConfig) SetTopk(v OptionalUInt32) error { + return capnp.Struct(s).SetPtr(3, capnp.Struct(v).ToPtr()) +} + +// NewTopk sets the topk field to a newly +// allocated OptionalUInt32 struct, preferring placement in s's segment. +func (s ProgramConfig) NewTopk() (OptionalUInt32, error) { + ss, err := NewOptionalUInt32(capnp.Struct(s).Segment()) + if err != nil { + return OptionalUInt32{}, err + } + err = capnp.Struct(s).SetPtr(3, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ProgramConfig) EnableLogitMasking() bool { + return capnp.Struct(s).Bit(196) +} + +func (s ProgramConfig) SetEnableLogitMasking(v bool) { + capnp.Struct(s).SetBit(196, v) +} + +func (s ProgramConfig) SwaChunked() bool { + return capnp.Struct(s).Bit(197) +} + +func (s ProgramConfig) SetSwaChunked(v bool) { + capnp.Struct(s).SetBit(197, v) +} + +func (s ProgramConfig) NumSwaDkvcBlocks() uint32 { + return capnp.Struct(s).Uint32(28) +} + +func (s ProgramConfig) SetNumSwaDkvcBlocks(v uint32) { + capnp.Struct(s).SetUint32(28, v) +} + +// ProgramConfig_List is a list of ProgramConfig. +type ProgramConfig_List = capnp.StructList[ProgramConfig] + +// NewProgramConfig creates a new list of ProgramConfig. +func NewProgramConfig_List(s *capnp.Segment, sz int32) (ProgramConfig_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 32, PointerCount: 4}, sz) + return capnp.StructList[ProgramConfig](l), err +} + +// ProgramConfig_Future is a wrapper for a ProgramConfig promised by a client call. +type ProgramConfig_Future struct{ *capnp.Future } + +func (f ProgramConfig_Future) Struct() (ProgramConfig, error) { + p, err := f.Future.Ptr() + return ProgramConfig(p.Struct()), err +} +func (p ProgramConfig_Future) Dkvc() DkvcConfig_Future { + return DkvcConfig_Future{Future: p.Future.Field(2, nil)} +} +func (p ProgramConfig_Future) Topk() OptionalUInt32_Future { + return OptionalUInt32_Future{Future: p.Future.Field(3, nil)} +} + +type DkvcConfig capnp.Struct + +// DkvcConfig_TypeID is the unique identifier for the type DkvcConfig. +const DkvcConfig_TypeID = 0xf04def427d04d873 + +func NewDkvcConfig(s *capnp.Segment) (DkvcConfig, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return DkvcConfig(st), err +} + +func NewRootDkvcConfig(s *capnp.Segment) (DkvcConfig, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return DkvcConfig(st), err +} + +func ReadRootDkvcConfig(msg *capnp.Message) (DkvcConfig, error) { + root, err := msg.Root() + return DkvcConfig(root.Struct()), err +} + +func (s DkvcConfig) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (DkvcConfig) DecodeFromPtr(p capnp.Ptr) DkvcConfig { + return DkvcConfig(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s DkvcConfig) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s DkvcConfig) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s DkvcConfig) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s DkvcConfig) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s DkvcConfig) NumBlocksPerKvCache() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s DkvcConfig) SetNumBlocksPerKvCache(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +// DkvcConfig_List is a list of DkvcConfig. +type DkvcConfig_List = capnp.StructList[DkvcConfig] + +// NewDkvcConfig creates a new list of DkvcConfig. +func NewDkvcConfig_List(s *capnp.Segment, sz int32) (DkvcConfig_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}, sz) + return capnp.StructList[DkvcConfig](l), err +} + +// DkvcConfig_Future is a wrapper for a DkvcConfig promised by a client call. +type DkvcConfig_Future struct{ *capnp.Future } + +func (f DkvcConfig_Future) Struct() (DkvcConfig, error) { + p, err := f.Future.Ptr() + return DkvcConfig(p.Struct()), err +} + +type PropSyncChain capnp.Struct + +// PropSyncChain_TypeID is the unique identifier for the type PropSyncChain. +const PropSyncChain_TypeID = 0xe4a2574339b7691f + +func NewPropSyncChain(s *capnp.Segment) (PropSyncChain, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}) + return PropSyncChain(st), err +} + +func NewRootPropSyncChain(s *capnp.Segment) (PropSyncChain, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}) + return PropSyncChain(st), err +} + +func ReadRootPropSyncChain(msg *capnp.Message) (PropSyncChain, error) { + root, err := msg.Root() + return PropSyncChain(root.Struct()), err +} + +func (s PropSyncChain) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (PropSyncChain) DecodeFromPtr(p capnp.Ptr) PropSyncChain { + return PropSyncChain(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s PropSyncChain) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s PropSyncChain) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s PropSyncChain) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s PropSyncChain) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s PropSyncChain) PartitionIds() (capnp.UInt32List, error) { + p, err := capnp.Struct(s).Ptr(0) + return capnp.UInt32List(p.List()), err +} + +func (s PropSyncChain) HasPartitionIds() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s PropSyncChain) SetPartitionIds(v capnp.UInt32List) error { + return capnp.Struct(s).SetPtr(0, v.ToPtr()) +} + +// NewPartitionIds sets the partitionIds field to a newly +// allocated capnp.UInt32List, preferring placement in s's segment. +func (s PropSyncChain) NewPartitionIds(n int32) (capnp.UInt32List, error) { + l, err := capnp.NewUInt32List(capnp.Struct(s).Segment(), n) + if err != nil { + return capnp.UInt32List{}, err + } + err = capnp.Struct(s).SetPtr(0, l.ToPtr()) + return l, err +} + +// PropSyncChain_List is a list of PropSyncChain. +type PropSyncChain_List = capnp.StructList[PropSyncChain] + +// NewPropSyncChain creates a new list of PropSyncChain. +func NewPropSyncChain_List(s *capnp.Segment, sz int32) (PropSyncChain_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}, sz) + return capnp.StructList[PropSyncChain](l), err +} + +// PropSyncChain_Future is a wrapper for a PropSyncChain promised by a client call. +type PropSyncChain_Future struct{ *capnp.Future } + +func (f PropSyncChain_Future) Struct() (PropSyncChain, error) { + p, err := f.Future.Ptr() + return PropSyncChain(p.Struct()), err +} + +type RuntimeIoConfig capnp.Struct + +// RuntimeIoConfig_TypeID is the unique identifier for the type RuntimeIoConfig. +const RuntimeIoConfig_TypeID = 0xd56d56fafa085238 + +func NewRuntimeIoConfig(s *capnp.Segment) (RuntimeIoConfig, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 0}) + return RuntimeIoConfig(st), err +} + +func NewRootRuntimeIoConfig(s *capnp.Segment) (RuntimeIoConfig, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 0}) + return RuntimeIoConfig(st), err +} + +func ReadRootRuntimeIoConfig(msg *capnp.Message) (RuntimeIoConfig, error) { + root, err := msg.Root() + return RuntimeIoConfig(root.Struct()), err +} + +func (s RuntimeIoConfig) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (RuntimeIoConfig) DecodeFromPtr(p capnp.Ptr) RuntimeIoConfig { + return RuntimeIoConfig(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s RuntimeIoConfig) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s RuntimeIoConfig) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s RuntimeIoConfig) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s RuntimeIoConfig) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s RuntimeIoConfig) Protocol() uint16 { + return capnp.Struct(s).Uint16(0) +} + +func (s RuntimeIoConfig) SetProtocol(v uint16) { + capnp.Struct(s).SetUint16(0, v) +} + +func (s RuntimeIoConfig) Reserved1() uint16 { + return capnp.Struct(s).Uint16(2) +} + +func (s RuntimeIoConfig) SetReserved1(v uint16) { + capnp.Struct(s).SetUint16(2, v) +} + +func (s RuntimeIoConfig) IoFpgaCount() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s RuntimeIoConfig) SetIoFpgaCount(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +func (s RuntimeIoConfig) FanoutFactor() uint32 { + return capnp.Struct(s).Uint32(8) +} + +func (s RuntimeIoConfig) SetFanoutFactor(v uint32) { + capnp.Struct(s).SetUint32(8, v) +} + +// RuntimeIoConfig_List is a list of RuntimeIoConfig. +type RuntimeIoConfig_List = capnp.StructList[RuntimeIoConfig] + +// NewRuntimeIoConfig creates a new list of RuntimeIoConfig. +func NewRuntimeIoConfig_List(s *capnp.Segment, sz int32) (RuntimeIoConfig_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 16, PointerCount: 0}, sz) + return capnp.StructList[RuntimeIoConfig](l), err +} + +// RuntimeIoConfig_Future is a wrapper for a RuntimeIoConfig promised by a client call. +type RuntimeIoConfig_Future struct{ *capnp.Future } + +func (f RuntimeIoConfig_Future) Struct() (RuntimeIoConfig, error) { + p, err := f.Future.Ptr() + return RuntimeIoConfig(p.Struct()), err +} + +type ArtifactInfo capnp.Struct + +// ArtifactInfo_TypeID is the unique identifier for the type ArtifactInfo. +const ArtifactInfo_TypeID = 0xfd4e240e22cb8a78 + +func NewArtifactInfo(s *capnp.Segment) (ArtifactInfo, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 7}) + return ArtifactInfo(st), err +} + +func NewRootArtifactInfo(s *capnp.Segment) (ArtifactInfo, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 7}) + return ArtifactInfo(st), err +} + +func ReadRootArtifactInfo(msg *capnp.Message) (ArtifactInfo, error) { + root, err := msg.Root() + return ArtifactInfo(root.Struct()), err +} + +func (s ArtifactInfo) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (ArtifactInfo) DecodeFromPtr(p capnp.Ptr) ArtifactInfo { + return ArtifactInfo(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s ArtifactInfo) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s ArtifactInfo) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s ArtifactInfo) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s ArtifactInfo) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s ArtifactInfo) Partitions() (PartitionInfo_List, error) { + p, err := capnp.Struct(s).Ptr(0) + return PartitionInfo_List(p.List()), err +} + +func (s ArtifactInfo) HasPartitions() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s ArtifactInfo) SetPartitions(v PartitionInfo_List) error { + return capnp.Struct(s).SetPtr(0, v.ToPtr()) +} + +// NewPartitions sets the partitions field to a newly +// allocated PartitionInfo_List, preferring placement in s's segment. +func (s ArtifactInfo) NewPartitions(n int32) (PartitionInfo_List, error) { + l, err := NewPartitionInfo_List(capnp.Struct(s).Segment(), n) + if err != nil { + return PartitionInfo_List{}, err + } + err = capnp.Struct(s).SetPtr(0, l.ToPtr()) + return l, err +} +func (s ArtifactInfo) Reserved1() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(1) +} + +func (s ArtifactInfo) HasReserved1() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s ArtifactInfo) SetReserved1(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(1, v) +} +func (s ArtifactInfo) Reserved2() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(2) +} + +func (s ArtifactInfo) HasReserved2() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s ArtifactInfo) SetReserved2(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(2, v) +} +func (s ArtifactInfo) Reserved3() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(3) +} + +func (s ArtifactInfo) HasReserved3() bool { + return capnp.Struct(s).HasPtr(3) +} + +func (s ArtifactInfo) SetReserved3(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(3, v) +} +func (s ArtifactInfo) RuntimeAssets() (RuntimeAssets, error) { + p, err := capnp.Struct(s).Ptr(4) + return RuntimeAssets(p.Struct()), err +} + +func (s ArtifactInfo) HasRuntimeAssets() bool { + return capnp.Struct(s).HasPtr(4) +} + +func (s ArtifactInfo) SetRuntimeAssets(v RuntimeAssets) error { + return capnp.Struct(s).SetPtr(4, capnp.Struct(v).ToPtr()) +} + +// NewRuntimeAssets sets the runtimeAssets field to a newly +// allocated RuntimeAssets struct, preferring placement in s's segment. +func (s ArtifactInfo) NewRuntimeAssets() (RuntimeAssets, error) { + ss, err := NewRuntimeAssets(capnp.Struct(s).Segment()) + if err != nil { + return RuntimeAssets{}, err + } + err = capnp.Struct(s).SetPtr(4, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ArtifactInfo) PartSelect() (PartSelect, error) { + p, err := capnp.Struct(s).Ptr(5) + return PartSelect(p.Struct()), err +} + +func (s ArtifactInfo) HasPartSelect() bool { + return capnp.Struct(s).HasPtr(5) +} + +func (s ArtifactInfo) SetPartSelect(v PartSelect) error { + return capnp.Struct(s).SetPtr(5, capnp.Struct(v).ToPtr()) +} + +// NewPartSelect sets the partSelect field to a newly +// allocated PartSelect struct, preferring placement in s's segment. +func (s ArtifactInfo) NewPartSelect() (PartSelect, error) { + ss, err := NewPartSelect(capnp.Struct(s).Segment()) + if err != nil { + return PartSelect{}, err + } + err = capnp.Struct(s).SetPtr(5, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s ArtifactInfo) Reserved6() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(6) +} + +func (s ArtifactInfo) HasReserved6() bool { + return capnp.Struct(s).HasPtr(6) +} + +func (s ArtifactInfo) SetReserved6(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(6, v) +} + +// ArtifactInfo_List is a list of ArtifactInfo. +type ArtifactInfo_List = capnp.StructList[ArtifactInfo] + +// NewArtifactInfo creates a new list of ArtifactInfo. +func NewArtifactInfo_List(s *capnp.Segment, sz int32) (ArtifactInfo_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 7}, sz) + return capnp.StructList[ArtifactInfo](l), err +} + +// ArtifactInfo_Future is a wrapper for a ArtifactInfo promised by a client call. +type ArtifactInfo_Future struct{ *capnp.Future } + +func (f ArtifactInfo_Future) Struct() (ArtifactInfo, error) { + p, err := f.Future.Ptr() + return ArtifactInfo(p.Struct()), err +} +func (p ArtifactInfo_Future) Reserved1() *capnp.Future { + return p.Future.Field(1, nil) +} +func (p ArtifactInfo_Future) Reserved2() *capnp.Future { + return p.Future.Field(2, nil) +} +func (p ArtifactInfo_Future) Reserved3() *capnp.Future { + return p.Future.Field(3, nil) +} +func (p ArtifactInfo_Future) RuntimeAssets() RuntimeAssets_Future { + return RuntimeAssets_Future{Future: p.Future.Field(4, nil)} +} +func (p ArtifactInfo_Future) PartSelect() PartSelect_Future { + return PartSelect_Future{Future: p.Future.Field(5, nil)} +} +func (p ArtifactInfo_Future) Reserved6() *capnp.Future { + return p.Future.Field(6, nil) +} + +type RuntimeAssets capnp.Struct + +// RuntimeAssets_TypeID is the unique identifier for the type RuntimeAssets. +const RuntimeAssets_TypeID = 0xd7d44733b7c9406d + +func NewRuntimeAssets(s *capnp.Segment) (RuntimeAssets, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return RuntimeAssets(st), err +} + +func NewRootRuntimeAssets(s *capnp.Segment) (RuntimeAssets, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return RuntimeAssets(st), err +} + +func ReadRootRuntimeAssets(msg *capnp.Message) (RuntimeAssets, error) { + root, err := msg.Root() + return RuntimeAssets(root.Struct()), err +} + +func (s RuntimeAssets) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (RuntimeAssets) DecodeFromPtr(p capnp.Ptr) RuntimeAssets { + return RuntimeAssets(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s RuntimeAssets) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s RuntimeAssets) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s RuntimeAssets) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s RuntimeAssets) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s RuntimeAssets) TokenEmbeddingsPath() (string, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.Text(), err +} + +func (s RuntimeAssets) HasTokenEmbeddingsPath() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s RuntimeAssets) TokenEmbeddingsPathBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.TextBytes(), err +} + +func (s RuntimeAssets) SetTokenEmbeddingsPath(v string) error { + return capnp.Struct(s).SetText(0, v) +} + +func (s RuntimeAssets) Reserved1() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(1) +} + +func (s RuntimeAssets) HasReserved1() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s RuntimeAssets) SetReserved1(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(1, v) +} + +// RuntimeAssets_List is a list of RuntimeAssets. +type RuntimeAssets_List = capnp.StructList[RuntimeAssets] + +// NewRuntimeAssets creates a new list of RuntimeAssets. +func NewRuntimeAssets_List(s *capnp.Segment, sz int32) (RuntimeAssets_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz) + return capnp.StructList[RuntimeAssets](l), err +} + +// RuntimeAssets_Future is a wrapper for a RuntimeAssets promised by a client call. +type RuntimeAssets_Future struct{ *capnp.Future } + +func (f RuntimeAssets_Future) Struct() (RuntimeAssets, error) { + p, err := f.Future.Ptr() + return RuntimeAssets(p.Struct()), err +} +func (p RuntimeAssets_Future) Reserved1() *capnp.Future { + return p.Future.Field(1, nil) +} + +type PartitionInfo capnp.Struct +type PartitionInfo_detail PartitionInfo +type PartitionInfo_detail_Which uint16 + +const ( + PartitionInfo_detail_Which_lpu PartitionInfo_detail_Which = 0 + PartitionInfo_detail_Which_cuda PartitionInfo_detail_Which = 1 + PartitionInfo_detail_Which_cpu PartitionInfo_detail_Which = 2 +) + +func (w PartitionInfo_detail_Which) String() string { + const s = "lpucudacpu" + switch w { + case PartitionInfo_detail_Which_lpu: + return s[0:3] + case PartitionInfo_detail_Which_cuda: + return s[3:7] + case PartitionInfo_detail_Which_cpu: + return s[7:10] + + } + return "PartitionInfo_detail_Which(" + strconv.FormatUint(uint64(w), 10) + ")" +} + +// PartitionInfo_TypeID is the unique identifier for the type PartitionInfo. +const PartitionInfo_TypeID = 0xc2825cc7c53e1399 + +func NewPartitionInfo(s *capnp.Segment) (PartitionInfo, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return PartitionInfo(st), err +} + +func NewRootPartitionInfo(s *capnp.Segment) (PartitionInfo, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return PartitionInfo(st), err +} + +func ReadRootPartitionInfo(msg *capnp.Message) (PartitionInfo, error) { + root, err := msg.Root() + return PartitionInfo(root.Struct()), err +} + +func (s PartitionInfo) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (PartitionInfo) DecodeFromPtr(p capnp.Ptr) PartitionInfo { + return PartitionInfo(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s PartitionInfo) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s PartitionInfo) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s PartitionInfo) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s PartitionInfo) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s PartitionInfo) Partition() (PartitionRef, error) { + p, err := capnp.Struct(s).Ptr(0) + return PartitionRef(p.Struct()), err +} + +func (s PartitionInfo) HasPartition() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s PartitionInfo) SetPartition(v PartitionRef) error { + return capnp.Struct(s).SetPtr(0, capnp.Struct(v).ToPtr()) +} + +// NewPartition sets the partition field to a newly +// allocated PartitionRef struct, preferring placement in s's segment. +func (s PartitionInfo) NewPartition() (PartitionRef, error) { + ss, err := NewPartitionRef(capnp.Struct(s).Segment()) + if err != nil { + return PartitionRef{}, err + } + err = capnp.Struct(s).SetPtr(0, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s PartitionInfo) Reserved1() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(1) +} + +func (s PartitionInfo) HasReserved1() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s PartitionInfo) SetReserved1(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(1, v) +} +func (s PartitionInfo) Reserved2() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(2) +} + +func (s PartitionInfo) HasReserved2() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s PartitionInfo) SetReserved2(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(2, v) +} +func (s PartitionInfo) Detail() PartitionInfo_detail { return PartitionInfo_detail(s) } + +func (s PartitionInfo_detail) Which() PartitionInfo_detail_Which { + return PartitionInfo_detail_Which(capnp.Struct(s).Uint16(0)) +} +func (s PartitionInfo_detail) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s PartitionInfo_detail) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s PartitionInfo_detail) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s PartitionInfo_detail) Lpu() (LpuPartitionArtifact, error) { + if capnp.Struct(s).Uint16(0) != 0 { + panic("Which() != lpu") + } + p, err := capnp.Struct(s).Ptr(3) + return LpuPartitionArtifact(p.Struct()), err +} + +func (s PartitionInfo_detail) HasLpu() bool { + if capnp.Struct(s).Uint16(0) != 0 { + return false + } + return capnp.Struct(s).HasPtr(3) +} + +func (s PartitionInfo_detail) SetLpu(v LpuPartitionArtifact) error { + capnp.Struct(s).SetUint16(0, 0) + return capnp.Struct(s).SetPtr(3, capnp.Struct(v).ToPtr()) +} + +// NewLpu sets the lpu field to a newly +// allocated LpuPartitionArtifact struct, preferring placement in s's segment. +func (s PartitionInfo_detail) NewLpu() (LpuPartitionArtifact, error) { + capnp.Struct(s).SetUint16(0, 0) + ss, err := NewLpuPartitionArtifact(capnp.Struct(s).Segment()) + if err != nil { + return LpuPartitionArtifact{}, err + } + err = capnp.Struct(s).SetPtr(3, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s PartitionInfo_detail) Cuda() (CudaPartitionArtifact, error) { + if capnp.Struct(s).Uint16(0) != 1 { + panic("Which() != cuda") + } + p, err := capnp.Struct(s).Ptr(3) + return CudaPartitionArtifact(p.Struct()), err +} + +func (s PartitionInfo_detail) HasCuda() bool { + if capnp.Struct(s).Uint16(0) != 1 { + return false + } + return capnp.Struct(s).HasPtr(3) +} + +func (s PartitionInfo_detail) SetCuda(v CudaPartitionArtifact) error { + capnp.Struct(s).SetUint16(0, 1) + return capnp.Struct(s).SetPtr(3, capnp.Struct(v).ToPtr()) +} + +// NewCuda sets the cuda field to a newly +// allocated CudaPartitionArtifact struct, preferring placement in s's segment. +func (s PartitionInfo_detail) NewCuda() (CudaPartitionArtifact, error) { + capnp.Struct(s).SetUint16(0, 1) + ss, err := NewCudaPartitionArtifact(capnp.Struct(s).Segment()) + if err != nil { + return CudaPartitionArtifact{}, err + } + err = capnp.Struct(s).SetPtr(3, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s PartitionInfo_detail) Cpu() (CpuPartitionArtifact, error) { + if capnp.Struct(s).Uint16(0) != 2 { + panic("Which() != cpu") + } + p, err := capnp.Struct(s).Ptr(3) + return CpuPartitionArtifact(p.Struct()), err +} + +func (s PartitionInfo_detail) HasCpu() bool { + if capnp.Struct(s).Uint16(0) != 2 { + return false + } + return capnp.Struct(s).HasPtr(3) +} + +func (s PartitionInfo_detail) SetCpu(v CpuPartitionArtifact) error { + capnp.Struct(s).SetUint16(0, 2) + return capnp.Struct(s).SetPtr(3, capnp.Struct(v).ToPtr()) +} + +// NewCpu sets the cpu field to a newly +// allocated CpuPartitionArtifact struct, preferring placement in s's segment. +func (s PartitionInfo_detail) NewCpu() (CpuPartitionArtifact, error) { + capnp.Struct(s).SetUint16(0, 2) + ss, err := NewCpuPartitionArtifact(capnp.Struct(s).Segment()) + if err != nil { + return CpuPartitionArtifact{}, err + } + err = capnp.Struct(s).SetPtr(3, capnp.Struct(ss).ToPtr()) + return ss, err +} + +// PartitionInfo_List is a list of PartitionInfo. +type PartitionInfo_List = capnp.StructList[PartitionInfo] + +// NewPartitionInfo creates a new list of PartitionInfo. +func NewPartitionInfo_List(s *capnp.Segment, sz int32) (PartitionInfo_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}, sz) + return capnp.StructList[PartitionInfo](l), err +} + +// PartitionInfo_Future is a wrapper for a PartitionInfo promised by a client call. +type PartitionInfo_Future struct{ *capnp.Future } + +func (f PartitionInfo_Future) Struct() (PartitionInfo, error) { + p, err := f.Future.Ptr() + return PartitionInfo(p.Struct()), err +} +func (p PartitionInfo_Future) Partition() PartitionRef_Future { + return PartitionRef_Future{Future: p.Future.Field(0, nil)} +} +func (p PartitionInfo_Future) Reserved1() *capnp.Future { + return p.Future.Field(1, nil) +} +func (p PartitionInfo_Future) Reserved2() *capnp.Future { + return p.Future.Field(2, nil) +} +func (p PartitionInfo_Future) Detail() PartitionInfo_detail_Future { + return PartitionInfo_detail_Future{p.Future} +} + +// PartitionInfo_detail_Future is a wrapper for a PartitionInfo_detail promised by a client call. +type PartitionInfo_detail_Future struct{ *capnp.Future } + +func (f PartitionInfo_detail_Future) Struct() (PartitionInfo_detail, error) { + p, err := f.Future.Ptr() + return PartitionInfo_detail(p.Struct()), err +} +func (p PartitionInfo_detail_Future) Lpu() LpuPartitionArtifact_Future { + return LpuPartitionArtifact_Future{Future: p.Future.Field(3, nil)} +} +func (p PartitionInfo_detail_Future) Cuda() CudaPartitionArtifact_Future { + return CudaPartitionArtifact_Future{Future: p.Future.Field(3, nil)} +} +func (p PartitionInfo_detail_Future) Cpu() CpuPartitionArtifact_Future { + return CpuPartitionArtifact_Future{Future: p.Future.Field(3, nil)} +} + +type PartitionRef capnp.Struct + +// PartitionRef_TypeID is the unique identifier for the type PartitionRef. +const PartitionRef_TypeID = 0xcc1f6d5da735cb49 + +func NewPartitionRef(s *capnp.Segment) (PartitionRef, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return PartitionRef(st), err +} + +func NewRootPartitionRef(s *capnp.Segment) (PartitionRef, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return PartitionRef(st), err +} + +func ReadRootPartitionRef(msg *capnp.Message) (PartitionRef, error) { + root, err := msg.Root() + return PartitionRef(root.Struct()), err +} + +func (s PartitionRef) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (PartitionRef) DecodeFromPtr(p capnp.Ptr) PartitionRef { + return PartitionRef(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s PartitionRef) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s PartitionRef) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s PartitionRef) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s PartitionRef) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s PartitionRef) DeviceType() DeviceType { + return DeviceType(capnp.Struct(s).Uint16(0)) +} + +func (s PartitionRef) SetDeviceType(v DeviceType) { + capnp.Struct(s).SetUint16(0, uint16(v)) +} + +func (s PartitionRef) PartitionId() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s PartitionRef) SetPartitionId(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +// PartitionRef_List is a list of PartitionRef. +type PartitionRef_List = capnp.StructList[PartitionRef] + +// NewPartitionRef creates a new list of PartitionRef. +func NewPartitionRef_List(s *capnp.Segment, sz int32) (PartitionRef_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}, sz) + return capnp.StructList[PartitionRef](l), err +} + +// PartitionRef_Future is a wrapper for a PartitionRef promised by a client call. +type PartitionRef_Future struct{ *capnp.Future } + +func (f PartitionRef_Future) Struct() (PartitionRef, error) { + p, err := f.Future.Ptr() + return PartitionRef(p.Struct()), err +} + +type LpuPartitionArtifact capnp.Struct + +// LpuPartitionArtifact_TypeID is the unique identifier for the type LpuPartitionArtifact. +const LpuPartitionArtifact_TypeID = 0xae800a9bf3fc6050 + +func NewLpuPartitionArtifact(s *capnp.Segment) (LpuPartitionArtifact, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return LpuPartitionArtifact(st), err +} + +func NewRootLpuPartitionArtifact(s *capnp.Segment) (LpuPartitionArtifact, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}) + return LpuPartitionArtifact(st), err +} + +func ReadRootLpuPartitionArtifact(msg *capnp.Message) (LpuPartitionArtifact, error) { + root, err := msg.Root() + return LpuPartitionArtifact(root.Struct()), err +} + +func (s LpuPartitionArtifact) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (LpuPartitionArtifact) DecodeFromPtr(p capnp.Ptr) LpuPartitionArtifact { + return LpuPartitionArtifact(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s LpuPartitionArtifact) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s LpuPartitionArtifact) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s LpuPartitionArtifact) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s LpuPartitionArtifact) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s LpuPartitionArtifact) Path() (string, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.Text(), err +} + +func (s LpuPartitionArtifact) HasPath() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s LpuPartitionArtifact) PathBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.TextBytes(), err +} + +func (s LpuPartitionArtifact) SetPath(v string) error { + return capnp.Struct(s).SetText(0, v) +} + +func (s LpuPartitionArtifact) Topology() (string, error) { + p, err := capnp.Struct(s).Ptr(1) + return p.Text(), err +} + +func (s LpuPartitionArtifact) HasTopology() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s LpuPartitionArtifact) TopologyBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(1) + return p.TextBytes(), err +} + +func (s LpuPartitionArtifact) SetTopology(v string) error { + return capnp.Struct(s).SetText(1, v) +} + +func (s LpuPartitionArtifact) NumChips() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s LpuPartitionArtifact) SetNumChips(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +func (s LpuPartitionArtifact) TopologyMetadata() (TopologyMetadata, error) { + p, err := capnp.Struct(s).Ptr(2) + return TopologyMetadata(p.Struct()), err +} + +func (s LpuPartitionArtifact) HasTopologyMetadata() bool { + return capnp.Struct(s).HasPtr(2) +} + +func (s LpuPartitionArtifact) SetTopologyMetadata(v TopologyMetadata) error { + return capnp.Struct(s).SetPtr(2, capnp.Struct(v).ToPtr()) +} + +// NewTopologyMetadata sets the topologyMetadata field to a newly +// allocated TopologyMetadata struct, preferring placement in s's segment. +func (s LpuPartitionArtifact) NewTopologyMetadata() (TopologyMetadata, error) { + ss, err := NewTopologyMetadata(capnp.Struct(s).Segment()) + if err != nil { + return TopologyMetadata{}, err + } + err = capnp.Struct(s).SetPtr(2, capnp.Struct(ss).ToPtr()) + return ss, err +} + +func (s LpuPartitionArtifact) Reserved4() (capnp.Ptr, error) { + return capnp.Struct(s).Ptr(3) +} + +func (s LpuPartitionArtifact) HasReserved4() bool { + return capnp.Struct(s).HasPtr(3) +} + +func (s LpuPartitionArtifact) SetReserved4(v capnp.Ptr) error { + return capnp.Struct(s).SetPtr(3, v) +} +func (s LpuPartitionArtifact) DevicesPerNode() uint32 { + return capnp.Struct(s).Uint32(4) +} + +func (s LpuPartitionArtifact) SetDevicesPerNode(v uint32) { + capnp.Struct(s).SetUint32(4, v) +} + +// LpuPartitionArtifact_List is a list of LpuPartitionArtifact. +type LpuPartitionArtifact_List = capnp.StructList[LpuPartitionArtifact] + +// NewLpuPartitionArtifact creates a new list of LpuPartitionArtifact. +func NewLpuPartitionArtifact_List(s *capnp.Segment, sz int32) (LpuPartitionArtifact_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}, sz) + return capnp.StructList[LpuPartitionArtifact](l), err +} + +// LpuPartitionArtifact_Future is a wrapper for a LpuPartitionArtifact promised by a client call. +type LpuPartitionArtifact_Future struct{ *capnp.Future } + +func (f LpuPartitionArtifact_Future) Struct() (LpuPartitionArtifact, error) { + p, err := f.Future.Ptr() + return LpuPartitionArtifact(p.Struct()), err +} +func (p LpuPartitionArtifact_Future) TopologyMetadata() TopologyMetadata_Future { + return TopologyMetadata_Future{Future: p.Future.Field(2, nil)} +} +func (p LpuPartitionArtifact_Future) Reserved4() *capnp.Future { + return p.Future.Field(3, nil) +} + +type CudaPartitionArtifact capnp.Struct + +// CudaPartitionArtifact_TypeID is the unique identifier for the type CudaPartitionArtifact. +const CudaPartitionArtifact_TypeID = 0x9eaa468d1b6452a4 + +func NewCudaPartitionArtifact(s *capnp.Segment) (CudaPartitionArtifact, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}) + return CudaPartitionArtifact(st), err +} + +func NewRootCudaPartitionArtifact(s *capnp.Segment) (CudaPartitionArtifact, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}) + return CudaPartitionArtifact(st), err +} + +func ReadRootCudaPartitionArtifact(msg *capnp.Message) (CudaPartitionArtifact, error) { + root, err := msg.Root() + return CudaPartitionArtifact(root.Struct()), err +} + +func (s CudaPartitionArtifact) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (CudaPartitionArtifact) DecodeFromPtr(p capnp.Ptr) CudaPartitionArtifact { + return CudaPartitionArtifact(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s CudaPartitionArtifact) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s CudaPartitionArtifact) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s CudaPartitionArtifact) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s CudaPartitionArtifact) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} + +// CudaPartitionArtifact_List is a list of CudaPartitionArtifact. +type CudaPartitionArtifact_List = capnp.StructList[CudaPartitionArtifact] + +// NewCudaPartitionArtifact creates a new list of CudaPartitionArtifact. +func NewCudaPartitionArtifact_List(s *capnp.Segment, sz int32) (CudaPartitionArtifact_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}, sz) + return capnp.StructList[CudaPartitionArtifact](l), err +} + +// CudaPartitionArtifact_Future is a wrapper for a CudaPartitionArtifact promised by a client call. +type CudaPartitionArtifact_Future struct{ *capnp.Future } + +func (f CudaPartitionArtifact_Future) Struct() (CudaPartitionArtifact, error) { + p, err := f.Future.Ptr() + return CudaPartitionArtifact(p.Struct()), err +} + +type CpuPartitionArtifact capnp.Struct + +// CpuPartitionArtifact_TypeID is the unique identifier for the type CpuPartitionArtifact. +const CpuPartitionArtifact_TypeID = 0x9c491b4f77040820 + +func NewCpuPartitionArtifact(s *capnp.Segment) (CpuPartitionArtifact, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}) + return CpuPartitionArtifact(st), err +} + +func NewRootCpuPartitionArtifact(s *capnp.Segment) (CpuPartitionArtifact, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}) + return CpuPartitionArtifact(st), err +} + +func ReadRootCpuPartitionArtifact(msg *capnp.Message) (CpuPartitionArtifact, error) { + root, err := msg.Root() + return CpuPartitionArtifact(root.Struct()), err +} + +func (s CpuPartitionArtifact) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (CpuPartitionArtifact) DecodeFromPtr(p capnp.Ptr) CpuPartitionArtifact { + return CpuPartitionArtifact(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s CpuPartitionArtifact) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s CpuPartitionArtifact) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s CpuPartitionArtifact) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s CpuPartitionArtifact) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} + +// CpuPartitionArtifact_List is a list of CpuPartitionArtifact. +type CpuPartitionArtifact_List = capnp.StructList[CpuPartitionArtifact] + +// NewCpuPartitionArtifact creates a new list of CpuPartitionArtifact. +func NewCpuPartitionArtifact_List(s *capnp.Segment, sz int32) (CpuPartitionArtifact_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 0}, sz) + return capnp.StructList[CpuPartitionArtifact](l), err +} + +// CpuPartitionArtifact_Future is a wrapper for a CpuPartitionArtifact promised by a client call. +type CpuPartitionArtifact_Future struct{ *capnp.Future } + +func (f CpuPartitionArtifact_Future) Struct() (CpuPartitionArtifact, error) { + p, err := f.Future.Ptr() + return CpuPartitionArtifact(p.Struct()), err +} + +type PartSelect capnp.Struct + +// PartSelect_TypeID is the unique identifier for the type PartSelect. +const PartSelect_TypeID = 0x906961f37780d08b + +func NewPartSelect(s *capnp.Segment) (PartSelect, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}) + return PartSelect(st), err +} + +func NewRootPartSelect(s *capnp.Segment) (PartSelect, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}) + return PartSelect(st), err +} + +func ReadRootPartSelect(msg *capnp.Message) (PartSelect, error) { + root, err := msg.Root() + return PartSelect(root.Struct()), err +} + +func (s PartSelect) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (PartSelect) DecodeFromPtr(p capnp.Ptr) PartSelect { + return PartSelect(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s PartSelect) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s PartSelect) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s PartSelect) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s PartSelect) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s PartSelect) Partitions() (PartitionRef_List, error) { + p, err := capnp.Struct(s).Ptr(0) + return PartitionRef_List(p.List()), err +} + +func (s PartSelect) HasPartitions() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s PartSelect) SetPartitions(v PartitionRef_List) error { + return capnp.Struct(s).SetPtr(0, v.ToPtr()) +} + +// NewPartitions sets the partitions field to a newly +// allocated PartitionRef_List, preferring placement in s's segment. +func (s PartSelect) NewPartitions(n int32) (PartitionRef_List, error) { + l, err := NewPartitionRef_List(capnp.Struct(s).Segment(), n) + if err != nil { + return PartitionRef_List{}, err + } + err = capnp.Struct(s).SetPtr(0, l.ToPtr()) + return l, err +} + +// PartSelect_List is a list of PartSelect. +type PartSelect_List = capnp.StructList[PartSelect] + +// NewPartSelect creates a new list of PartSelect. +func NewPartSelect_List(s *capnp.Segment, sz int32) (PartSelect_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}, sz) + return capnp.StructList[PartSelect](l), err +} + +// PartSelect_Future is a wrapper for a PartSelect promised by a client call. +type PartSelect_Future struct{ *capnp.Future } + +func (f PartSelect_Future) Struct() (PartSelect, error) { + p, err := f.Future.Ptr() + return PartSelect(p.Struct()), err +} + +type DeviceType uint16 + +// DeviceType_TypeID is the unique identifier for the type DeviceType. +const DeviceType_TypeID = 0x9e4e2af44ac2e94e + +// Values of DeviceType. +const ( + DeviceType_lpu DeviceType = 0 + DeviceType_cuda DeviceType = 1 + DeviceType_cpu DeviceType = 2 +) + +// String returns the enum's constant name. +func (c DeviceType) String() string { + switch c { + case DeviceType_lpu: + return "lpu" + case DeviceType_cuda: + return "cuda" + case DeviceType_cpu: + return "cpu" + + default: + return "" + } +} + +// DeviceTypeFromString returns the enum value with a name, +// or the zero value if there's no such value. +func DeviceTypeFromString(c string) DeviceType { + switch c { + case "lpu": + return DeviceType_lpu + case "cuda": + return DeviceType_cuda + case "cpu": + return DeviceType_cpu + + default: + return 0 + } +} + +type DeviceType_List = capnp.EnumList[DeviceType] + +func NewDeviceType_List(s *capnp.Segment, sz int32) (DeviceType_List, error) { + return capnp.NewEnumList[DeviceType](s, sz) +} + +type TopologyMetadata capnp.Struct + +// TopologyMetadata_TypeID is the unique identifier for the type TopologyMetadata. +const TopologyMetadata_TypeID = 0x9ac600e40b2f2bd2 + +func NewTopologyMetadata(s *capnp.Segment) (TopologyMetadata, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return TopologyMetadata(st), err +} + +func NewRootTopologyMetadata(s *capnp.Segment) (TopologyMetadata, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}) + return TopologyMetadata(st), err +} + +func ReadRootTopologyMetadata(msg *capnp.Message) (TopologyMetadata, error) { + root, err := msg.Root() + return TopologyMetadata(root.Struct()), err +} + +func (s TopologyMetadata) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (TopologyMetadata) DecodeFromPtr(p capnp.Ptr) TopologyMetadata { + return TopologyMetadata(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s TopologyMetadata) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s TopologyMetadata) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s TopologyMetadata) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s TopologyMetadata) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s TopologyMetadata) TopologyFamily() (string, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.Text(), err +} + +func (s TopologyMetadata) HasTopologyFamily() bool { + return capnp.Struct(s).HasPtr(0) +} + +func (s TopologyMetadata) TopologyFamilyBytes() ([]byte, error) { + p, err := capnp.Struct(s).Ptr(0) + return p.TextBytes(), err +} + +func (s TopologyMetadata) SetTopologyFamily(v string) error { + return capnp.Struct(s).SetText(0, v) +} + +func (s TopologyMetadata) PartitionShape() (capnp.UInt32List, error) { + p, err := capnp.Struct(s).Ptr(1) + return capnp.UInt32List(p.List()), err +} + +func (s TopologyMetadata) HasPartitionShape() bool { + return capnp.Struct(s).HasPtr(1) +} + +func (s TopologyMetadata) SetPartitionShape(v capnp.UInt32List) error { + return capnp.Struct(s).SetPtr(1, v.ToPtr()) +} + +// NewPartitionShape sets the partitionShape field to a newly +// allocated capnp.UInt32List, preferring placement in s's segment. +func (s TopologyMetadata) NewPartitionShape(n int32) (capnp.UInt32List, error) { + l, err := capnp.NewUInt32List(capnp.Struct(s).Segment(), n) + if err != nil { + return capnp.UInt32List{}, err + } + err = capnp.Struct(s).SetPtr(1, l.ToPtr()) + return l, err +} + +// TopologyMetadata_List is a list of TopologyMetadata. +type TopologyMetadata_List = capnp.StructList[TopologyMetadata] + +// NewTopologyMetadata creates a new list of TopologyMetadata. +func NewTopologyMetadata_List(s *capnp.Segment, sz int32) (TopologyMetadata_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz) + return capnp.StructList[TopologyMetadata](l), err +} + +// TopologyMetadata_Future is a wrapper for a TopologyMetadata promised by a client call. +type TopologyMetadata_Future struct{ *capnp.Future } + +func (f TopologyMetadata_Future) Struct() (TopologyMetadata, error) { + p, err := f.Future.Ptr() + return TopologyMetadata(p.Struct()), err +} + +type OptionalUInt32 capnp.Struct + +// OptionalUInt32_TypeID is the unique identifier for the type OptionalUInt32. +const OptionalUInt32_TypeID = 0x83496be49a2be62e + +func NewOptionalUInt32(s *capnp.Segment) (OptionalUInt32, error) { + st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return OptionalUInt32(st), err +} + +func NewRootOptionalUInt32(s *capnp.Segment) (OptionalUInt32, error) { + st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}) + return OptionalUInt32(st), err +} + +func ReadRootOptionalUInt32(msg *capnp.Message) (OptionalUInt32, error) { + root, err := msg.Root() + return OptionalUInt32(root.Struct()), err +} + +func (s OptionalUInt32) EncodeAsPtr(seg *capnp.Segment) capnp.Ptr { + return capnp.Struct(s).EncodeAsPtr(seg) +} + +func (OptionalUInt32) DecodeFromPtr(p capnp.Ptr) OptionalUInt32 { + return OptionalUInt32(capnp.Struct{}.DecodeFromPtr(p)) +} + +func (s OptionalUInt32) ToPtr() capnp.Ptr { + return capnp.Struct(s).ToPtr() +} +func (s OptionalUInt32) IsValid() bool { + return capnp.Struct(s).IsValid() +} + +func (s OptionalUInt32) Message() *capnp.Message { + return capnp.Struct(s).Message() +} + +func (s OptionalUInt32) Segment() *capnp.Segment { + return capnp.Struct(s).Segment() +} +func (s OptionalUInt32) Value() uint32 { + return capnp.Struct(s).Uint32(0) +} + +func (s OptionalUInt32) SetValue(v uint32) { + capnp.Struct(s).SetUint32(0, v) +} + +// OptionalUInt32_List is a list of OptionalUInt32. +type OptionalUInt32_List = capnp.StructList[OptionalUInt32] + +// NewOptionalUInt32 creates a new list of OptionalUInt32. +func NewOptionalUInt32_List(s *capnp.Segment, sz int32) (OptionalUInt32_List, error) { + l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 0}, sz) + return capnp.StructList[OptionalUInt32](l), err +} + +// OptionalUInt32_Future is a wrapper for a OptionalUInt32 promised by a client call. +type OptionalUInt32_Future struct{ *capnp.Future } + +func (f OptionalUInt32_Future) Struct() (OptionalUInt32, error) { + p, err := f.Future.Ptr() + return OptionalUInt32(p.Struct()), err +} diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/LICENSE b/deploy/operator/internal/thirdparty/lpxscheduler/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/deploy/operator/internal/thirdparty/lpxscheduler/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go new file mode 100644 index 000000000000..48d93b159641 --- /dev/null +++ b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:skip + +package v1alpha1 diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go new file mode 100644 index 000000000000..fa4c6d817a31 --- /dev/null +++ b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go @@ -0,0 +1,1627 @@ +// Code generated by lpx-scheduler. DO NOT EDIT. + +package v1alpha1 + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "strconv" + "unicode/utf8" +) + +const ( + APIGroup = "scheduling.lpu.nvidia.com" + APIVersion = "v1alpha1" +) + +var GroupVersion = schema.GroupVersion{Group: APIGroup, Version: APIVersion} + +const ( + PodRoleAnnotation = "scheduling.lpu.nvidia.com/role" + PodModelAnnotation = "scheduling.lpu.nvidia.com/model" + PodPartitionIDAnnotation = "scheduling.lpu.nvidia.com/partition-id" + PodRankInPartitionAnnotation = "scheduling.lpu.nvidia.com/rank-in-partition" + PodRoleAgent = "agent" + PodRoleConductor = "conductor" + PodRoleCyborgWorker = "cyborg-worker" + MaxModelAnnotationBytes = 261982 +) + +type PodLogicalRow struct { + Model string + ModelPartitionID uint32 + RankInPartition uint16 +} + +func ParsePodRole(value string) (string, error) { + switch value { + case PodRoleAgent, PodRoleConductor, PodRoleCyborgWorker: + return value, nil + default: + return "", fmt.Errorf("Pod role annotation %q is not an accepted producer role", value) + } +} + +func ValidateModelAnnotation(value string) error { + if value == "" { + return fmt.Errorf("model annotation must not be empty") + } + if !utf8.ValidString(value) { + return fmt.Errorf("model annotation is not valid UTF-8") + } + if len(value) > MaxModelAnnotationBytes { + return fmt.Errorf("model annotation contains %d UTF-8 bytes; at most %d are supported", len(value), MaxModelAnnotationBytes) + } + first, _ := utf8.DecodeRuneInString(value) + last, _ := utf8.DecodeLastRuneInString(value) + if isAnnotationBoundaryWhitespace(first) || isAnnotationBoundaryWhitespace(last) { + return fmt.Errorf("model annotation %q has noncanonical boundary whitespace", value) + } + return nil +} + +func ParseModelPartitionID(value string) (uint32, error) { + parsed, err := parseCanonicalUnsigned(PodPartitionIDAnnotation, value, 32) + return uint32(parsed), err +} + +func ParseRankInPartition(value string) (uint16, error) { + parsed, err := parseCanonicalUnsigned(PodRankInPartitionAnnotation, value, 16) + return uint16(parsed), err +} + +func ParsePodLogicalRow(annotations map[string]string) (PodLogicalRow, error) { + role, err := requiredPodAnnotation(annotations, PodRoleAnnotation) + if err != nil { + return PodLogicalRow{}, err + } + if _, err := ParsePodRole(role); err != nil { + return PodLogicalRow{}, err + } + if role != PodRoleAgent { + return PodLogicalRow{}, fmt.Errorf("logical-row parsing requires role=agent, got %q", role) + } + model, err := requiredPodAnnotation(annotations, PodModelAnnotation) + if err != nil { + return PodLogicalRow{}, err + } + if err := ValidateModelAnnotation(model); err != nil { + return PodLogicalRow{}, err + } + partition, err := requiredPodAnnotation(annotations, PodPartitionIDAnnotation) + if err != nil { + return PodLogicalRow{}, err + } + modelPartitionID, err := ParseModelPartitionID(partition) + if err != nil { + return PodLogicalRow{}, err + } + rank, err := requiredPodAnnotation(annotations, PodRankInPartitionAnnotation) + if err != nil { + return PodLogicalRow{}, err + } + rankInPartition, err := ParseRankInPartition(rank) + if err != nil { + return PodLogicalRow{}, err + } + return PodLogicalRow{Model: model, ModelPartitionID: modelPartitionID, RankInPartition: rankInPartition}, nil +} + +func requiredPodAnnotation(annotations map[string]string, key string) (string, error) { + value, ok := annotations[key] + if !ok { + return "", fmt.Errorf("required Pod annotation %q is missing", key) + } + return value, nil +} + +func parseCanonicalUnsigned(key, value string, bitSize int) (uint64, error) { + if value == "" { + return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) + } + for index := 0; index < len(value); index++ { + if value[index] < '0' || value[index] > '9' { + return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) + } + } + parsed, err := strconv.ParseUint(value, 10, bitSize) + if err != nil { + return 0, fmt.Errorf("Pod annotation %q value %q exceeds its public integer range", key, value) + } + if strconv.FormatUint(parsed, 10) != value { + return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) + } + return parsed, nil +} + +func isAnnotationBoundaryWhitespace(value rune) bool { + switch { + case value >= 0x0009 && value <= 0x000d: + return true + case value == 0x0020: + return true + case value == 0x0085: + return true + case value == 0x00a0: + return true + case value == 0x1680: + return true + case value >= 0x2000 && value <= 0x200a: + return true + case value >= 0x2028 && value <= 0x2029: + return true + case value == 0x202f: + return true + case value == 0x205f: + return true + case value == 0x3000: + return true + default: + return false + } +} + +type LPUPipelineRequest struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec LPUPipelineRequestSpec `json:"spec"` + Status *LPUPipelineRequestStatus `json:"status,omitempty"` +} + +type LPUPipelineRequestList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LPUPipelineRequest `json:"items"` +} + +type AtomicLPUMemberIdentity struct { + Coordinate LPUFamilyCoordinate `json:"coordinate"` + Device ExactDeviceIdentity `json:"device"` + FabricID string `json:"fabricId"` +} + +func (in *AtomicLPUMemberIdentity) DeepCopyInto(out *AtomicLPUMemberIdentity) { + *out = *in + in.Coordinate.DeepCopyInto(&out.Coordinate) + in.Device.DeepCopyInto(&out.Device) +} + +func (in *AtomicLPUMemberIdentity) DeepCopy() *AtomicLPUMemberIdentity { + if in == nil { + return nil + } + out := new(AtomicLPUMemberIdentity) + in.DeepCopyInto(out) + return out +} + +type BindingTarget struct { + BindAttempt *int64 `json:"bindAttempt,omitempty"` + BindingRevision int64 `json:"bindingRevision"` + Phase BindingTargetPhase `json:"phase"` + PodRef ObjectReference `json:"podRef"` + PodResourceVersion *string `json:"podResourceVersion,omitempty"` +} + +func (in *BindingTarget) DeepCopyInto(out *BindingTarget) { + *out = *in + if in.BindAttempt != nil { + value := *in.BindAttempt + out.BindAttempt = &value + } + in.PodRef.DeepCopyInto(&out.PodRef) + if in.PodResourceVersion != nil { + value := *in.PodResourceVersion + out.PodResourceVersion = &value + } +} + +func (in *BindingTarget) DeepCopy() *BindingTarget { + if in == nil { + return nil + } + out := new(BindingTarget) + in.DeepCopyInto(out) + return out +} + +type BindingTargetPhase string + +const ( + BindingTargetPhaseRecorded BindingTargetPhase = "recorded" + BindingTargetPhaseClaimsAuthorized BindingTargetPhase = "claimsAuthorized" + BindingTargetPhaseBindingAttempted BindingTargetPhase = "bindingAttempted" +) + +type ClaimLifecyclePhase string + +const ( + ClaimLifecyclePhaseRecorded ClaimLifecyclePhase = "recorded" + ClaimLifecyclePhaseAllocated ClaimLifecyclePhase = "allocated" + ClaimLifecyclePhaseReleased ClaimLifecyclePhase = "released" +) + +type ClaimReference struct { + Name string `json:"name"` + UID string `json:"uid"` +} + +func (in *ClaimReference) DeepCopyInto(out *ClaimReference) { + *out = *in +} + +func (in *ClaimReference) DeepCopy() *ClaimReference { + if in == nil { + return nil + } + out := new(ClaimReference) + in.DeepCopyInto(out) + return out +} + +type ClaimState struct { + PartitionID string `json:"partitionId"` + Phase ClaimLifecyclePhase `json:"phase"` +} + +func (in *ClaimState) DeepCopyInto(out *ClaimState) { + *out = *in +} + +func (in *ClaimState) DeepCopy() *ClaimState { + if in == nil { + return nil + } + out := new(ClaimState) + in.DeepCopyInto(out) + return out +} + +type CommitFence struct { + ID string `json:"id"` + Outcome CommitFenceOutcome `json:"outcome"` +} + +func (in *CommitFence) DeepCopyInto(out *CommitFence) { + *out = *in +} + +func (in *CommitFence) DeepCopy() *CommitFence { + if in == nil { + return nil + } + out := new(CommitFence) + in.DeepCopyInto(out) + return out +} + +type CommitFenceOutcome string + +const ( + CommitFenceOutcomeCommitted CommitFenceOutcome = "committed" + CommitFenceOutcomeAborted CommitFenceOutcome = "aborted" +) + +type Committed struct { + Execution CommittedExecution `json:"execution"` + Plan CommittedPlan `json:"plan"` +} + +func (in *Committed) DeepCopyInto(out *Committed) { + *out = *in + in.Execution.DeepCopyInto(&out.Execution) + in.Plan.DeepCopyInto(&out.Plan) +} + +func (in *Committed) DeepCopy() *Committed { + if in == nil { + return nil + } + out := new(Committed) + in.DeepCopyInto(out) + return out +} + +type CommittedExecution struct { + AcceptedGeneration *int64 `json:"acceptedGeneration,omitempty"` + NodeLocal *NodeLocalExecution `json:"nodeLocal,omitempty"` + Release *ReleaseJournal `json:"release,omitempty"` + RemoteDra *RemoteDraExecution `json:"remoteDra,omitempty"` +} + +func (in *CommittedExecution) DeepCopyInto(out *CommittedExecution) { + *out = *in + if in.AcceptedGeneration != nil { + value := *in.AcceptedGeneration + out.AcceptedGeneration = &value + } + if in.NodeLocal != nil { + out.NodeLocal = in.NodeLocal.DeepCopy() + } + if in.Release != nil { + out.Release = in.Release.DeepCopy() + } + if in.RemoteDra != nil { + out.RemoteDra = in.RemoteDra.DeepCopy() + } +} + +func (in *CommittedExecution) DeepCopy() *CommittedExecution { + if in == nil { + return nil + } + out := new(CommittedExecution) + in.DeepCopyInto(out) + return out +} + +type CommittedPlan struct { + Placement PlanPlacement `json:"placement"` + PlanDigest PlanDigest `json:"planDigest"` + PlannedFromGeneration int64 `json:"plannedFromGeneration"` + RequestIntentDigest RequestIntentDigest `json:"requestIntentDigest"` + Revision int64 `json:"revision"` +} + +func (in *CommittedPlan) DeepCopyInto(out *CommittedPlan) { + *out = *in + in.Placement.DeepCopyInto(&out.Placement) +} + +func (in *CommittedPlan) DeepCopy() *CommittedPlan { + if in == nil { + return nil + } + out := new(CommittedPlan) + in.DeepCopyInto(out) + return out +} + +type ConnectorAssignment struct { + FromPartitionID string `json:"fromPartitionId"` + FromSelectedMemberIDs []string `json:"fromSelectedMemberIds"` + HealthEvidence ConnectorHealthEvidence `json:"healthEvidence"` + RealizedGap *int64 `json:"realizedGap,omitempty"` + Requirement PropSyncConnectorRequirement `json:"requirement"` + SelectedLaneMultiplicity *int64 `json:"selectedLaneMultiplicity,omitempty"` + ToPartitionID string `json:"toPartitionId"` + ToSelectedMemberIDs []string `json:"toSelectedMemberIds"` +} + +func (in *ConnectorAssignment) DeepCopyInto(out *ConnectorAssignment) { + *out = *in + if in.FromSelectedMemberIDs != nil { + out.FromSelectedMemberIDs = append(out.FromSelectedMemberIDs[:0:0], in.FromSelectedMemberIDs...) + } + in.HealthEvidence.DeepCopyInto(&out.HealthEvidence) + if in.RealizedGap != nil { + value := *in.RealizedGap + out.RealizedGap = &value + } + in.Requirement.DeepCopyInto(&out.Requirement) + if in.SelectedLaneMultiplicity != nil { + value := *in.SelectedLaneMultiplicity + out.SelectedLaneMultiplicity = &value + } + if in.ToSelectedMemberIDs != nil { + out.ToSelectedMemberIDs = append(out.ToSelectedMemberIDs[:0:0], in.ToSelectedMemberIDs...) + } +} + +func (in *ConnectorAssignment) DeepCopy() *ConnectorAssignment { + if in == nil { + return nil + } + out := new(ConnectorAssignment) + in.DeepCopyInto(out) + return out +} + +type ConnectorHealthEvidence struct { + Identity string `json:"identity"` + Source ConnectorHealthSource `json:"source"` +} + +func (in *ConnectorHealthEvidence) DeepCopyInto(out *ConnectorHealthEvidence) { + *out = *in +} + +func (in *ConnectorHealthEvidence) DeepCopy() *ConnectorHealthEvidence { + if in == nil { + return nil + } + out := new(ConnectorHealthEvidence) + in.DeepCopyInto(out) + return out +} + +type ConnectorHealthSource string + +const ( + ConnectorHealthSourceGeneratedHealthyProfile ConnectorHealthSource = "generatedHealthyProfile" +) + +type ConsumerBinding struct { + ConsumerID string `json:"consumerId"` + Current *CurrentBinding `json:"current,omitempty"` + Target *BindingTarget `json:"target,omitempty"` +} + +func (in *ConsumerBinding) DeepCopyInto(out *ConsumerBinding) { + *out = *in + if in.Current != nil { + out.Current = in.Current.DeepCopy() + } + if in.Target != nil { + out.Target = in.Target.DeepCopy() + } +} + +func (in *ConsumerBinding) DeepCopy() *ConsumerBinding { + if in == nil { + return nil + } + out := new(ConsumerBinding) + in.DeepCopyInto(out) + return out +} + +type ConsumerRequest struct { + ClaimNames []string `json:"claimNames"` + EndpointID *string `json:"endpointId,omitempty"` + ID string `json:"id"` + PodRef NamespacedName `json:"podRef"` +} + +func (in *ConsumerRequest) DeepCopyInto(out *ConsumerRequest) { + *out = *in + if in.ClaimNames != nil { + out.ClaimNames = append(out.ClaimNames[:0:0], in.ClaimNames...) + } + if in.EndpointID != nil { + value := *in.EndpointID + out.EndpointID = &value + } + in.PodRef.DeepCopyInto(&out.PodRef) +} + +func (in *ConsumerRequest) DeepCopy() *ConsumerRequest { + if in == nil { + return nil + } + out := new(ConsumerRequest) + in.DeepCopyInto(out) + return out +} + +type ConsumerSlot struct { + Claims []ClaimReference `json:"claims"` + ConsumerID string `json:"consumerId"` + EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` + EndpointID string `json:"endpointId"` + GroveLineage *GroveLineage `json:"groveLineage,omitempty"` + NodeName string `json:"nodeName"` + SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` +} + +func (in *ConsumerSlot) DeepCopyInto(out *ConsumerSlot) { + *out = *in + if in.Claims != nil { + out.Claims = make([]ClaimReference, len(in.Claims)) + for i := range in.Claims { + in.Claims[i].DeepCopyInto(&out.Claims[i]) + } + } + if in.EffectiveNodeRequests != nil { + out.EffectiveNodeRequests = make(map[string]int64, len(in.EffectiveNodeRequests)) + for key, value := range in.EffectiveNodeRequests { + out.EffectiveNodeRequests[key] = value + } + } + if in.GroveLineage != nil { + out.GroveLineage = in.GroveLineage.DeepCopy() + } +} + +func (in *ConsumerSlot) DeepCopy() *ConsumerSlot { + if in == nil { + return nil + } + out := new(ConsumerSlot) + in.DeepCopyInto(out) + return out +} + +type CurrentBinding struct { + BindingRevision int64 `json:"bindingRevision"` + PodRef ObjectReference `json:"podRef"` +} + +func (in *CurrentBinding) DeepCopyInto(out *CurrentBinding) { + *out = *in + in.PodRef.DeepCopyInto(&out.PodRef) +} + +func (in *CurrentBinding) DeepCopy() *CurrentBinding { + if in == nil { + return nil + } + out := new(CurrentBinding) + in.DeepCopyInto(out) + return out +} + +type CyborgPodCliqueReference struct { + Name string `json:"name"` + UID string `json:"uid"` +} + +func (in *CyborgPodCliqueReference) DeepCopyInto(out *CyborgPodCliqueReference) { + *out = *in +} + +func (in *CyborgPodCliqueReference) DeepCopy() *CyborgPodCliqueReference { + if in == nil { + return nil + } + out := new(CyborgPodCliqueReference) + in.DeepCopyInto(out) + return out +} + +type DerivedStageAssignment struct { + FabricID string `json:"fabricId"` + ID string `json:"id"` + Ordinal int64 `json:"ordinal"` + PartitionIDs []string `json:"partitionIds"` +} + +func (in *DerivedStageAssignment) DeepCopyInto(out *DerivedStageAssignment) { + *out = *in + if in.PartitionIDs != nil { + out.PartitionIDs = append(out.PartitionIDs[:0:0], in.PartitionIDs...) + } +} + +func (in *DerivedStageAssignment) DeepCopy() *DerivedStageAssignment { + if in == nil { + return nil + } + out := new(DerivedStageAssignment) + in.DeepCopyInto(out) + return out +} + +type ExactDeviceIdentity struct { + Device string `json:"device"` + Driver string `json:"driver"` + Pool string `json:"pool"` +} + +func (in *ExactDeviceIdentity) DeepCopyInto(out *ExactDeviceIdentity) { + *out = *in +} + +func (in *ExactDeviceIdentity) DeepCopy() *ExactDeviceIdentity { + if in == nil { + return nil + } + out := new(ExactDeviceIdentity) + in.DeepCopyInto(out) + return out +} + +type ExecutionBackend string + +const ( + ExecutionBackendRemoteDra ExecutionBackend = "remoteDra" + ExecutionBackendNodeLocal ExecutionBackend = "nodeLocal" +) + +type GroveLineage struct { + PodCliqueRef ObjectReference `json:"podCliqueRef"` + PodCliqueSetReplicaIndex int64 `json:"podCliqueSetReplicaIndex"` + PodIndex int64 `json:"podIndex"` + PodTemplateHash string `json:"podTemplateHash"` +} + +func (in *GroveLineage) DeepCopyInto(out *GroveLineage) { + *out = *in + in.PodCliqueRef.DeepCopyInto(&out.PodCliqueRef) +} + +func (in *GroveLineage) DeepCopy() *GroveLineage { + if in == nil { + return nil + } + out := new(GroveLineage) + in.DeepCopyInto(out) + return out +} + +type HxLogicalConnection struct { + FromLogicalDevice int64 `json:"fromLogicalDevice"` + ToLogicalDevice int64 `json:"toLogicalDevice"` +} + +func (in *HxLogicalConnection) DeepCopyInto(out *HxLogicalConnection) { + *out = *in +} + +func (in *HxLogicalConnection) DeepCopy() *HxLogicalConnection { + if in == nil { + return nil + } + out := new(HxLogicalConnection) + in.DeepCopyInto(out) + return out +} + +type LPUFamilyCoordinate struct { + Family TargetFamily `json:"family"` + Hemisphere *int64 `json:"hemisphere,omitempty"` + Node *int64 `json:"node,omitempty"` + Rack int64 `json:"rack"` + Tray *int64 `json:"tray,omitempty"` +} + +func (in *LPUFamilyCoordinate) DeepCopyInto(out *LPUFamilyCoordinate) { + *out = *in + if in.Hemisphere != nil { + value := *in.Hemisphere + out.Hemisphere = &value + } + if in.Node != nil { + value := *in.Node + out.Node = &value + } + if in.Tray != nil { + value := *in.Tray + out.Tray = &value + } +} + +func (in *LPUFamilyCoordinate) DeepCopy() *LPUFamilyCoordinate { + if in == nil { + return nil + } + out := new(LPUFamilyCoordinate) + in.DeepCopyInto(out) + return out +} + +type LPUPartitionPlacement struct { + ClaimName string `json:"claimName"` + ClaimUID string `json:"claimUid"` + Family TargetFamily `json:"family"` + OrderedMembers []AtomicLPUMemberIdentity `json:"orderedMembers"` + PartitionID string `json:"partitionId"` + RequestedExtent *[]int64 `json:"requestedExtent,omitempty"` + XtShape *Xt8888PartitionShape `json:"xtShape,omitempty"` +} + +func (in *LPUPartitionPlacement) DeepCopyInto(out *LPUPartitionPlacement) { + *out = *in + if in.OrderedMembers != nil { + out.OrderedMembers = make([]AtomicLPUMemberIdentity, len(in.OrderedMembers)) + for i := range in.OrderedMembers { + in.OrderedMembers[i].DeepCopyInto(&out.OrderedMembers[i]) + } + } + if in.RequestedExtent != nil { + value := make([]int64, len(*in.RequestedExtent)) + copy(value, *in.RequestedExtent) + out.RequestedExtent = &value + } + if in.XtShape != nil { + value := *in.XtShape + out.XtShape = &value + } +} + +func (in *LPUPartitionPlacement) DeepCopy() *LPUPartitionPlacement { + if in == nil { + return nil + } + out := new(LPUPartitionPlacement) + in.DeepCopyInto(out) + return out +} + +type LPUPipelineRequestSpec struct { + AllocationMetadata runtime.RawExtension `json:"allocationMetadata"` + CyborgPodCliqueRef *CyborgPodCliqueReference `json:"cyborgPodCliqueRef,omitempty"` + ExecutionBackend ExecutionBackend `json:"executionBackend"` + NodeLocal *NodeLocalRequest `json:"nodeLocal,omitempty"` + Partitions []PartitionRequest `json:"partitions"` + PodGangRef NamespacedName `json:"podGangRef"` + PropSyncConnectors []PropSyncConnectorRequest `json:"propSyncConnectors"` + RemoteDra *RemoteDraRequest `json:"remoteDra,omitempty"` + RepairPolicy *RepairPolicy `json:"repairPolicy,omitempty"` + TargetFamily TargetFamily `json:"targetFamily"` + WorkloadMode WorkloadMode `json:"workloadMode"` +} + +func (in *LPUPipelineRequestSpec) DeepCopyInto(out *LPUPipelineRequestSpec) { + *out = *in + in.AllocationMetadata.DeepCopyInto(&out.AllocationMetadata) + if in.CyborgPodCliqueRef != nil { + out.CyborgPodCliqueRef = in.CyborgPodCliqueRef.DeepCopy() + } + if in.NodeLocal != nil { + out.NodeLocal = in.NodeLocal.DeepCopy() + } + if in.Partitions != nil { + out.Partitions = make([]PartitionRequest, len(in.Partitions)) + for i := range in.Partitions { + in.Partitions[i].DeepCopyInto(&out.Partitions[i]) + } + } + in.PodGangRef.DeepCopyInto(&out.PodGangRef) + if in.PropSyncConnectors != nil { + out.PropSyncConnectors = make([]PropSyncConnectorRequest, len(in.PropSyncConnectors)) + for i := range in.PropSyncConnectors { + in.PropSyncConnectors[i].DeepCopyInto(&out.PropSyncConnectors[i]) + } + } + if in.RemoteDra != nil { + out.RemoteDra = in.RemoteDra.DeepCopy() + } + if in.RepairPolicy != nil { + out.RepairPolicy = in.RepairPolicy.DeepCopy() + } +} + +func (in *LPUPipelineRequestSpec) DeepCopy() *LPUPipelineRequestSpec { + if in == nil { + return nil + } + out := new(LPUPipelineRequestSpec) + in.DeepCopyInto(out) + return out +} + +type LPUPipelineRequestStatus struct { + CommitFence *CommitFence `json:"commitFence,omitempty"` + Committed *Committed `json:"committed,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + Diagnostics []StatusDiagnostic `json:"diagnostics,omitempty"` + LastPlanRevision int64 `json:"lastPlanRevision"` + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + Phase RequestPhase `json:"phase"` +} + +func (in *LPUPipelineRequestStatus) DeepCopyInto(out *LPUPipelineRequestStatus) { + *out = *in + if in.CommitFence != nil { + out.CommitFence = in.CommitFence.DeepCopy() + } + if in.Committed != nil { + out.Committed = in.Committed.DeepCopy() + } + if in.Conditions != nil { + out.Conditions = make([]metav1.Condition, len(in.Conditions)) + for i := range in.Conditions { + in.Conditions[i].DeepCopyInto(&out.Conditions[i]) + } + } + if in.Diagnostics != nil { + out.Diagnostics = make([]StatusDiagnostic, len(in.Diagnostics)) + for i := range in.Diagnostics { + in.Diagnostics[i].DeepCopyInto(&out.Diagnostics[i]) + } + } + if in.ObservedGeneration != nil { + value := *in.ObservedGeneration + out.ObservedGeneration = &value + } +} + +func (in *LPUPipelineRequestStatus) DeepCopy() *LPUPipelineRequestStatus { + if in == nil { + return nil + } + out := new(LPUPipelineRequestStatus) + in.DeepCopyInto(out) + return out +} + +type NamespacedName struct { + Name string `json:"name"` + Namespace *string `json:"namespace,omitempty"` +} + +func (in *NamespacedName) DeepCopyInto(out *NamespacedName) { + *out = *in + if in.Namespace != nil { + value := *in.Namespace + out.Namespace = &value + } +} + +func (in *NamespacedName) DeepCopy() *NamespacedName { + if in == nil { + return nil + } + out := new(NamespacedName) + in.DeepCopyInto(out) + return out +} + +type NodeLocalConnectorEvidence struct { + FromPartitionID string `json:"fromPartitionId"` + HealthEvidence ConnectorHealthEvidence `json:"healthEvidence"` + RealizedGap *int64 `json:"realizedGap,omitempty"` + SelectedLaneMultiplicity *int64 `json:"selectedLaneMultiplicity,omitempty"` + ToPartitionID string `json:"toPartitionId"` +} + +func (in *NodeLocalConnectorEvidence) DeepCopyInto(out *NodeLocalConnectorEvidence) { + *out = *in + in.HealthEvidence.DeepCopyInto(&out.HealthEvidence) + if in.RealizedGap != nil { + value := *in.RealizedGap + out.RealizedGap = &value + } + if in.SelectedLaneMultiplicity != nil { + value := *in.SelectedLaneMultiplicity + out.SelectedLaneMultiplicity = &value + } +} + +func (in *NodeLocalConnectorEvidence) DeepCopy() *NodeLocalConnectorEvidence { + if in == nil { + return nil + } + out := new(NodeLocalConnectorEvidence) + in.DeepCopyInto(out) + return out +} + +type NodeLocalEndpoint struct { + Coordinate LPUFamilyCoordinate `json:"coordinate"` + EndpointKey string `json:"endpointKey"` + FabricID string `json:"fabricId"` +} + +func (in *NodeLocalEndpoint) DeepCopyInto(out *NodeLocalEndpoint) { + *out = *in + in.Coordinate.DeepCopyInto(&out.Coordinate) +} + +func (in *NodeLocalEndpoint) DeepCopy() *NodeLocalEndpoint { + if in == nil { + return nil + } + out := new(NodeLocalEndpoint) + in.DeepCopyInto(out) + return out +} + +type NodeLocalExecution struct { + PartitionSelections []NodeLocalPartitionExecution `json:"partitionSelections"` +} + +func (in *NodeLocalExecution) DeepCopyInto(out *NodeLocalExecution) { + *out = *in + if in.PartitionSelections != nil { + out.PartitionSelections = make([]NodeLocalPartitionExecution, len(in.PartitionSelections)) + for i := range in.PartitionSelections { + in.PartitionSelections[i].DeepCopyInto(&out.PartitionSelections[i]) + } + } +} + +func (in *NodeLocalExecution) DeepCopy() *NodeLocalExecution { + if in == nil { + return nil + } + out := new(NodeLocalExecution) + in.DeepCopyInto(out) + return out +} + +type NodeLocalPartitionExecution struct { + PartitionID string `json:"partitionId"` + SelectedRows []NodeLocalRowExecution `json:"selectedRows"` +} + +func (in *NodeLocalPartitionExecution) DeepCopyInto(out *NodeLocalPartitionExecution) { + *out = *in + if in.SelectedRows != nil { + out.SelectedRows = make([]NodeLocalRowExecution, len(in.SelectedRows)) + for i := range in.SelectedRows { + in.SelectedRows[i].DeepCopyInto(&out.SelectedRows[i]) + } + } +} + +func (in *NodeLocalPartitionExecution) DeepCopy() *NodeLocalPartitionExecution { + if in == nil { + return nil + } + out := new(NodeLocalPartitionExecution) + in.DeepCopyInto(out) + return out +} + +type NodeLocalPartitionMapping struct { + ModelPartitionID int64 `json:"modelPartitionId"` + PartitionID string `json:"partitionId"` +} + +func (in *NodeLocalPartitionMapping) DeepCopyInto(out *NodeLocalPartitionMapping) { + *out = *in +} + +func (in *NodeLocalPartitionMapping) DeepCopy() *NodeLocalPartitionMapping { + if in == nil { + return nil + } + out := new(NodeLocalPartitionMapping) + in.DeepCopyInto(out) + return out +} + +type NodeLocalPartitionSelection struct { + ModelPartitionID int64 `json:"modelPartitionId"` + PartitionID string `json:"partitionId"` + SelectedRows []NodeLocalSelectedRow `json:"selectedRows"` +} + +func (in *NodeLocalPartitionSelection) DeepCopyInto(out *NodeLocalPartitionSelection) { + *out = *in + if in.SelectedRows != nil { + out.SelectedRows = make([]NodeLocalSelectedRow, len(in.SelectedRows)) + for i := range in.SelectedRows { + in.SelectedRows[i].DeepCopyInto(&out.SelectedRows[i]) + } + } +} + +func (in *NodeLocalPartitionSelection) DeepCopy() *NodeLocalPartitionSelection { + if in == nil { + return nil + } + out := new(NodeLocalPartitionSelection) + in.DeepCopyInto(out) + return out +} + +type NodeLocalPlacement struct { + CompilerMetadataDigest string `json:"compilerMetadataDigest"` + ConnectorEvidence []NodeLocalConnectorEvidence `json:"connectorEvidence,omitempty"` + EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` + InventoryRevision string `json:"inventoryRevision"` + Model string `json:"model"` + Objective PlanObjective `json:"objective"` + PartitionSelections []NodeLocalPartitionSelection `json:"partitionSelections"` + PodGangRef ObjectReference `json:"podGangRef"` + SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` + WorkloadMode WorkloadMode `json:"workloadMode"` +} + +func (in *NodeLocalPlacement) DeepCopyInto(out *NodeLocalPlacement) { + *out = *in + if in.ConnectorEvidence != nil { + out.ConnectorEvidence = make([]NodeLocalConnectorEvidence, len(in.ConnectorEvidence)) + for i := range in.ConnectorEvidence { + in.ConnectorEvidence[i].DeepCopyInto(&out.ConnectorEvidence[i]) + } + } + if in.EffectiveNodeRequests != nil { + out.EffectiveNodeRequests = make(map[string]int64, len(in.EffectiveNodeRequests)) + for key, value := range in.EffectiveNodeRequests { + out.EffectiveNodeRequests[key] = value + } + } + in.Objective.DeepCopyInto(&out.Objective) + if in.PartitionSelections != nil { + out.PartitionSelections = make([]NodeLocalPartitionSelection, len(in.PartitionSelections)) + for i := range in.PartitionSelections { + in.PartitionSelections[i].DeepCopyInto(&out.PartitionSelections[i]) + } + } + in.PodGangRef.DeepCopyInto(&out.PodGangRef) +} + +func (in *NodeLocalPlacement) DeepCopy() *NodeLocalPlacement { + if in == nil { + return nil + } + out := new(NodeLocalPlacement) + in.DeepCopyInto(out) + return out +} + +type NodeLocalRequest struct { + Model string `json:"model"` + PartitionMappings []NodeLocalPartitionMapping `json:"partitionMappings"` +} + +func (in *NodeLocalRequest) DeepCopyInto(out *NodeLocalRequest) { + *out = *in + if in.PartitionMappings != nil { + out.PartitionMappings = make([]NodeLocalPartitionMapping, len(in.PartitionMappings)) + for i := range in.PartitionMappings { + in.PartitionMappings[i].DeepCopyInto(&out.PartitionMappings[i]) + } + } +} + +func (in *NodeLocalRequest) DeepCopy() *NodeLocalRequest { + if in == nil { + return nil + } + out := new(NodeLocalRequest) + in.DeepCopyInto(out) + return out +} + +type NodeLocalRowExecution struct { + Current *CurrentBinding `json:"current,omitempty"` + RankInPartition int64 `json:"rankInPartition"` + Target *BindingTarget `json:"target,omitempty"` +} + +func (in *NodeLocalRowExecution) DeepCopyInto(out *NodeLocalRowExecution) { + *out = *in + if in.Current != nil { + out.Current = in.Current.DeepCopy() + } + if in.Target != nil { + out.Target = in.Target.DeepCopy() + } +} + +func (in *NodeLocalRowExecution) DeepCopy() *NodeLocalRowExecution { + if in == nil { + return nil + } + out := new(NodeLocalRowExecution) + in.DeepCopyInto(out) + return out +} + +type NodeLocalSelectedRow struct { + Endpoint NodeLocalEndpoint `json:"endpoint"` + GroveLineage *GroveLineage `json:"groveLineage,omitempty"` + NodeRef NodeReference `json:"nodeRef"` + RankInPartition int64 `json:"rankInPartition"` +} + +func (in *NodeLocalSelectedRow) DeepCopyInto(out *NodeLocalSelectedRow) { + *out = *in + in.Endpoint.DeepCopyInto(&out.Endpoint) + if in.GroveLineage != nil { + out.GroveLineage = in.GroveLineage.DeepCopy() + } + in.NodeRef.DeepCopyInto(&out.NodeRef) +} + +func (in *NodeLocalSelectedRow) DeepCopy() *NodeLocalSelectedRow { + if in == nil { + return nil + } + out := new(NodeLocalSelectedRow) + in.DeepCopyInto(out) + return out +} + +type NodeReference struct { + Name string `json:"name"` + UID string `json:"uid"` +} + +func (in *NodeReference) DeepCopyInto(out *NodeReference) { + *out = *in +} + +func (in *NodeReference) DeepCopy() *NodeReference { + if in == nil { + return nil + } + out := new(NodeReference) + in.DeepCopyInto(out) + return out +} + +type ObjectReference struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + UID string `json:"uid"` +} + +func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { + *out = *in +} + +func (in *ObjectReference) DeepCopy() *ObjectReference { + if in == nil { + return nil + } + out := new(ObjectReference) + in.DeepCopyInto(out) + return out +} + +type PartitionClaimRequest struct { + ClaimName string `json:"claimName"` + PartitionID string `json:"partitionId"` +} + +func (in *PartitionClaimRequest) DeepCopyInto(out *PartitionClaimRequest) { + *out = *in +} + +func (in *PartitionClaimRequest) DeepCopy() *PartitionClaimRequest { + if in == nil { + return nil + } + out := new(PartitionClaimRequest) + in.DeepCopyInto(out) + return out +} + +type PartitionConductorAssignment struct { + ClaimName string `json:"claimName"` + ConductorID string `json:"conductorId"` + EndpointID string `json:"endpointId"` + PartitionID string `json:"partitionId"` + StageID string `json:"stageId"` +} + +func (in *PartitionConductorAssignment) DeepCopyInto(out *PartitionConductorAssignment) { + *out = *in +} + +func (in *PartitionConductorAssignment) DeepCopy() *PartitionConductorAssignment { + if in == nil { + return nil + } + out := new(PartitionConductorAssignment) + in.DeepCopyInto(out) + return out +} + +type PartitionRequest struct { + CompilerPartitionID int64 `json:"compilerPartitionId"` + Extent *[]int64 `json:"extent,omitempty"` + ID string `json:"id"` + Ordinal int64 `json:"ordinal"` + XtShape *Xt8888PartitionShape `json:"xtShape,omitempty"` +} + +func (in *PartitionRequest) DeepCopyInto(out *PartitionRequest) { + *out = *in + if in.Extent != nil { + value := make([]int64, len(*in.Extent)) + copy(value, *in.Extent) + out.Extent = &value + } + if in.XtShape != nil { + value := *in.XtShape + out.XtShape = &value + } +} + +func (in *PartitionRequest) DeepCopy() *PartitionRequest { + if in == nil { + return nil + } + out := new(PartitionRequest) + in.DeepCopyInto(out) + return out +} + +type PlanDigest string + +type PlanObjective struct { + LoadImbalance int64 `json:"loadImbalance"` + NonPreferredAnchorCount int64 `json:"nonPreferredAnchorCount"` + TraversalCost int64 `json:"traversalCost"` + UsedFabricPodCount int64 `json:"usedFabricPodCount"` +} + +func (in *PlanObjective) DeepCopyInto(out *PlanObjective) { + *out = *in +} + +func (in *PlanObjective) DeepCopy() *PlanObjective { + if in == nil { + return nil + } + out := new(PlanObjective) + in.DeepCopyInto(out) + return out +} + +type PlanPlacement struct { + ExecutionBackend ExecutionBackend `json:"executionBackend"` + NodeLocal *NodeLocalPlacement `json:"nodeLocal,omitempty"` + RemoteDra *RemoteDraPlacement `json:"remoteDra,omitempty"` +} + +func (in *PlanPlacement) DeepCopyInto(out *PlanPlacement) { + *out = *in + if in.NodeLocal != nil { + out.NodeLocal = in.NodeLocal.DeepCopy() + } + if in.RemoteDra != nil { + out.RemoteDra = in.RemoteDra.DeepCopy() + } +} + +func (in *PlanPlacement) DeepCopy() *PlanPlacement { + if in == nil { + return nil + } + out := new(PlanPlacement) + in.DeepCopyInto(out) + return out +} + +type PropSyncConnectorKind string + +const ( + PropSyncConnectorKindHxPropSyncV1 PropSyncConnectorKind = "hxPropSyncV1" + PropSyncConnectorKindXt8888Gap PropSyncConnectorKind = "xt8888Gap" +) + +type PropSyncConnectorRequest struct { + FromPartitionID string `json:"fromPartitionId"` + Requirement PropSyncConnectorRequirement `json:"requirement"` + ToPartitionID string `json:"toPartitionId"` +} + +func (in *PropSyncConnectorRequest) DeepCopyInto(out *PropSyncConnectorRequest) { + *out = *in + in.Requirement.DeepCopyInto(&out.Requirement) +} + +func (in *PropSyncConnectorRequest) DeepCopy() *PropSyncConnectorRequest { + if in == nil { + return nil + } + out := new(PropSyncConnectorRequest) + in.DeepCopyInto(out) + return out +} + +type PropSyncConnectorRequirement struct { + AcceptableLaneMultiplicities *[]int64 `json:"acceptableLaneMultiplicities,omitempty"` + Connections *[]HxLogicalConnection `json:"connections,omitempty"` + Kind PropSyncConnectorKind `json:"kind"` + MaxInterPartitionOffset *int64 `json:"maxInterPartitionOffset,omitempty"` +} + +func (in *PropSyncConnectorRequirement) DeepCopyInto(out *PropSyncConnectorRequirement) { + *out = *in + if in.AcceptableLaneMultiplicities != nil { + value := make([]int64, len(*in.AcceptableLaneMultiplicities)) + copy(value, *in.AcceptableLaneMultiplicities) + out.AcceptableLaneMultiplicities = &value + } + if in.Connections != nil { + value := make([]HxLogicalConnection, len(*in.Connections)) + copy(value, *in.Connections) + out.Connections = &value + } + if in.MaxInterPartitionOffset != nil { + value := *in.MaxInterPartitionOffset + out.MaxInterPartitionOffset = &value + } +} + +func (in *PropSyncConnectorRequirement) DeepCopy() *PropSyncConnectorRequirement { + if in == nil { + return nil + } + out := new(PropSyncConnectorRequirement) + in.DeepCopyInto(out) + return out +} + +type ReleaseJournal struct { + Reason ReleaseReason `json:"reason"` + RequestedAtGeneration int64 `json:"requestedAtGeneration"` +} + +func (in *ReleaseJournal) DeepCopyInto(out *ReleaseJournal) { + *out = *in +} + +func (in *ReleaseJournal) DeepCopy() *ReleaseJournal { + if in == nil { + return nil + } + out := new(ReleaseJournal) + in.DeepCopyInto(out) + return out +} + +type ReleaseReason string + +const ( + ReleaseReasonRequestChanged ReleaseReason = "requestChanged" + ReleaseReasonRequestDeleted ReleaseReason = "requestDeleted" + ReleaseReasonDependencyChanged ReleaseReason = "dependencyChanged" + ReleaseReasonBindingFailed ReleaseReason = "bindingFailed" +) + +type RemoteDraExecution struct { + ClaimStates []ClaimState `json:"claimStates"` + ConsumerBindings []ConsumerBinding `json:"consumerBindings"` +} + +func (in *RemoteDraExecution) DeepCopyInto(out *RemoteDraExecution) { + *out = *in + if in.ClaimStates != nil { + out.ClaimStates = make([]ClaimState, len(in.ClaimStates)) + for i := range in.ClaimStates { + in.ClaimStates[i].DeepCopyInto(&out.ClaimStates[i]) + } + } + if in.ConsumerBindings != nil { + out.ConsumerBindings = make([]ConsumerBinding, len(in.ConsumerBindings)) + for i := range in.ConsumerBindings { + in.ConsumerBindings[i].DeepCopyInto(&out.ConsumerBindings[i]) + } + } +} + +func (in *RemoteDraExecution) DeepCopy() *RemoteDraExecution { + if in == nil { + return nil + } + out := new(RemoteDraExecution) + in.DeepCopyInto(out) + return out +} + +type RemoteDraPlacement struct { + ActiveFabricIDs *[]string `json:"activeFabricIds,omitempty"` + CompilerMetadataDigest string `json:"compilerMetadataDigest"` + ConnectorAssignments []ConnectorAssignment `json:"connectorAssignments,omitempty"` + ConsumerSlots []ConsumerSlot `json:"consumerSlots"` + DerivedStages []DerivedStageAssignment `json:"derivedStages"` + InventoryRevision string `json:"inventoryRevision"` + Objective PlanObjective `json:"objective"` + PartitionConductors []PartitionConductorAssignment `json:"partitionConductors"` + PartitionPlacements []LPUPartitionPlacement `json:"partitionPlacements"` + PodGangRef ObjectReference `json:"podGangRef"` + WorkloadMode WorkloadMode `json:"workloadMode"` +} + +func (in *RemoteDraPlacement) DeepCopyInto(out *RemoteDraPlacement) { + *out = *in + if in.ActiveFabricIDs != nil { + value := make([]string, len(*in.ActiveFabricIDs)) + copy(value, *in.ActiveFabricIDs) + out.ActiveFabricIDs = &value + } + if in.ConnectorAssignments != nil { + out.ConnectorAssignments = make([]ConnectorAssignment, len(in.ConnectorAssignments)) + for i := range in.ConnectorAssignments { + in.ConnectorAssignments[i].DeepCopyInto(&out.ConnectorAssignments[i]) + } + } + if in.ConsumerSlots != nil { + out.ConsumerSlots = make([]ConsumerSlot, len(in.ConsumerSlots)) + for i := range in.ConsumerSlots { + in.ConsumerSlots[i].DeepCopyInto(&out.ConsumerSlots[i]) + } + } + if in.DerivedStages != nil { + out.DerivedStages = make([]DerivedStageAssignment, len(in.DerivedStages)) + for i := range in.DerivedStages { + in.DerivedStages[i].DeepCopyInto(&out.DerivedStages[i]) + } + } + in.Objective.DeepCopyInto(&out.Objective) + if in.PartitionConductors != nil { + out.PartitionConductors = make([]PartitionConductorAssignment, len(in.PartitionConductors)) + for i := range in.PartitionConductors { + in.PartitionConductors[i].DeepCopyInto(&out.PartitionConductors[i]) + } + } + if in.PartitionPlacements != nil { + out.PartitionPlacements = make([]LPUPartitionPlacement, len(in.PartitionPlacements)) + for i := range in.PartitionPlacements { + in.PartitionPlacements[i].DeepCopyInto(&out.PartitionPlacements[i]) + } + } + in.PodGangRef.DeepCopyInto(&out.PodGangRef) +} + +func (in *RemoteDraPlacement) DeepCopy() *RemoteDraPlacement { + if in == nil { + return nil + } + out := new(RemoteDraPlacement) + in.DeepCopyInto(out) + return out +} + +type RemoteDraRequest struct { + Consumers []ConsumerRequest `json:"consumers"` + PartitionClaims []PartitionClaimRequest `json:"partitionClaims"` +} + +func (in *RemoteDraRequest) DeepCopyInto(out *RemoteDraRequest) { + *out = *in + if in.Consumers != nil { + out.Consumers = make([]ConsumerRequest, len(in.Consumers)) + for i := range in.Consumers { + in.Consumers[i].DeepCopyInto(&out.Consumers[i]) + } + } + if in.PartitionClaims != nil { + out.PartitionClaims = make([]PartitionClaimRequest, len(in.PartitionClaims)) + for i := range in.PartitionClaims { + in.PartitionClaims[i].DeepCopyInto(&out.PartitionClaims[i]) + } + } +} + +func (in *RemoteDraRequest) DeepCopy() *RemoteDraRequest { + if in == nil { + return nil + } + out := new(RemoteDraRequest) + in.DeepCopyInto(out) + return out +} + +type RepairPolicy struct { + Mode RepairPolicyMode `json:"mode"` +} + +func (in *RepairPolicy) DeepCopyInto(out *RepairPolicy) { + *out = *in +} + +func (in *RepairPolicy) DeepCopy() *RepairPolicy { + if in == nil { + return nil + } + out := new(RepairPolicy) + in.DeepCopyInto(out) + return out +} + +type RepairPolicyMode string + +const ( + RepairPolicyModeColdReplacement RepairPolicyMode = "coldReplacement" +) + +type RequestIntentDigest string + +type RequestPhase string + +const ( + RequestPhasePending RequestPhase = "pending" + RequestPhaseNoFit RequestPhase = "noFit" + RequestPhaseUnsupported RequestPhase = "unsupported" + RequestPhasePlanned RequestPhase = "planned" + RequestPhaseReserving RequestPhase = "reserving" + RequestPhaseBinding RequestPhase = "binding" + RequestPhaseBound RequestPhase = "bound" + RequestPhaseDegraded RequestPhase = "degraded" + RequestPhaseReleasing RequestPhase = "releasing" + RequestPhaseReleased RequestPhase = "released" +) + +type StatusDiagnostic struct { + Code string `json:"code"` + Detail string `json:"detail"` + Subject string `json:"subject"` +} + +func (in *StatusDiagnostic) DeepCopyInto(out *StatusDiagnostic) { + *out = *in +} + +func (in *StatusDiagnostic) DeepCopy() *StatusDiagnostic { + if in == nil { + return nil + } + out := new(StatusDiagnostic) + in.DeepCopyInto(out) + return out +} + +type TargetFamily string + +const ( + TargetFamilyXt8888 TargetFamily = "xt8888" + TargetFamilyHx16x8x2x3 TargetFamily = "hx16x8x2x3" +) + +type WorkloadMode string + +const ( + WorkloadModeV2LPUOnly WorkloadMode = "v2LpuOnly" + WorkloadModeV2StrictHybrid WorkloadMode = "v2StrictHybrid" + WorkloadModeV3HxLPUOnly WorkloadMode = "v3HxLpuOnly" + WorkloadModeV3HxStrictHybrid WorkloadMode = "v3HxStrictHybrid" +) + +type Xt8888PartitionShape string + +const ( + Xt8888PartitionShapeC8 Xt8888PartitionShape = "c8" + Xt8888PartitionShapeC16 Xt8888PartitionShape = "c16" + Xt8888PartitionShapeC24 Xt8888PartitionShape = "c24" + Xt8888PartitionShapeC32 Xt8888PartitionShape = "c32" + Xt8888PartitionShapeC40 Xt8888PartitionShape = "c40" + Xt8888PartitionShapeC48 Xt8888PartitionShape = "c48" + Xt8888PartitionShapeC56 Xt8888PartitionShape = "c56" + Xt8888PartitionShapeC64 Xt8888PartitionShape = "c64" + Xt8888PartitionShapeC96 Xt8888PartitionShape = "c96" + Xt8888PartitionShapeC128 Xt8888PartitionShape = "c128" +) + +var SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { + scheme.AddKnownTypeWithName(GroupVersion.WithKind("LpuPipelineRequest"), &LPUPipelineRequest{}) + scheme.AddKnownTypeWithName(GroupVersion.WithKind("LpuPipelineRequestList"), &LPUPipelineRequestList{}) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +}) + +var AddToScheme = SchemeBuilder.AddToScheme + +func (in *LPUPipelineRequest) DeepCopyInto(out *LPUPipelineRequest) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + out.Status = in.Status.DeepCopy() + } +} + +func (in *LPUPipelineRequest) DeepCopy() *LPUPipelineRequest { + if in == nil { + return nil + } + out := new(LPUPipelineRequest) + in.DeepCopyInto(out) + return out +} + +func (in *LPUPipelineRequest) DeepCopyObject() runtime.Object { + if copied := in.DeepCopy(); copied != nil { + return copied + } + return nil +} + +func (in *LPUPipelineRequestList) DeepCopyInto(out *LPUPipelineRequestList) { + *out = *in + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + out.Items = make([]LPUPipelineRequest, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *LPUPipelineRequestList) DeepCopy() *LPUPipelineRequestList { + if in == nil { + return nil + } + out := new(LPUPipelineRequestList) + in.DeepCopyInto(out) + return out +} + +func (in *LPUPipelineRequestList) DeepCopyObject() runtime.Object { + if copied := in.DeepCopy(); copied != nil { + return copied + } + return nil +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go index f45da6671813..9e8fb69c69e0 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go @@ -58,7 +58,6 @@ func NewDGDDefaulter(operatorVersion string) *DGDDefaulter { // On every operation: defaults nil component Replicas to 1 and persists the // replica counts implied by explicit multinode roles. // On CREATE: sets the controller-owned workload provider from routing intent before provider-specific defaults. -// Existing unannotated DGDs remain unselected for controller-side workload adoption. // On the Grove pathway: defaults nil MinAvailable to 1. Scaling to replicas=0 // does not rewrite MinAvailable; it remains the component's configured minimum viable unit. // On CREATE: stamps nvidia.com/dynamo-operator-origin-version with the operator version. diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go index 7d9719b185d9..e804d9c819b2 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go @@ -166,6 +166,34 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { }), wantWebhookErrs: []string{"spec.roles[0].podTemplate: Forbidden: is not supported for this component role"}, }, + { + name: "v1alpha1 rejects the legacy lpu component type", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.ComponentType = "lpu" + }), + wantCELErr: "spec: Invalid value: componentType lpu is not supported; use lpx", + }, + { + name: "standalone v1alpha1 canonical lpx component is rejected", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.ComponentType = "lpx" + dcd.Spec.LPX = &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"} + dcd.Spec.Roles = []nvidiacomv1alpha1.ComponentRoleSpec{{Name: nvidiacomv1alpha1.ComponentRoleLeader}} + }), + wantCELErr: "spec: Invalid value: standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment", + wantWarnings: []string{`unknown field "spec.lpx"`}, + }, + { + name: "standalone canonical lpx component is rejected", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.ComponentType = nvidiacomv1beta1.ComponentTypeLPX + dcd.Spec.PodTemplate = nil + dcd.Spec.LPX = &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"} + dcd.Spec.Roles = []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLeader}} + }), + wantCELErr: "spec: Invalid value: standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment", + wantWarnings: []string{`unknown field "spec.lpx"`}, + }, { name: "v1beta1 main image is required when pod template is absent on create", deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { @@ -1802,7 +1830,6 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { oldObject: tt.oldDeployment, gates: gates, seedWithoutWebhook: tt.seedWithoutWebhook, - withoutTopology: true, wantSchemaError: tt.wantSchemaErr, wantCELError: tt.wantCELErr, wantWebhookErrors: tt.wantWebhookErrs, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 49fedeefde5a..cb4b7982a82a 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -49,12 +49,8 @@ type DynamoGraphDeploymentValidator struct { // NewDynamoGraphDeploymentValidator creates a validator for v1beta1 DynamoGraphDeployment. // mgr must not be nil. -func NewDynamoGraphDeploymentValidator( - mgr ctrl.Manager, -) *DynamoGraphDeploymentValidator { - return &DynamoGraphDeploymentValidator{ - mgr: mgr, - } +func NewDynamoGraphDeploymentValidator(mgr ctrl.Manager) *DynamoGraphDeploymentValidator { + return &DynamoGraphDeploymentValidator{mgr: mgr} } // dynamoGraphDeploymentValidation carries DGD-specific request state. @@ -66,7 +62,7 @@ type dynamoGraphDeploymentValidation struct { } type dynamoGraphDeploymentSpecValidationOptions struct { - dgdName string + pcsName string generation int64 workloadProvider string grovePathway bool @@ -212,7 +208,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( oldComponents = componentsByName(oldDGD.Spec.Components) } specOpts := dynamoGraphDeploymentSpecValidationOptions{ - dgdName: dgd.Name, + pcsName: dynamo.PCSNameForDGD(dgd.Name, dgd.Spec.Components), generation: dgd.Generation, workloadProvider: workloadProvider, grovePathway: grovePathway, @@ -221,6 +217,11 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( } allErrs = append(allErrs, v.validateDynamoGraphDeploymentSpec(&dgd.Spec, field.NewPath("spec"), specOpts)...) + // LPX children require the durable Grove route before runtime validation can be deferred. + if dgd.HasLPXComponent() && !grovePathway { + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "components"), grovePathwayRequirement)) + } + return allErrs } @@ -267,7 +268,6 @@ func (v *dynamoGraphDeploymentValidation) validateObjectMeta( []string{"pod", "container"}, )) } - // Restrict the durable workload provider to programs implemented by the controller. if value, exists := objectMeta.Annotations[consts.KubeAnnotationWorkloadProvider]; exists && !isSupportedWorkloadProvider(value) { @@ -289,7 +289,9 @@ func (v *dynamoGraphDeploymentValidation) validateObjectMeta( return allErrs } -// validateDynamoGraphDeploymentSpec validates spec. spec and fldPath must not be nil. +// validateDynamoGraphDeploymentSpec validates dgd.Spec. dgd and fldPath must not be nil. +// +//nolint:gocyclo // Keep the complete graph validation pass and its field paths together. func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( spec *nvidiacomv1beta1.DynamoGraphDeploymentSpec, fldPath *field.Path, @@ -325,24 +327,31 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( component := &spec.Components[i] componentPath := componentsPath.Index(i) - if opts.grovePathway { - combinedLength, detail := dgdComponentResourceNameLength(opts.dgdName, spec.Components, component) - if combinedLength > maxCombinedResourceNameLength { + // Externally managed components validate their generated names in their own controller. + if opts.grovePathway && !component.ManagedByExternalController() { + combinedLength := len(opts.pcsName) + dynamo.ComponentNameBudget(component) + if combinedLength > consts.MaxCombinedGroveResourceNameLength { allErrs = append(allErrs, field.Invalid( componentPath.Child("name"), component.ComponentName, fmt.Sprintf( - "combined resource name length %d exceeds the %d-character pod-name limit (%s); shorten DynamoGraphDeployment name %q or component name %q", + "combined Grove resource name length %d exceeds the %d-character limit; shorten the deployment or component name", combinedLength, - maxCombinedResourceNameLength, - detail, - opts.dgdName, - component.ComponentName, + consts.MaxCombinedGroveResourceNameLength, ), )) } } + // Ordinary components cannot select the scheduler reserved for LPX-managed Pods. + if !component.ManagedByExternalController() && component.PodTemplate != nil && + component.PodTemplate.Spec.SchedulerName == consts.LPXSchedulerBackend { + allErrs = append(allErrs, field.Forbidden( + componentPath.Child("podTemplate", "spec", "schedulerName"), + "LPX schedulerName is controller-owned; declare an LPX component instead", + )) + } + gms := gpuMemoryServiceFor(component) if gms != nil && effectiveGMSMode(gms.Mode) == nvidiacomv1beta1.GMSModeInterPod { modePath := componentPath.Child("experimental", "gpuMemoryService", "mode") @@ -595,16 +604,16 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentExperimen grovePathway bool, grovePathwayRequirement string, ) field.ErrorList { - if experimental.KvTransferPolicy == nil { - return nil - } - return v.validateKvTransferPolicy( - experimental.KvTransferPolicy, - fldPath.Child("kvTransferPolicy"), - generation, - grovePathway, - grovePathwayRequirement, - ) + if experimental.KvTransferPolicy != nil { + return v.validateKvTransferPolicy( + experimental.KvTransferPolicy, + fldPath.Child("kvTransferPolicy"), + generation, + grovePathway, + grovePathwayRequirement, + ) + } + return nil } // validateKvTransferPolicy validates policy. policy and fldPath must not be nil. @@ -669,7 +678,6 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentUpdate( &oldDGD.Spec, field.NewPath("spec"), )...) - if oldDGD.Status.RollingUpdate != nil { phase := oldDGD.Status.RollingUpdate.Phase if phase == nvidiacomv1beta1.RollingUpdatePhasePending || phase == nvidiacomv1beta1.RollingUpdatePhaseInProgress { @@ -832,18 +840,12 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpecUpdat )) } - if newSpec.Experimental != nil { + if newSpec.Experimental != nil || oldSpec.Experimental != nil { allErrs = append(allErrs, v.validateDynamoGraphDeploymentExperimentalSpecUpdate( newSpec.Experimental, oldSpec.Experimental, fldPath.Child("experimental"), )...) - } else if oldPolicy := kvTransferPolicyFor(oldSpec.Experimental); oldPolicy != nil { - allErrs = append(allErrs, field.Invalid( - fldPath.Child("experimental", "kvTransferPolicy"), - newSpec.Experimental, - "is immutable and cannot be added, removed, or changed after creation; delete and recreate the DynamoGraphDeployment to change the KV transfer policy", - )) } return allErrs @@ -908,14 +910,14 @@ func (v *dynamoGraphDeploymentValidation) validateSpecTopologyConstraintUpdate( )} } -// validateDynamoGraphDeploymentExperimentalSpecUpdate validates an experimental spec update. -// newExperimental and fldPath must not be nil; oldExperimental may be nil for an addition. +// validateDynamoGraphDeploymentExperimentalSpecUpdate validates an experimental +// spec update. fldPath must not be nil; either experimental value may be nil. func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentExperimentalSpecUpdate( newExperimental *nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec, oldExperimental *nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec, fldPath *field.Path, ) field.ErrorList { - newPolicy := newExperimental.KvTransferPolicy + newPolicy := kvTransferPolicyFor(newExperimental) oldPolicy := kvTransferPolicyFor(oldExperimental) if newPolicy != nil { return v.validateKvTransferPolicyUpdate(newPolicy, oldPolicy, fldPath.Child("kvTransferPolicy")) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_handler_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_handler_test.go index e6b12b40e1c3..186aad905861 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_handler_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_handler_test.go @@ -36,7 +36,10 @@ import ( ) func TestDynamoGraphDeploymentHandlerValidateCreate(t *testing.T) { - handler := NewDynamoGraphDeploymentHandler(newGroveTopologyTestManager(t), "system:serviceaccount:dynamo:dynamo-operator") + handler := NewDynamoGraphDeploymentHandler( + newGroveTopologyTestManager(t), + "system:serviceaccount:dynamo:dynamo-operator", + ) dgd := newBetaDGDForValidation() warnings, err := handler.ValidateCreate(dgdAdmissionContext(admissionv1.Create, nvidiacomv1beta1.DynamoGraphDeploymentGVK), dgd) @@ -63,7 +66,10 @@ func TestDynamoGraphDeploymentHandlerValidateCreate(t *testing.T) { } func TestDynamoGraphDeploymentHandlerValidateUpdate(t *testing.T) { - handler := NewDynamoGraphDeploymentHandler(newGroveTopologyTestManager(t), "system:serviceaccount:dynamo:dynamo-operator") + handler := NewDynamoGraphDeploymentHandler( + newGroveTopologyTestManager(t), + "system:serviceaccount:dynamo:dynamo-operator", + ) ctx := dgdAdmissionContext(admissionv1.Update, nvidiacomv1beta1.DynamoGraphDeploymentGVK) t.Run("valid", func(t *testing.T) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go index c6acf512ffa2..7e6c1317b886 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go @@ -28,7 +28,6 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -38,11 +37,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -const ( - // maxCombinedResourceNameLength is kept as a local alias for readability. - maxCombinedResourceNameLength = consts.MaxCombinedGroveResourceNameLength -) - type clusterTopologyInfo struct { name string domainIndex map[string]int @@ -186,24 +180,6 @@ func grovePathwayForDynamoGraphDeployment( return true, "" } -func dgdComponentResourceNameLength( - dgdName string, - components []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, - component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, -) (int, string) { - pcsName := dynamo.PCSNameForDGD(dgdName, components) - componentName := component.ComponentName - combinedLength := len(pcsName) + len(strings.ToLower(componentName)) - detail := "PCS name + component name" - - if component.UsesPCSG() { - longestPodCliqueName := dynamo.LongestPodCliqueNameForDGDComponent(componentName, component) - combinedLength += len(longestPodCliqueName) - detail = fmt.Sprintf("PCS name + PCSG name + longest PodClique name %q", longestPodCliqueName) - } - return combinedLength, detail -} - func hasIntraPodFailover(spec *nvidiacomv1beta1.DynamoGraphDeploymentSpec) bool { for i := range spec.Components { failover := failoverFor(&spec.Components[i]) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go new file mode 100644 index 000000000000..22c9da15eeff --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -0,0 +1,473 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation_test + +import ( + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + k8sptr "k8s.io/utils/ptr" +) + +// lpxDGDAdmissionCases builds fresh LPX scenarios for the single native DGD admission table. +func lpxDGDAdmissionCases() []dgdAdmissionTestCase { + const longLPXComponentName = "abcdefghijklmnopqrstuvwxyzabcd" + + // Keep LPX inputs and oracles together without a separate admission execution path. + return []dgdAdmissionTestCase{ + { + name: "valid singleton lpx deployment with implicit conductor", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles = component.Roles[:1] + component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} + component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, + }}} + component.LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"stop_tokens":[1],"custom_runtime_knob":{"enabled":true}}`)} + component.Roles[0].PodTemplate.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelHostname, Operator: corev1.NodeSelectorOpIn, Values: []string{"lpu-node-a"}, + }}}}, + }, + }} + }), + }, + { + name: "valid canonical v1alpha1 lpx deployment", + deployment: alphaLPXDGDForAdmission(nil), + }, + { + name: "LPX admits nine hybrid replicas with lowered conductor volumes", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Replicas = k8sptr.To(int32(9)) + component.Roles[1].PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + // LPX lowering supplies these volume definitions after admission. + Containers: []corev1.Container{{Name: "main", Image: "gpu-runtime", VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/config"}, + {Name: "infiniband", MountPath: "/dev/infiniband"}, + }}}, + }} + }), + }, + { + name: "LPX native template schema rejects duplicate main container names", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles[1].PodTemplate = component.Roles[0].PodTemplate.DeepCopy() + component.Roles[1].PodTemplate.Spec.Containers = []corev1.Container{ + {Name: "main", Image: "runtime"}, + {Name: "main", Image: "runtime"}, + } + }), + wantSchemaErr: `spec.components[0].roles[1].podTemplate.spec.containers[1]: Duplicate value: map[string]interface {}{"name":"main"}`, + }, + { + name: "LPX replica bounds are enforced by the schema", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Replicas = k8sptr.To(int32(10)) + }), + wantCELErr: "spec.components[0]: Invalid value: replicas must be between 1 and 9 when type is lpx", + }, + { + name: "LPX autoscaling is rejected by the schema", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].ScalingAdapter = &nvidiacomv1beta1.ScalingAdapter{} + }), + wantCELErr: "spec.components[0]: Invalid value: scalingAdapter is not supported when type is lpx", + }, + { + name: "LPX settings must be an object", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`[]`)} + }), + wantSchemaErr: `spec.components[0].lpx.settings: Invalid value: spec.components[0].lpx.settings in body must be of type object: "array"`, + }, + { + name: "LPX integration availability is deferred to the child controller", + lpxDisabled: true, + deployment: betaLPXDGDForAdmission(nil), + }, + { + name: "LPX rejects the component pathway selected when Grove is disabled", + groveDisabled: true, + deployment: betaLPXDGDForAdmission(nil), + wantWebhookErrs: []string{`spec.components: Forbidden: requires the Grove pathway, but workload provider "component" is selected`}, + }, + { + name: "LPX topology and checkpoint support is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ + ClusterTopologyName: "grove-topology", PackDomain: "rack", + } + component := &dgd.Spec.Components[0] + component.TopologyConstraint = &nvidiacomv1beta1.TopologyConstraint{PackDomain: "rack"} + component.Experimental = &nvidiacomv1beta1.ExperimentalSpec{ + Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true}, + } + }), + }, + { + name: "LPX admits an ordinary component checkpoint", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := *betaWorkerComponent(betaDGDForAdmission(nil)) + worker.Experimental = &nvidiacomv1beta1.ExperimentalSpec{ + Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true}, + } + dgd.Spec.Components = append(dgd.Spec.Components, worker) + }), + }, + { + name: "LPX shell allocation target validation is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + container := &dgd.Spec.Components[0].Roles[0].PodTemplate.Spec.Containers[0] + container.Command = []string{"/usr/bin/env"} + container.Args = []string{"sh", "-c", "exec /opt/dynamo-lpu serve"} + }), + }, + { + name: "LPX runtime command updates are admitted while the integration is unavailable", + lpxDisabled: true, + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + container := &dgd.Spec.Components[0].Roles[0].PodTemplate.Spec.Containers[0] + container.Command = []string{"sh", "-c"} + container.Args = []string{"exec /opt/dynamo-lpu serve"} + }), + }, + { + name: "LPX rejects an unknown Pod role", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[1].Name = "unknown" + }), + wantWebhookErrs: []string{`spec.components[0].roles[1].name: Unsupported value: "unknown": supported values: "leader", "worker"`}, + }, + { + name: "v1alpha1 LPX image errors aggregate at the authored role indices", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + component := dgd.Spec.Services["lpx"] + component.Roles[0], component.Roles[1] = component.Roles[1], component.Roles[0] + spec := &component.Roles[1].PodTemplate.Spec + spec.Containers = append(spec.Containers, corev1.Container{Name: "sidecar"}) + spec.InitContainers = []corev1.Container{{Name: "init"}} + }), + wantWebhookErrs: []string{ + "spec.components[0].roles[1].podTemplate.spec.containers[1].image: Required value: must specify a non-empty image", + "spec.components[0].roles[1].podTemplate.spec.initContainers[0].image: Required value: must specify a non-empty image", + }, + }, + { + name: "LPX rejects duplicate Pod roles", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles[1] = *component.Roles[0].DeepCopy() + }), + wantSchemaErr: `spec.components[0].roles[1]: Duplicate value: map[string]interface {}{"name":"worker"}`, + }, + { + name: "physical LPU rejects Grove opt out", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Annotations = map[string]string{consts.KubeAnnotationEnableGrove: consts.KubeLabelValueFalse} + }), + wantWebhookErrs: []string{`spec.components: Forbidden: requires the Grove pathway, but workload provider "component" is selected`}, + }, + { + name: "v1alpha1 SpecDecode draft replicas must be positive", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, k8sptr.To(int32(0))) + }), + wantCELErr: "spec.services[draft]: Invalid value: replicas must be between 1 and 9 when componentType is lpx", + }, + // Engine-local template and build validation. + { + name: "LPX rejects a component-level PodTemplate", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].PodTemplate = dgd.Spec.Components[0].Roles[0].PodTemplate.DeepCopy() + }), + wantCELErr: "spec.components[0]: Invalid value: LPX Pod templates belong to roles", + }, + { + name: "LPX rejects a worker without a Pod template", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[0].PodTemplate = nil + }), + wantWebhookErrs: []string{"spec.components[0].roles[0].podTemplate: Required value: the LPX worker role requires a podTemplate"}, + }, + { + name: "LPX requires a build for an LPU engine", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].LPX.BuildID = "" + }), + wantSchemaErr: "spec.components[0].lpx.buildId: Invalid value: \"\": spec.components[0].lpx.buildId in body should be at least 1 chars long", + }, + { + name: "v1alpha1 LPX root template remains rejected when runtime validation is deferred", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + component := dgd.Spec.Services["lpx"] + component.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "runtime"}} + component.LPX.BuildID = " " + }), + wantWebhookErrs: []string{ + "spec.components[0].podTemplate: Forbidden: LPX Pod templates belong to roles", + }, + }, + { + name: "v1alpha1 LPX whitespace build ID validation is deferred to the child controller", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].LPX.BuildID = " " + }), + }, + // Selected LPX workload shapes. + { + name: "LPX roles reject provider overrides", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[0].ProviderOverride = groveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) + }), + wantWebhookErrs: []string{"spec.components[0].roles[0].providerOverride: Forbidden: LPX roles do not support provider overrides"}, + }, + { + name: "LPX can remove an explicit leader and retain its implicit fallback", + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles = dgd.Spec.Components[0].Roles[:1] + }), + }, + { + name: "LPX provider override support on update is deferred to the child controller", + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.ProviderOverride = groveProviderOverride("", `{"spec":{"template":{"topologyConstraint":{"pack":{"required":"rack"}}}}}`) + dgd.Spec.Components[0].ProviderOverride = groveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) + }), + }, + { + name: "v1alpha1 LPX provider override support is deferred after conversion", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.ProviderOverride = alphaGroveProviderOverride("", `{"spec":{"template":{"topologyConstraint":{"pack":{"required":"rack"}}}}}`) + dgd.Spec.Services["lpx"].ProviderOverride = alphaGroveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) + }), + }, + { + name: "LPX admits an ordinary component provider override", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + frontend := *betaDGDForAdmission(nil).GetComponentByName("frontend") + frontend.ProviderOverride = groveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) + dgd.Spec.Components = append(dgd.Spec.Components, frontend) + }), + }, + { + name: "selected v1beta1 LPX derives SpecDecode from two models", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Annotations = map[string]string{ + consts.KubeAnnotationLPXSchedulerBackend: "ignored-scheduler", + consts.KubeAnnotationLPXExecutionBackend: "ignored-execution", + } + dgd.Spec.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(30))} + component := &dgd.Spec.Components[0] + component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} + component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, + }}} + setBetaLPXSpecDec(dgd, k8sptr.To(int32(2))) + dgd.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"draft_runtime":{"enabled":true}}`)} + dgd.Spec.Components[1].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`null`)} + }), + }, + { + name: "selected v1alpha1 LPX derives SpecDecode from two models", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(30))} + setAlphaLPXSpecDec(dgd, nil) + }), + }, + { + name: "LPX Agent placement validation is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, nil) + spec := &dgd.Spec.Components[0].Roles[0].PodTemplate.Spec + spec.SchedulerName = "custom" + spec.Hostname = "custom-host" + spec.Subdomain = "custom-subdomain" + spec.NodeName = "lpu-node-a" + spec.NodeSelector = map[string]string{corev1.LabelHostname: "lpu-node-a"} + spec.Affinity = &corev1.Affinity{PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + TopologyKey: corev1.LabelHostname, + }}, + }} + spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, TopologyKey: corev1.LabelHostname, WhenUnsatisfiable: corev1.DoNotSchedule, + }} + spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: "custom"}} + spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "device", ResourceClaimName: k8sptr.To("device-claim")}} + }), + }, + { + name: "LPX SpecDecode draft replica support is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(9))) + }), + }, + { + name: "lpx requires a worker role", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles = nil + }), + wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "worker" role`}, + }, + { + name: "v1alpha1 lpx requires a worker role", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Roles = nil + }), + wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "worker" role`}, + }, + { + name: "v1alpha1 lpx requires lpx", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].LPX = nil + }), + wantCELErr: "spec.services[lpx]: Invalid value: lpx is required when componentType is lpx", + }, + { + name: "v1alpha1 non-lpx service rejects lpx", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].ComponentType = consts.ComponentTypeWorker + dgd.Spec.Services["lpx"].Roles = nil + }), + wantCELErr: "spec.services[lpx]: Invalid value: lpx may only be set when componentType is lpx", + }, + { + name: "LPX shared leader composition is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + second := dgd.Spec.Components[0] + second.ComponentName = "lpx-2" + dgd.Spec.Components = append(dgd.Spec.Components, second) + }), + }, + { + name: "shared LPX draft does not consume the target Grove name budget", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(2))) + dgd.Spec.Components[0].ComponentName = longLPXComponentName + }), + }, + { + name: "LPX PCSG name overflow validation is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].ComponentName = longLPXComponentName + }), + }, + { + name: "implicit LPX conductor name overflow validation is deferred to the child controller", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.ComponentName = longLPXComponentName + component.Roles = component.Roles[:1] + }), + }, + // Ordinary components have no LPX child to reject controller-owned scheduler selection. + { + name: "ordinary graph rejects manual LPX scheduler selection", + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + betaWorkerComponent(dgd).PodTemplate.Spec.SchedulerName = consts.LPXSchedulerBackend + }), + wantWebhookErrs: []string{"spec.components[1].podTemplate.spec.schedulerName: Forbidden: LPX schedulerName is controller-owned; declare an LPX component instead"}, + }, + { + name: "mixed graph rejects manual LPX scheduler selection on an ordinary component", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := *betaWorkerComponent(betaDGDForAdmission(nil)) + worker.PodTemplate.Spec.SchedulerName = consts.LPXSchedulerBackend + dgd.Spec.Components = append(dgd.Spec.Components, worker) + }), + wantWebhookErrs: []string{"spec.components[1].podTemplate.spec.schedulerName: Forbidden: LPX schedulerName is controller-owned; declare an LPX component instead"}, + }, + // An omitted type passes CEL; the LPX topology guard must reject the update. + { + name: "setting a previously unset component type to LPX is rejected by the webhook", + oldDeployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components = dgd.Spec.Components[:1] + dgd.Spec.Components[0].ComponentName = "lpx" + dgd.Spec.Components[0].ComponentType = "" + }), + deployment: betaLPXDGDForAdmission(nil), + wantWebhookErrs: []string{`spec.components[0].type: Invalid value: "lpx": cannot change node topology between LPX and non-LPX after creation`}, + }, + } +} + +func betaLPXDGDForAdmission( + mutate func(*nvidiacomv1beta1.DynamoGraphDeployment), +) *nvidiacomv1beta1.DynamoGraphDeployment { + dgd := betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "lpx", + ComponentType: nvidiacomv1beta1.ComponentTypeLPX, + Replicas: k8sptr.To(int32(1)), + LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"}, + Roles: []nvidiacomv1beta1.ComponentRoleSpec{ + {Name: nvidiacomv1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, + }}}, + {Name: nvidiacomv1beta1.ComponentRoleLeader}, + }, + }, + } + }) + if mutate != nil { + mutate(dgd) + } + return dgd +} + +func alphaLPXDGDForAdmission( + mutate func(*nvidiacomv1alpha1.DynamoGraphDeployment), +) *nvidiacomv1alpha1.DynamoGraphDeployment { + dgd := alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services = map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ + "lpx": { + ComponentType: consts.ComponentTypeLPX, + Replicas: k8sptr.To(int32(1)), + LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"}, + Roles: []nvidiacomv1alpha1.ComponentRoleSpec{ + {Name: nvidiacomv1alpha1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, + }}}, + {Name: nvidiacomv1alpha1.ComponentRoleLeader}, + }, + }, + } + }) + if mutate != nil { + mutate(dgd) + } + return dgd +} + +// setBetaLPXSpecDec splits the fixture into independently authored model components. +func setBetaLPXSpecDec(dgd *nvidiacomv1beta1.DynamoGraphDeployment, draftReplicas *int32) { + draft := dgd.Spec.Components[0].DeepCopy() + target := dgd.Spec.Components[0].DeepCopy() + draft.ComponentName, target.ComponentName = "draft", "target" + draft.LPX.BuildID, target.LPX.BuildID = "test/draft", "test/target" + draft.Roles = draft.Roles[:1] + draft.Replicas = draftReplicas + dgd.Spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{*draft, *target} +} + +// setAlphaLPXSpecDec exercises the equivalent native alpha service representation. +func setAlphaLPXSpecDec(dgd *nvidiacomv1alpha1.DynamoGraphDeployment, draftReplicas *int32) { + draft := dgd.Spec.Services["lpx"].DeepCopy() + target := dgd.Spec.Services["lpx"].DeepCopy() + draft.LPX.BuildID, target.LPX.BuildID = "test/draft", "test/target" + draft.Roles = draft.Roles[:1] + draft.Replicas = draftReplicas + dgd.Spec.Services = map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{"draft": draft, "target": target} +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index 370643c754b2..f746d575a7c4 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -46,34 +46,37 @@ const ( const sglangBackendFramework = "sglang" +// dgdAdmissionTestCase keeps the native admission and post-admission expectations together. +type dgdAdmissionTestCase struct { + name string + deployment runtime.Object + oldDeployment runtime.Object + oldBeforeUpdate runtime.Object + mutateRequest func(*testing.T, map[string]any) // mutates the source-version request map + groveDisabled bool // disables the configured Grove pathway + lpxDisabled bool // disables the LPX integration + checkpointOff bool // disables checkpoint creation and restore + seedWithoutWebhook bool // seeds oldDeployment without validating it + terminating bool // deletes the seeded object so the update runs while it terminates + username string // supplies the admission request identity + + wantSchemaErr string + wantCELErr string + wantAdmissionErrs []string + wantWebhookErrs []string + wantWarnings []string + notWantErr string + wantPodAnnotations map[string]string + wantProvider string + wantRoleReplicas map[string]int32 +} + func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { longDGDName := "test-graph-" + strings.Repeat("x", 50) boundaryComponentName := "w" + strings.Repeat("x", 36) tooLongComponentName := boundaryComponentName + "x" - tests := []struct { - name string - deployment runtime.Object - oldDeployment runtime.Object - oldBeforeUpdate runtime.Object - mutateRequest func(*testing.T, map[string]any) // mutates the source-version request map - withoutTopology bool // omits the default cluster topology fixture - groveDisabled bool // disables the configured Grove pathway - checkpointOff bool // disables checkpoint creation and restore - seedWithoutWebhook bool // seeds oldDeployment without validating it - terminating bool // deletes the seeded object so the update runs while it terminates - username string // supplies the admission request identity - - wantSchemaErr string - wantCELErr string - wantAdmissionErrs []string - wantWebhookErrs []string - wantWarnings []string - notWantErr string - wantPodAnnotations map[string]string - wantProvider string - wantRoleReplicas map[string]int32 - }{ + tests := []dgdAdmissionTestCase{ // Baseline create-path rules. { name: "valid deployment with components", @@ -1099,7 +1102,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { {Name: nvidiacomv1beta1.ComponentRoleLeader}, } }), - wantWebhookErrs: []string{"spec.components[1].roles: Forbidden: roles are supported only for component shapes that define a role schema; this release supports multinode components"}, + wantWebhookErrs: []string{"spec.components[1].roles: Forbidden: roles are supported only for component shapes that define a role schema; this release supports multinode and LPX components"}, }, { name: "explicit multinode roles require the complete role set", @@ -1362,8 +1365,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { }), }, { - name: "KV-transfer cluster topology policy rejects missing topology", - withoutTopology: true, + name: "KV-transfer cluster topology policy rejects missing topology", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Experimental = &nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec{ KvTransferPolicy: &nvidiacomv1beta1.KvTransferPolicy{ClusterTopologyName: "missing-topology", Domain: "rack"}, @@ -1872,9 +1874,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { betaWorkerComponent(dgd).ComponentName = tooLongComponentName }), wantWebhookErrs: []string{fmt.Sprintf( - "spec.components[1].name: Invalid value: %q: combined resource name length 46 exceeds the 45-character pod-name limit (PCS name + component name); shorten DynamoGraphDeployment name %q or component name %q", - tooLongComponentName, - longDGDName, + "spec.components[1].name: Invalid value: %q: combined Grove resource name length 46 exceeds the 45-character limit; shorten the deployment or component name", tooLongComponentName, )}, }, @@ -1953,8 +1953,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { }), }, { - name: "missing cluster topology is rejected", - withoutTopology: true, + name: "missing cluster topology is rejected", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ ClusterTopologyName: "missing-topology", @@ -1964,8 +1963,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { wantWebhookErrs: []string{`spec.topologyConstraint.clusterTopologyName: Invalid value: "missing-topology": references a ClusterTopologyBinding resource that was not found`}, }, { - name: "independent topology errors aggregate", - withoutTopology: true, + name: "independent topology errors aggregate", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ClusterTopologyName: "missing-topology"} }), @@ -3028,17 +3026,22 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { }, } + t.Log("Add the LPX scenarios to the same DGD admission table") + tests = append(tests, lpxDGDAdmissionCases()...) + + t.Log("Own one topology fixture for all DGD admission scenarios") + createTestClusterTopology(t, admissionEnv.ForTest(t)) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Log("Assemble the admission scenario from the table entry") - gates := features.Gates{Checkpoint: !tt.checkpointOff, Grove: !tt.groveDisabled} + gates := features.Gates{Checkpoint: !tt.checkpointOff, Grove: !tt.groveDisabled, LPX: !tt.lpxDisabled} test := admissionTestCase{ object: tt.deployment, oldObject: tt.oldDeployment, oldBeforeUpdate: tt.oldBeforeUpdate, mutateObject: tt.mutateRequest, gates: gates, - withoutTopology: tt.withoutTopology, seedWithoutWebhook: tt.seedWithoutWebhook, terminating: tt.terminating, username: tt.username, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 35242b44d500..f8906c6e7b76 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -485,7 +485,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { oldObject: tt.oldRequest, gates: gates, seedWithoutWebhook: tt.seedWithoutWebhook, - withoutTopology: true, wantSchemaError: tt.wantSchemaErr, wantCELError: tt.wantCELErr, wantWebhookErrors: tt.wantWebhook, diff --git a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go index b735b3248805..fa4368a194ff 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go @@ -79,7 +79,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecV1alpha1( } // Validate runtime compatibility against the source-version fields. - if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) && !spec.IsLPX() { image, imagePath := runtimeVersionImageAndPathV1Alpha1(spec, fldPath) if err := eppRuntimeCompatibilityError( eppRuntimeContractV1Alpha1(spec, image), @@ -111,7 +111,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdateV1al allErrs := field.ErrorList{} // Ratchet only complete, unchanged source-version runtime contract violations. - if v.hasRuntimeVersionSource(runtimeVersionSourceV1Alpha1) { + if v.hasRuntimeVersionSource(runtimeVersionSourceV1Alpha1) && !newSpec.IsLPX() { newImage, imagePath := runtimeVersionImageAndPathV1Alpha1(newSpec, fldPath) oldImage, _ := runtimeVersionImageAndPathV1Alpha1(oldSpec, fldPath) overrideChanged := newSpec.RuntimeVersionOverride != oldSpec.RuntimeVersionOverride diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index 2a5f822d5b93..5dd1c1fbdbab 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -39,6 +39,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) +// validateLPXPodTemplateSpec checks image and annotation semantics beyond the Pod schema. +// LPX intent validation additionally checks main-container and placement rules. +// template and fldPath are non-nil. +func (v *sharedValidation) validateLPXPodTemplateSpec(template *corev1.PodTemplateSpec, fldPath *field.Path) field.ErrorList { + allErrs := apivalidation.ValidateAnnotations(template.Annotations, fldPath.Child("metadata", "annotations")) + for _, group := range []struct { + name string + containers []corev1.Container + }{ + {"containers", template.Spec.Containers}, + {"initContainers", template.Spec.InitContainers}, + } { + for index, container := range group.containers { + if strings.TrimSpace(container.Image) == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("spec", group.name).Index(index).Child("image"), "must specify a non-empty image")) + } + } + } + return allErrs +} + // sharedValidation carries request-wide dependencies and accumulation used by // validation for API types shared by multiple resources. type sharedValidation struct { @@ -68,6 +89,8 @@ type dynamoComponentDeploymentSharedSpecValidationOptions struct { // validateDynamoComponentDeploymentSharedSpec validates spec. spec and fldPath must not be nil. // Options are supplied by the owning resource. +// +//nolint:gocyclo // Shared validation reports the combined upstream and LPX contracts in one pass. func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( spec *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, fldPath *field.Path, @@ -89,6 +112,11 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( )...) } + // Preserve the LPX role-template boundary after conversion from the alpha schema. + if spec.IsLPX() && spec.PodTemplate != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("podTemplate"), "LPX Pod templates belong to roles")) + } + // Enforce Grove-only availability semantics before validating later fields. if spec.MinAvailable != nil && !options.grovePathway { allErrs = append(allErrs, field.Forbidden( @@ -98,7 +126,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( } // Validate the complete role schema against the enclosing component shape. - if spec.Roles != nil { + if spec.Roles != nil || spec.IsLPX() { allErrs = append(allErrs, v.validateComponentRoles( spec, fldPath.Child("roles"), @@ -170,7 +198,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( } // Validate runtime compatibility against the source-version fields. - if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Beta1) { + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Beta1) && !spec.IsLPX() { image, imagePath := runtimeVersionImageAndPath(spec, fldPath) if err := eppRuntimeCompatibilityError( eppRuntimeContractV1Beta1(spec, image), @@ -341,14 +369,21 @@ func (v *sharedValidation) validateComponentRoles( workloadProvider string, ) field.ErrorList { allErrs := field.ErrorList{} - if component.Multinode == nil { + if !component.IsLPX() && component.Multinode == nil { return field.ErrorList{field.Forbidden( fldPath, - "roles are supported only for component shapes that define a role schema; this release supports multinode components", + "roles are supported only for component shapes that define a role schema; this release supports multinode and LPX components", )} } - // Multinode explicit mode is a closed schema: exactly one leader and worker. + // Each component shape defines its supported roles and required subset. + supportedRoles := []string{nvidiacomv1beta1.ComponentRoleLeader, nvidiacomv1beta1.ComponentRoleWorker} + requiredRoles := supportedRoles + if component.IsLPX() { + requiredRoles = []string{nvidiacomv1beta1.ComponentRoleWorker} + } + + // Validate each authored role once, preserving its index in every error path. seen := make(map[string]struct{}, len(component.Roles)) for i := range component.Roles { role := &component.Roles[i] @@ -358,7 +393,7 @@ func (v *sharedValidation) validateComponentRoles( allErrs = append(allErrs, field.NotSupported( rolePath.Child("name"), role.Name, - []string{nvidiacomv1beta1.ComponentRoleLeader, nvidiacomv1beta1.ComponentRoleWorker}, + supportedRoles, )) } else if _, exists := seen[role.Name]; exists { allErrs = append(allErrs, field.Duplicate(rolePath.Child("name"), role.Name)) @@ -366,7 +401,7 @@ func (v *sharedValidation) validateComponentRoles( seen[role.Name] = struct{}{} } - if role.Replicas != nil && knownRole { + if !component.IsLPX() && role.Replicas != nil && knownRole { expected := int32(1) if role.Name == nvidiacomv1beta1.ComponentRoleWorker { expected = component.Multinode.NodeCount - 1 @@ -380,7 +415,7 @@ func (v *sharedValidation) validateComponentRoles( } } - if knownRole { + if knownRole || component.IsLPX() { allErrs = append(allErrs, v.validateComponentRoleSpec( role, rolePath, @@ -389,15 +424,13 @@ func (v *sharedValidation) validateComponentRoles( workloadProvider: workloadProvider, scope: scope, component: component, + podTemplateAllowed: component.IsLPX(), }, )...) } } - for _, requiredRole := range []string{ - nvidiacomv1beta1.ComponentRoleLeader, - nvidiacomv1beta1.ComponentRoleWorker, - } { + for _, requiredRole := range requiredRoles { if _, exists := seen[requiredRole]; !exists { allErrs = append(allErrs, field.Required( fldPath, @@ -430,6 +463,20 @@ func (v *sharedValidation) validateComponentRoleSpec( "is not supported for this component role", )) } + + // LPX renders role templates directly and has no provider-override lowering. + if options.component != nil && options.component.IsLPX() { + if role.PodTemplate != nil { + allErrs = append(allErrs, v.validateLPXPodTemplateSpec(role.PodTemplate, fldPath.Child("podTemplate"))...) + } else if role.Name == nvidiacomv1beta1.ComponentRoleWorker { + allErrs = append(allErrs, field.Required(fldPath.Child("podTemplate"), "the LPX worker role requires a podTemplate")) + } + if role.ProviderOverride != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("providerOverride"), "LPX roles do not support provider overrides")) + } + return allErrs + } + if role.ProviderOverride == nil { return allErrs } @@ -692,7 +739,9 @@ func (v *sharedValidation) validateComponentCheckpointConfig( fldPath, "checkpoint functionality requires component type to be explicitly set to worker, prefill, or decode", )) - } else if checkpointConfig.Enabled && !dynamo.IsWorkerComponent(string(componentType)) { + } else if checkpointConfig.Enabled && + componentType != nvidiacomv1beta1.ComponentTypeLPX && + !dynamo.IsWorkerComponent(string(componentType)) { allErrs = append(allErrs, field.Forbidden( fldPath, "checkpoint functionality is supported only for worker, prefill, and decode components", @@ -760,7 +809,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( "cannot change node topology between single-node and multi-node after creation", )) } - } else { + } else if !newComponent.IsLPX() && !oldComponent.IsLPX() { if !hasUnsupportedMultinode(newComponent) && newComponent.Multinode != nil && oldComponent.Multinode != nil && newComponent.Multinode.NodeCount != oldComponent.Multinode.NodeCount { @@ -777,6 +826,14 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( )...) } + if newComponent.IsLPX() != oldComponent.IsLPX() { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("type"), + newComponent.ComponentType, + "cannot change node topology between LPX and non-LPX after creation", + )) + } + // Protect replica ownership when a scaling adapter is present in either state. if (newComponent.ScalingAdapter != nil || oldComponent.ScalingAdapter != nil) && !canModifyReplicas && k8sptr.Deref(newComponent.Replicas, int32(1)) != k8sptr.Deref(oldComponent.Replicas, int32(1)) { @@ -842,7 +899,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( } // Ratchet only complete, unchanged source-version runtime contract violations. - if v.hasRuntimeVersionSource(runtimeVersionSourceV1Beta1) { + if v.hasRuntimeVersionSource(runtimeVersionSourceV1Beta1) && !newComponent.IsLPX() { newImage, imagePath := runtimeVersionImageAndPath(newComponent, fldPath) oldImage, _ := runtimeVersionImageAndPath(oldComponent, fldPath) overrideChanged := newComponent.RuntimeVersionOverride != oldComponent.RuntimeVersionOverride diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 0dc783605582..6a86ed026f9c 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "testing" + "time" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -23,13 +24,16 @@ import ( webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -57,6 +61,9 @@ func alphaGroveProviderOverride(target, value string) *nvidiacomv1alpha1.Provide } var ( + // Fixture barriers observe the same cache used by webhook validation. + admissionReader client.Reader + // Admission cases must remain sequential because they share this gate and a cluster-scoped topology fixture. admissionGate = &mutableFeatureGate{} admissionEnv = operatorenv.New(operatorenv.Options{ @@ -76,6 +83,7 @@ func TestMain(m *testing.M) { } func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptions) error { + admissionReader = mgr.GetClient() return webhooksetup.Setup(mgr, webhooksetup.Options{ Config: opts.OperatorConfig, RuntimeConfig: opts.RuntimeConfig, @@ -87,7 +95,7 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio type mutableFeatureGate struct { mu sync.RWMutex - gates features.Gates + gates features.Gate } func (g *mutableFeatureGate) Enabled(name features.Name) bool { @@ -96,7 +104,7 @@ func (g *mutableFeatureGate) Enabled(name features.Name) bool { return g.gates.Enabled(name) } -func (g *mutableFeatureGate) set(gates features.Gates) { +func (g *mutableFeatureGate) set(gates features.Gate) { g.mu.Lock() defer g.mu.Unlock() g.gates = gates @@ -107,10 +115,9 @@ type admissionTestCase struct { oldObject runtime.Object oldBeforeUpdate runtime.Object mutateObject func(*testing.T, map[string]any) - gates features.Gates + gates features.Gate seedGates *features.Gates seedWithoutWebhook bool - withoutTopology bool terminating bool username string @@ -130,11 +137,6 @@ func runAdmissionTest(t *testing.T, test admissionTestCase) *unstructured.Unstru warnings := &warningRecorder{} resourceClient := newAdmissionResourceClient(t, env, test.object, test.username, warnings) - if !test.withoutTopology { - t.Log("Install the test-owned cluster topology used by DGD validation") - createTestClusterTopology(t, env) - } - admissionGate.set(test.gates) current, originalNamespace := admissionObject(t, test.object, env.Namespace(), test.mutateObject) @@ -455,15 +457,9 @@ func mustUnstructured(t *testing.T, object runtime.Object) map[string]any { func createTestClusterTopology(t *testing.T, env *operatorenv.TestEnv) { t.Helper() - client, err := dynamic.NewForConfig(env.RESTConfig()) - if err != nil { - t.Fatalf("create topology client: %v", err) - } - resource := client.Resource(schema.GroupVersionResource{ - Group: "grove.io", Version: "v1alpha1", Resource: "clustertopologybindings", - }) + + t.Log("Create the DGD table's immutable cluster topology") topology := &grovev1alpha1.ClusterTopologyBinding{ - TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "ClusterTopologyBinding"}, ObjectMeta: metav1.ObjectMeta{Name: "grove-topology"}, Spec: grovev1alpha1.ClusterTopologyBindingSpec{ Levels: []grovev1alpha1.TopologyLevel{ @@ -472,18 +468,36 @@ func createTestClusterTopology(t *testing.T, env *operatorenv.TestEnv) { }, }, } - value, err := runtime.DefaultUnstructuredConverter.ToUnstructured(topology) - if err != nil { - t.Fatalf("convert cluster topology: %v", err) - } - if _, err := resource.Create(t.Context(), &unstructured.Unstructured{Object: value}, metav1.CreateOptions{}); err != nil { + if err := env.Client().Create(t.Context(), topology); err != nil { t.Fatalf("create cluster topology: %v", err) } + key := client.ObjectKeyFromObject(topology) + + // Observe deletion before another test invocation can reuse this cluster-scoped name. t.Cleanup(func() { - if err := resource.Delete(context.Background(), topology.Name, metav1.DeleteOptions{}); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := env.Client().Delete(ctx, topology); err != nil { t.Errorf("delete cluster topology: %v", err) + return + } + if err := wait.PollUntilContextCancel(ctx, 10*time.Millisecond, true, func(ctx context.Context) (bool, error) { + err := admissionReader.Get(ctx, key, &grovev1alpha1.ClusterTopologyBinding{}) + return apierrors.IsNotFound(err), client.IgnoreNotFound(err) + }); err != nil { + t.Errorf("wait for deleted cluster topology to leave webhook cache: %v", err) } }) + + t.Log("Wait for the webhook cache to observe this exact topology incarnation") + if err := wait.PollUntilContextTimeout(t.Context(), 10*time.Millisecond, 10*time.Second, true, + func(ctx context.Context) (bool, error) { + observed := &grovev1alpha1.ClusterTopologyBinding{} + err := admissionReader.Get(ctx, key, observed) + return err == nil && observed.UID == topology.UID, client.IgnoreNotFound(err) + }); err != nil { + t.Fatalf("wait for created cluster topology in webhook cache: %v", err) + } } func expectedAdmissionErrors(t *testing.T, test admissionTestCase) []string { diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index 781be4e83917..fa91e5c3006f 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -257,6 +257,35 @@ vLLM and SGLang take tensor/pipeline/data parallelism as CLI flags (covered in t You normally do not need to set the top-level `spec.backendFramework` field — the operator infers the backend from the worker command. Set it explicitly (`vllm`, `sglang`, or `trtllm`) only when a feature needs the framework known up front, such as GMS failover or multinode TensorRT-LLM. +For LPX on LPU hardware, declare the build and runtime roles in a `type: lpx` component: + +```yaml +spec: + components: + - name: lpx + type: lpx + lpx: + buildId: my-model/build + roles: + - name: worker + podTemplate: + spec: + containers: + - name: main + image: + volumeMounts: + - name: model-storage + mountPath: /models + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage +``` + +Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with a build available through the configured LPX model registry and `` with its compatible runtime image. The `worker` role configures Agent Pods; the optional `leader` role configures the conductor. With one LPX component, the conductor reuses the worker template unless the leader supplies a separate `podTemplate`. For a hybrid engine, configure the hybrid runtime template on the leader role. + +For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only a `worker` role; the target has `leader` and `worker` roles. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. + For per-backend setup and tuning, see [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/overview.md), [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/overview.md), and [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/overview.md). > [!NOTE] diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index fcdc3c856cd6..019e5cb90661 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -152,7 +152,7 @@ _Appears in:_ | `name` _string_ | Name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | Replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
Multinode.NodeCount; leader must be 1 and worker must be
Multinode.NodeCount minus 1. | | Minimum: 1
Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | ProviderOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. No component type supports it in this release. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. LPX components support role-specific Pod templates. | | Optional: \{\}
| #### ConfigMapKeySelector @@ -275,7 +275,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `model` _string_ | Model is the model identifier (e.g., "meta-llama/Llama-3-70B").
Deprecated: legacy identity only. | | Required: \{\}
| -| `backendFramework` _string_ | BackendFramework is the runtime framework (vllm, sglang, trtllm).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| +| `backendFramework` _string_ | BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm lpu]
Required: \{\}
| | `dynamoVersion` _string_ | DynamoVersion is the Dynamo platform version.
Deprecated: legacy identity only. | | Optional: \{\}
| | `tensorParallelSize` _integer_ | TensorParallelSize is the tensor parallel configuration.
Deprecated: automatic capture derives compatibility from the worker. | 1 | Minimum: 1
Optional: \{\}
| | `pipelineParallelSize` _integer_ | PipelineParallelSize is the pipeline parallel configuration.
Deprecated: automatic capture derives compatibility from the worker. | 1 | Minimum: 1
Optional: \{\}
| @@ -333,15 +333,17 @@ _Appears in:_ | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| | `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| +| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the
operator-generated pod and container defaults. | | Enum: [override strategic]
Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| +| `lpx` _[LPXConfig](#lpxconfig)_ | LPX holds LPX integration configuration. Only meaningful when
ComponentType is "lpx". | | Optional: \{\}
| | `checkpoint` _[ServiceCheckpointConfig](#servicecheckpointconfig)_ | Checkpoint configures container checkpointing for this service.
When enabled, pods can be restored from a checkpoint files for faster cold start. | | Optional: \{\}
| | `topologyConstraint` _[TopologyConstraint](#topologyconstraint)_ | TopologyConstraint for this service. packDomain is required.
When both this and spec.topologyConstraint.packDomain are set, packDomain
must be narrower than or equal to the spec-level packDomain. | | Optional: \{\}
| | `gpuMemoryService` _[GPUMemoryServiceSpec](#gpumemoryservicespec)_ | GPUMemoryService configures the GPU Memory Service (GMS) sidecar.
When enabled, a GMS sidecar is injected and GPU access is managed via DRA. | | Optional: \{\}
| @@ -361,7 +363,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `backendFramework` _string_ | BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") | | Enum: [sglang vllm trtllm]
| +| `backendFramework` _string_ | BackendFramework specifies the GPU backend framework (for example,
"sglang", "vllm", or "trtllm"). | | Enum: [sglang vllm trtllm]
| | `annotations` _object (keys:string, values:string)_ | Annotations to add to generated Kubernetes resources for this component
(such as Pod, Service, and Ingress when applicable). | | | | `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | | | `serviceName` _string_ | The name of the component | | | @@ -379,12 +381,13 @@ _Appears in:_ | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| | `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| +| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the
operator-generated pod and container defaults. | | Enum: [override strategic]
Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -606,6 +609,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | ProviderOverride configures the root resource generated by the selected
graph-level workload provider. For Grove it uses apiVersion
`grove.io/v1alpha1`, target `PodCliqueSet`, and may set only
`spec.template.topologyConstraint`. It cannot select or change the provider. | | Optional: \{\}
| +| `scheduling` _[SchedulingSpec](#schedulingspec)_ | Scheduling configures attempts for LPX components. Omission means no deadline. | | Optional: \{\}
| | `annotations` _object (keys:string, values:string)_ | Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates).
Service-level annotations take precedence over these values. | | Optional: \{\}
| | `labels` _object (keys:string, values:string)_ | Labels to propagate to all child resources (PCS, DCD, Deployments, and pod templates).
Service-level labels take precedence over these values. | | Optional: \{\}
| | `priorityClassName` _string_ | PriorityClassName is the name of the PriorityClass to use for Grove PodCliqueSets.
Requires the Grove pathway. | | Optional: \{\}
| @@ -637,8 +641,8 @@ _Appears in:_ | `services` _object (keys:string, values:[ServiceReplicaStatus](#servicereplicastatus))_ | Services contains per-service replica status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| +| `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | LPX contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placement` _[PlacementStatus](#placementstatus)_ | Placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| #### DynamoModel @@ -774,6 +778,26 @@ _Appears in:_ | `mainContainer` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#container-v1-core)_ | | | | +#### ExtraPodSpecMergeStrategy + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [override strategic] + +_Appears in:_ +- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) +- [PodGenerationConfiguration](#podgenerationconfiguration) + +| Field | Description | +| --- | --- | +| `override` | | +| `strategic` | | + + #### FailoverSpec @@ -1031,47 +1055,6 @@ _Appears in:_ | `volumeAccessMode` _[PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#persistentvolumeaccessmode-v1-core)_ | VolumeAccessMode is the volume access mode of the PVC. Required when create is true. | | | -#### PlacementScoreState - -_Underlying type:_ _string_ - -PlacementScoreState describes whether placement score is available and how -complete the reported score is for a graph deployment. See the v1beta1 -PlacementScoreState for the authoritative semantics of each value. - -_Validation:_ -- Enum: [Reported Partial Unsupported Unknown] - -_Appears in:_ -- [PlacementStatus](#placementstatus) - -| Field | Description | -| --- | --- | -| `Reported` | | -| `Partial` | | -| `Unsupported` | | -| `Unknown` | | - - -#### PlacementStatus - - - -PlacementStatus groups DGD-level scheduler placement fields under a single -status object so future placement signals can be added without a schema -break. See the v1beta1 PlacementStatus for the authoritative field docs. - - - -_Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `score` _float_ | Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0]
where higher is better and 1.0 is the best possible placement. | | Maximum: 1
Minimum: 0
Optional: \{\}
| -| `state` _[PlacementScoreState](#placementscorestate)_ | State indicates placement score reporting state. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| - - #### ProfilingConfigSpec @@ -1393,6 +1376,7 @@ _Appears in:_ | `runtimeNamespace` _string_ | RuntimeNamespace is the effective Dynamo runtime namespace for this
component. Worker components may include a generation suffix; non-workers and
Grove-backed workers use the base namespace. During rolling updates, worker
status keeps the old active revision namespace until cutover completes. | | Optional: \{\}
| | `gpusPerEngine` _integer_ | GPUsPerEngine is the number of GPUs assigned to one inference engine in a
service replica, across all of its nodes. Independent auxiliary GPU
allocations are excluded. A present zero means the engine itself has no
GPUs; consult GPUsPerReplica for auxiliary allocations. | | Minimum: 0
Optional: \{\}
| | `gpusPerReplica` _integer_ | GPUsPerReplica is the unique GPU allocation added when this service scales
by one replica, across all nodes, application and initialization phases,
and provider-owned Pods. Scalar GPUs use the Kubernetes effective Pod
scheduling footprint; shared DRA claims are counted once. A present zero
records a successful non-GPU resolution; omission means no current shape
is available. | | Minimum: 0
Optional: \{\}
| +| `ready` _boolean_ | Ready is the binary determination of whether the correct number of replicas
are scheduled and available. | | Optional: \{\}
| | `replicas` _integer_ | Replicas is the total number of non-terminated replicas.
Required for all component kinds. | | Minimum: 0
| | `updatedReplicas` _integer_ | UpdatedReplicas is the number of replicas at the current/desired revision.
Required for all component kinds. | | Minimum: 0
| | `readyReplicas` _integer_ | ReadyReplicas is the number of ready replicas.
Populated for PodClique, Deployment, and LeaderWorkerSet.
Not available for PodCliqueScalingGroup.
When nil, the field is omitted from the API response. | | Minimum: 0
Optional: \{\}
| @@ -1704,6 +1688,7 @@ _Appears in:_ | `runtimeNamespace` _string_ | runtimeNamespace is the effective Dynamo runtime namespace for this
component. Worker components may include a generation suffix; non-workers
use the base namespace. During rolling updates, worker status keeps the old
active revision namespace until cutover completes. | | Optional: \{\}
| | `gpusPerEngine` _integer_ | gpusPerEngine is the number of GPUs assigned to one inference engine in a
component replica, across all of its nodes. Independent auxiliary GPU
allocations are excluded. A present zero means the engine itself has no
GPUs; consult gpusPerReplica for auxiliary allocations. | | Minimum: 0
Optional: \{\}
| | `gpusPerReplica` _integer_ | gpusPerReplica is the unique GPU allocation added when this component
scales by one replica, across all nodes, application and initialization
phases, and provider-owned Pods. Scalar GPUs use the Kubernetes effective
Pod scheduling footprint; shared DRA claims are counted once. A present
zero records a successful non-GPU resolution; omission means no current
shape is available. | | Minimum: 0
Optional: \{\}
| +| `ready` _boolean_ | ready is the binary determination of whether the correct number of replicas
are scheduled and available. | | Optional: \{\}
| | `replicas` _integer_ | replicas is the total number of non-terminated replicas. | | Minimum: 0
| | `updatedReplicas` _integer_ | updatedReplicas is the number of replicas at the current/desired revision. | | Minimum: 0
| | `readyReplicas` _integer_ | readyReplicas is the number of ready replicas. Populated for
`PodClique`, `Deployment`, and `LeaderWorkerSet`; not available for
`PodCliqueScalingGroup`. | | Minimum: 0
Optional: \{\}
| @@ -1728,7 +1713,7 @@ _Appears in:_ | `name` _string_ | name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
multinode.nodeCount; leader must be 1 and worker must be
multinode.nodeCount minus 1. | | Minimum: 1
Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. No component type supports it in this release. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. LPX components support role-specific Pod templates. | | Optional: \{\}
| #### ComponentType @@ -1742,7 +1727,7 @@ are first-class values: users can set them directly and downstream consumers (e.g., the EPP) can filter on the pod label `nvidia.com/dynamo-component-type`. _Validation:_ -- Enum: [frontend worker prefill decode planner epp] +- Enum: [frontend worker prefill decode planner epp lpx] _Appears in:_ - [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) @@ -1756,6 +1741,7 @@ _Appears in:_ | `decode` | | | `planner` | | | `epp` | | +| `lpx` | | #### DGDRPhase @@ -1833,7 +1819,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `model` _string_ | model is the model identifier (e.g. "meta-llama/Llama-3-70B").
Deprecated: legacy identity only. | | MinLength: 1
Required: \{\}
| -| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| +| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm lpu]
Required: \{\}
| | `dynamoVersion` _string_ | dynamoVersion is the Dynamo platform version.
Deprecated: legacy identity only. | | Optional: \{\}
| | `tensorParallelSize` _integer_ | tensorParallelSize is the tensor parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| | `pipelineParallelSize` _integer_ | pipelineParallelSize is the pipeline parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| @@ -1876,9 +1862,10 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides using strategic-merge-by-name -semantics. Users can add sidecars, init containers, and pod-level configuration -directly in `podTemplate` without any `extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides according to +`extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and +pod-level configuration directly in `podTemplate` without any +`extraPodSpec`-style escape hatch. @@ -1889,18 +1876,20 @@ _Appears in:_ | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the primary Grove unit representing this DGD
component. With apiVersion `grove.io/v1alpha1`, target is
`PodCliqueTemplateSpec` for a single-node component or
`PodCliqueScalingGroupConfig` for a PCSG-backed component; value may set
only `topologyConstraint`. Standalone DCD OpenAPI omits this field. | | Optional: \{\}
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| -| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp]
Optional: \{\}
| +| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| +| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | extraPodSpecMergeStrategy controls how podTemplate is merged with the
operator-generated pod and container defaults. When omitted, the operator
uses its configured defaultExtraPodSpecMergeStrategy, which defaults to
"override". | | Enum: [override strategic]
Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | eppConfig holds legacy Go-EPP configuration for Endpoint Picker Plugin
components. Only meaningful when `type` is `epp`.
Deprecated: omit this field for the native Rust EPP. Presence of
`eppConfig` selects the legacy Go EPP Pod contract (CLI flags + config
mount) so existing DGDs keep running across operator upgrades until
migration is started by clearing this field. | | Optional: \{\}
| +| `lpx` _[LPXConfig](#lpxconfig)_ | lpx holds LPX integration configuration. Only meaningful when
`type` is `lpx`. | | Optional: \{\}
| | `frontendSidecar` _string_ | frontendSidecar optionally designates a container in
`podTemplate.spec.containers` as the frontend sidecar. The value must
match the `name` of a container in that list; the operator merges its
frontend-sidecar defaults (auto-generated Dynamo env vars, ports,
health probes) into that container the same way it merges into `"main"`.
The full container definition (image, args, envFrom, env) lives in
`podTemplate` -- this eliminates the redundant `image`, `args`,
`envFromSecret`, and `envs` fields from v1alpha1's `FrontendSidecarSpec`.
The validation webhook rejects values that do not match any container
name in `podTemplate.spec.containers`. | | Optional: \{\}
| | `compilationCache` _[CompilationCacheConfig](#compilationcacheconfig)_ | compilationCache configures a PVC-backed compilation cache. The operator
handles backend-specific mount paths and environment variables, so
users do not need to hand-wire them into `podTemplate`. Extracted from
v1alpha1's `volumeMount.useAsCompilationCache` flag. | | Optional: \{\}
| | `topologyConstraint` _[TopologyConstraint](#topologyconstraint)_ | topologyConstraint applies to this component.
`topologyConstraint.packDomain` is required. When both this and
`spec.topologyConstraint.packDomain` are set, this field's `packDomain`
must be narrower than or equal to the spec-level value. | | Optional: \{\}
| @@ -1920,16 +1909,17 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `backendFramework` _string_ | backendFramework specifies the backend framework. | | Enum: [sglang vllm trtllm]
| +| `backendFramework` _string_ | backendFramework specifies the GPU backend framework (for example,
"sglang", "vllm", or "trtllm"). | | Enum: [sglang vllm trtllm]
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| -| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp]
Optional: \{\}
| +| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| +| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | extraPodSpecMergeStrategy controls how podTemplate is merged with the
operator-generated pod and container defaults. When omitted, the operator
uses its configured defaultExtraPodSpecMergeStrategy, which defaults to
"override". | | Enum: [override strategic]
Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| @@ -2002,6 +1992,25 @@ _Appears in:_ | `kvTransferPolicy` _[KvTransferPolicy](#kvtransferpolicy)_ | kvTransferPolicy configures topology-aware routing for KV-cache
transfers between prefill and decode workers. | | Optional: \{\}
| +#### DynamoGraphDeploymentLPXStatus + + + +DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. +Both served DGD versions use this type, so conversion preserves its live fields natively. + + + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `modelDownload` _[ModelDownloadStatus](#modeldownloadstatus)_ | modelDownload contains the model download status for remote builds. | | Optional: \{\}
| +| `placement` _[PlacementStatus](#placementstatus)_ | placement contains LPX scheduling progress and placement signals. | | Optional: \{\}
| + + #### v1beta1 DynamoGraphDeploymentRequest @@ -2163,6 +2172,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the root resource generated by the selected
graph-level workload provider. For Grove it uses apiVersion
`grove.io/v1alpha1`, target `PodCliqueSet`, and may set only
`spec.template.topologyConstraint`. It cannot select or change the provider. | | Optional: \{\}
| +| `scheduling` _[SchedulingSpec](#schedulingspec)_ | scheduling configures attempts for LPX components. Omission means no deadline. | | Optional: \{\}
| | `annotations` _object (keys:string, values:string)_ | annotations to propagate to all child resources (PCS, DCD, Deployments,
and pod templates). Component-level (`podTemplate`) values take precedence
on conflict. | | Optional: \{\}
| | `labels` _object (keys:string, values:string)_ | labels to propagate to all child resources. Same precedence rules as `annotations`. | | Optional: \{\}
| | `priorityClassName` _string_ | priorityClassName is the name of the PriorityClass to use for Grove PodCliqueSets.
Requires the Grove pathway. | | Optional: \{\}
| @@ -2194,8 +2204,8 @@ _Appears in:_ | `components` _object (keys:string, values:[ComponentReplicaStatus](#componentreplicastatus))_ | components contains per-component replica status information, keyed by component name. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| +| `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | lpx contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placement` _[PlacementStatus](#placementstatus)_ | placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| #### EPPConfig @@ -2247,6 +2257,25 @@ _Appears in:_ | `checkpoint` _[ComponentCheckpointConfig](#componentcheckpointconfig)_ | checkpoint configures container-image snapshotting and restore for
this component. Set `checkpoint.enabled: true` to opt in. Without
checkpointRef, the DGD controller creates a DGD-scoped SnapshotJob and
later restores pods in the same DGD generation from its PodSnapshot.
With checkpointRef, the DGD restores from the named
PodSnapshot in the same namespace. The user-facing shape of this field is
still settling, which is why it lives under `experimental` in v1beta1
instead of at the top level. | | Optional: \{\}
| +#### ExtraPodSpecMergeStrategy + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [override strategic] + +_Appears in:_ +- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) + +| Field | Description | +| --- | --- | +| `override` | | +| `strategic` | | + + #### FailoverSpec @@ -2477,6 +2506,63 @@ _Appears in:_ | `preferredWeight` _float_ | preferredWeight is required and used only when enforcement is
"preferred". Higher values create a stronger same-domain routing
preference, but do not guarantee same-domain selection. The value is not
a probability; worker selection still depends on load and other routing
inputs. A value of 0 disables the topology preference; 1 is the strongest
supported preference. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +#### LPXAttemptRequestStatus + + + +LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + + + +_Appears in:_ +- [LPXAttemptStatus](#lpxattemptstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | | | | +| `attemptDigest` _string_ | | | | +| `uid` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | + + +#### LPXAttemptStatus + + + +LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. + + + +_Appears in:_ +- [PlacementStatus](#placementstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | | | | +| `podCliqueSetUID` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | +| `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | MaxItems: 9
MinItems: 1
| + + +#### LPXConfig + + + +LPXConfig identifies the component's compiled model and runtime settings. + + + +_Appears in:_ +- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `buildId` _string_ | buildId references the immutable model build. | | MinLength: 1
| +| `settings` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | settings override the existing model configuration. | | Type: object
Optional: \{\}
| + + #### MockerSpec @@ -2511,6 +2597,23 @@ _Appears in:_ | `pvcMountPath` _string_ | PVCMountPath is the mount path for the PVC inside the container. | /opt/model-cache | Optional: \{\}
| +#### ModelDownloadStatus + + + +ModelDownloadStatus contains the status of remote LPU model downloads. + + + +_Appears in:_ +- [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `builds` _string array_ | builds is the sorted set of resolved remote LPU build URLs whose artifacts
were successfully downloaded into model-storage. | | Optional: \{\}
| +| `lastCheckedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. | | Optional: \{\}
| + + #### ModelReference @@ -2646,12 +2749,13 @@ conversion are landed here so downstream consumers can rely on the shape. _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `score` _float_ | score is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation
uses the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| | `state` _[PlacementScoreState](#placementscorestate)_ | state indicates placement score reporting state. See PlacementScoreState
for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| +| `lpxAttempt` _[LPXAttemptStatus](#lpxattemptstatus)_ | lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. | | Optional: \{\}
| #### ProfilingPhase @@ -2890,6 +2994,23 @@ _Appears in:_ +#### SchedulingSpec + + + +SchedulingSpec configures LPX scheduling attempts. + + + +_Appears in:_ +- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `attemptDeadlineSeconds` _integer_ | attemptDeadlineSeconds limits one aggregate LPX scheduling attempt.
Omission means unlimited; the value is not a solver budget. | | Maximum: 9.223372036e+09
Minimum: 1
Optional: \{\}
| + + #### SearchStrategy _Underlying type:_ _string_ @@ -3226,6 +3347,23 @@ _Appears in:_ +#### LPXConfiguration + + + +LPXConfiguration holds LPX scheduler integration and model-registry settings. + + + +_Appears in:_ +- [OperatorConfiguration](#operatorconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `enabled` _boolean_ | Enabled opts the operator into LPX request production. When true, startup
verifies that the separately installed LPX API serves
scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails
otherwise. | false | | +| `modelRegistryURL` _string_ | ModelRegistryURL configures the location of the LPU model registry used by LPX.
Supported values are an absolute local path (or file:// URL) and gs:// URLs. | | | + + #### LWSConfiguration @@ -3371,10 +3509,12 @@ OperatorConfiguration is the Schema for the operator configuration. | `rbac` _[RBACConfiguration](#rbacconfiguration)_ | RBAC configuration for cross-namespace resource management (cluster-wide mode) | | | | `mpi` _[MPIConfiguration](#mpiconfiguration)_ | MPI SSH secret configuration | | | | `checkpoint` _[CheckpointConfiguration](#checkpointconfiguration)_ | Checkpoint/restore configuration | | | +| `podGeneration` _[PodGenerationConfiguration](#podgenerationconfiguration)_ | Pod generation defaults | | | | `discovery` _[DiscoveryConfiguration](#discoveryconfiguration)_ | Discovery backend configuration | | | | `gpu` _[GPUConfiguration](#gpuconfiguration)_ | GPU discovery configuration | | | | `logging` _[LoggingConfiguration](#loggingconfiguration)_ | Logging configuration | | | | `security` _[SecurityConfiguration](#securityconfiguration)_ | HTTP/2 and TLS settings | | | +| `lpx` _[LPXConfiguration](#lpxconfiguration)_ | LPX scheduler integration and model-registry configuration. | | | #### OrchestratorConfiguration @@ -3396,6 +3536,22 @@ _Appears in:_ | `volcanoScheduler` _[VolcanoSchedulerConfiguration](#volcanoschedulerconfiguration)_ | VolcanoScheduler configuration | | | +#### PodGenerationConfiguration + + + +PodGenerationConfiguration holds pod generation defaults applied by the operator. + + + +_Appears in:_ +- [OperatorConfiguration](#operatorconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `defaultExtraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a
component leaves spec.extraPodSpecMergeStrategy unset. | override | Enum: [override strategic]
| + + #### RBACConfiguration diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 46b3e6ce269a..53bc540303be 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -41,6 +41,7 @@ The DCD spec is `backendFramework` plus the [shared component spec](#shared-comp Inference backend framework for this component. Drives backend-specific defaults the operator injects into the `main` container. Allowed values: sglang vllm trtllm + ### Shared component spec @@ -52,18 +53,28 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - Role of the component within the graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. `prefill` and `decode` are first-class values for disaggregated serving and can be set directly. At most one component per graph may be `epp`. + Role of the component within the graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. `prefill` and `decode` are first-class values for disaggregated serving and can be set directly. At most one component per graph may be `epp`. Standalone DCDs cannot use `lpx`. + + Allowed values: frontend worker prefill decode planner epp lpx + - Allowed values: frontend worker prefill decode planner epp + + DGD-only configuration for a `type: lpx` component; omitted from the standalone DCD schema. Set the required `buildId` and optional build `settings`. See [LPXConfig](full-api-reference.mdx#lpxconfig) for the full field set. + + Supported modes are LPU, hybrid, and LPU draft with LPU target. Declare one complete component, or separate speculative draft and target components. The draft has only a `worker` role; the target has `leader` and `worker` roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. Desired number of complete component instances. For a single-node component, each instance is one pod. For a multinode component, each instance contains all of its roles. Minimum `0`. When `scalingAdapter` is set, this field is owned by the [DynamoGraphDeploymentScalingAdapter](full-api-reference.mdx#dynamographdeploymentscalingadapter) and should not be modified directly. + + For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid engines support `1–9` fixed replicas; LPU-only requires `1`. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. Pod template for the component's pods. The operator injects its defaults (image, command, env, ports, probes, resources, volume mounts) into the container named `main`, merging your overrides by name. If no `main` container is present, the operator generates one. Every other container is treated as a user-managed sidecar and receives no injected defaults — sidecars must specify their own required fields such as `image`. Replaces the ten separate per-component fields (`resources`, `envs`, `livenessProbe`, and so on) that existed in `v1alpha1`. + For `type: lpx`, omit this field and use `roles[].podTemplate`. Each supplied LPX template requires an explicit `main` container image; see `roles` below for leader-template fallback. + core/v1.PodTemplateSpec @@ -78,18 +89,29 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - Optional explicit structure of a compound component. In this release, roles are supported for multinode components and must contain exactly one `leader` and one `worker`. Omit the field to retain the implicit leader/worker layout. + Optional explicit structure of a compound component. Multinode components must contain exactly one `leader` and one `worker`. Omit the field to retain the implicit leader/worker layout. + + LPX components use `worker` for Agent Pods and `leader` for the conductor. With one LPX component, the leader role is optional. In a speculative pair, the draft has only a worker role and the target declares both roles. Hybrid execution may also create Nova Pods from Agent configuration. + + Standalone DCD roles accept `name` and `replicas`. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). - Semantic role name. Multinode components allow `leader` and `worker`. + Semantic role name, either `leader` or `worker`. Names must be unique within the component. + + + Native Pod template for this role. LPX-only; required for the worker. Each supplied template requires a `main` container with an explicit image. An omitted leader template reuses the same component's worker template; a single LPX component also uses this fallback when its leader role is omitted. A speculative pair uses the target's leader. An explicit leader template merges with shared defaults independently, without inheriting the worker template. + + core/v1.PodTemplateSpec Logical cardinality of the role in one complete component instance. For multinode, admission defaults and persists an omitted value as `leader: 1` or `worker: multinode.nodeCount - 1`; an explicit value must match that fixed shape. + + For LPX, omitted leader replicas use `1`, subject to runtime-specific limits. Worker replicas are derived from the model build; an explicit count must match the build. - Provider configuration for the workload unit generated for this role. Available only on components embedded in a DGD; standalone DCD OpenAPI omits this field. + Provider configuration for the workload unit generated for this role. Available only on ordinary multinode components embedded in a DGD. LPX roles do not support it; standalone DCD OpenAPI omits this field. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 8f37c5ef3aa6..92aac3752ed4 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -45,7 +45,7 @@ spec: - Backend framework for the graph. Sets backend-specific defaults the operator injects into each component's `main` container. + GPU backend framework for worker components. When omitted, the operator infers the framework from each worker's command and arguments. When set, it must match the detected worker framework. LPX components omit this field. Allowed values: sglang vllm trtllm @@ -68,6 +68,17 @@ spec: Name of the `PriorityClass` to use for Grove `PodCliqueSet`s. Requires the Grove pathway. See [Multinode Orchestration](../../kubernetes/installation/multinode-orchestration.md). + + Configures scheduling attempts for `type: lpx` components. Omit this object for attempts without a deadline. + + + + + Wall-clock lifetime of the aggregate LPX attempt, from the first request's API-server creation time through `Bound`, `NoFit`, or `Unsupported`. Valid values are `1` through `9,223,372,036`; omission means unlimited. + Dynamo durably records the absolute deadline. Resumed scheduling keeps it; late terminal status loses to expiry. Expiry reports `LPXSchedulingDeadlineExceeded` and retires the exact attempt through LPX cleanup. It is neither a solver budget nor retry authorization. + + + Restart policy for the graph. Change `restart.id` to trigger a restart. @@ -157,6 +168,19 @@ The operator maintains observed state under `status`. Most recent `metadata.generation` observed by the controller. + + LPX download and placement observations for the current configuration. Omitted when no current LPX result exists, including ordinary-only deployments. LPX component entries and aggregate readiness remain under `status.components`, `status.state`, and `status.conditions`. + + + + + Downloaded builds and the time they were last checked. + + + Placement progress and scheduling-attempt details. The attempt's `observedGeneration` refers to this DGD's generation. + + + Status of a graph-level restart in progress. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 7e840fd5b403..1ff0ab5512cf 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,12 +10,12 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 3 Kubernetes API packages with 166 typed CRD and config sections. +Dynamo publishes 3 Kubernetes API packages with 174 typed CRD and config sections. -69 types -69 types -28 types +68 types +76 types +30 types
@@ -161,7 +161,7 @@ See [ProviderOverride](#provideroverride). -PodTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. No component type supports it in this release. +PodTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. LPX components support role-specific Pod templates. See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). **Validation:** Optional: \{\} @@ -293,8 +293,8 @@ Model is the model identifier (e.g., "meta-llama/Llama-3-70B"). Deprecated: lega -BackendFramework is the runtime framework (vllm, sglang, trtllm). Deprecated: legacy identity only. -**Validation:** Enum: [vllm sglang trtllm] Required: \{\} +BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). Deprecated: legacy identity only. +**Validation:** Enum: [vllm sglang trtllm lpu] Required: \{\} @@ -445,6 +445,12 @@ See [ExtraPodSpec](#extrapodspec). **Validation:** Optional: \{\} + +ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the operator-generated pod and container defaults. +See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). +**Validation:** Enum: [override strategic] Optional: \{\} + + LivenessProbe to detect and restart unhealthy containers. See [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core). @@ -471,7 +477,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. See [ComponentRoleSpec](#componentrolespec). **Validation:** Optional: \{\} @@ -494,6 +500,12 @@ See [FrontendSidecarSpec](#frontendsidecarspec). **Validation:** Optional: \{\} + +LPX holds LPX integration configuration. Only meaningful when ComponentType is "lpx". +See [LPXConfig](#lpxconfig). +**Validation:** Optional: \{\} + + Checkpoint configures container checkpointing for this service. When enabled, pods can be restored from a checkpoint files for faster cold start. See [ServiceCheckpointConfig](#servicecheckpointconfig). @@ -527,7 +539,7 @@ DynamoComponentDeploymentSpec defines the desired state of DynamoComponentDeploy **Appears in:** [DynamoComponentDeployment](#dynamocomponentdeployment) -BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") +BackendFramework specifies the GPU backend framework (for example, "sglang", "vllm", or "trtllm"). **Validation:** Enum: [sglang vllm trtllm] @@ -612,6 +624,12 @@ See [ExtraPodSpec](#extrapodspec). **Validation:** Optional: \{\} + +ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the operator-generated pod and container defaults. +See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). +**Validation:** Enum: [override strategic] Optional: \{\} + + LivenessProbe to detect and restart unhealthy containers. See [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core). @@ -638,7 +656,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} @@ -982,6 +1000,12 @@ See [ProviderOverride](#provideroverride). **Validation:** Optional: \{\} + +Scheduling configures attempts for LPX components. Omission means no deadline. +See [SchedulingSpec](#schedulingspec). +**Validation:** Optional: \{\} + + Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). Service-level annotations take precedence over these values. **Validation:** Optional: \{\} @@ -1080,15 +1104,15 @@ See [ServiceCheckpointStatus](#servicecheckpointstatus). **Validation:** Optional: \{\} - -RollingUpdate tracks the progress of operator manged rolling updates. Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). -See [RollingUpdateStatus](#rollingupdatestatus). + +LPX contains the status of the graph's LPX workload, when present. +See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). **Validation:** Optional: \{\} - -Placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). -See [PlacementStatus](#placementstatus). + +RollingUpdate tracks the progress of operator manged rolling updates. Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). +See [RollingUpdateStatus](#rollingupdatestatus). **Validation:** Optional: \{\} @@ -1253,6 +1277,19 @@ See [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1 + + +**Kind:** `enum` +**Underlying type:** `string` +**Validation:** Enum: [override strategic] +**Appears in:** [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec), [PodGenerationConfiguration](#podgenerationconfiguration) + +**Allowed values** + +- override +- strategic + + FailoverSpec configures active-passive failover for a worker component. For intraPod mode: requires gpuMemoryService.enabled; the main container is cloned @@ -1559,46 +1596,6 @@ See [PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/ - -PlacementScoreState describes whether placement score is available and how -complete the reported score is for a graph deployment. See the v1beta1 -PlacementScoreState for the authoritative semantics of each value. - - -**Kind:** `enum` -**Underlying type:** `string` -**Validation:** Enum: [Reported Partial Unsupported Unknown] -**Appears in:** [PlacementStatus](#placementstatus) - -**Allowed values** - -- Reported -- Partial -- Unsupported -- Unknown - - - -PlacementStatus groups DGD-level scheduler placement fields under a single -status object so future placement signals can be added without a schema -break. See the v1beta1 PlacementStatus for the authoritative field docs. - - -**Kind:** `type` -**Appears in:** [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) - - -Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] where higher is better and 1.0 is the best possible placement. -**Validation:** Maximum: 1 Minimum: 0 Optional: \{\} - - - -State indicates placement score reporting state. -See [PlacementScoreState](#placementscorestate). -**Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} - - - ProfilingConfigSpec defines configuration for the profiling process. This structure maps directly to the profile_sla.py config format. @@ -2016,6 +2013,11 @@ GPUsPerReplica is the unique GPU allocation added when this service scales by on **Validation:** Minimum: 0 Optional: \{\} + +Ready is the binary determination of whether the correct number of replicas are scheduled and available. +**Validation:** Optional: \{\} + + Replicas is the total number of non-terminated replicas. Required for all component kinds. **Validation:** Minimum: 0 @@ -2380,6 +2382,11 @@ gpusPerReplica is the unique GPU allocation added when this component scales by **Validation:** Minimum: 0 Optional: \{\} + +ready is the binary determination of whether the correct number of replicas are scheduled and available. +**Validation:** Optional: \{\} + + replicas is the total number of non-terminated replicas. **Validation:** Minimum: 0 @@ -2431,7 +2438,7 @@ See [ProviderOverride](#v1beta1-provideroverride). -podTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. No component type supports it in this release. +podTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. LPX components support role-specific Pod templates. See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). **Validation:** Optional: \{\} @@ -2447,7 +2454,7 @@ are first-class values: users can set them directly and downstream consumers **Kind:** `enum` **Underlying type:** `string` -**Validation:** Enum: [frontend worker prefill decode planner epp] +**Validation:** Enum: [frontend worker prefill decode planner epp lpx] **Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#v1beta1-dynamocomponentdeploymentspec) **Allowed values** @@ -2458,6 +2465,7 @@ are first-class values: users can set them directly and downstream consumers - decode - planner - epp +- lpx @@ -2529,8 +2537,8 @@ model is the model identifier (e.g. "meta-llama/Llama-3-70B"). Deprecated: legac -backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). Deprecated: legacy identity only. -**Validation:** Enum: [vllm sglang trtllm] Required: \{\} +backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). Deprecated: legacy identity only. +**Validation:** Enum: [vllm sglang trtllm lpu] Required: \{\} @@ -2602,9 +2610,10 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides using strategic-merge-by-name -semantics. Users can add sidecars, init containers, and pod-level configuration -directly in `podTemplate` without any `extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides according to +`extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and +pod-level configuration directly in `podTemplate` without any +`extraPodSpec`-style escape hatch. **Kind:** `type` @@ -2624,7 +2633,7 @@ name is the stable logical identifier for this component within its DynamoGraphD type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. See [ComponentType](#componenttype). -**Validation:** Enum: [frontend worker prefill decode planner epp] Optional: \{\} +**Validation:** Enum: [frontend worker prefill decode planner epp lpx] Optional: \{\} @@ -2643,6 +2652,12 @@ See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes- **Validation:** Optional: \{\} + +extraPodSpecMergeStrategy controls how podTemplate is merged with the operator-generated pod and container defaults. When omitted, the operator uses its configured defaultExtraPodSpecMergeStrategy, which defaults to "override". +See [ExtraPodSpecMergeStrategy](#v1beta1-extrapodspecmergestrategy). +**Validation:** Enum: [override strategic] Optional: \{\} + + replicas is the desired number of Pods for this component. When `scalingAdapter` is set on this component, this field is managed by the DynamoGraphDeploymentScalingAdapter and should not be modified directly. **Validation:** Minimum: 0 Optional: \{\} @@ -2660,7 +2675,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. See [ComponentRoleSpec](#v1beta1-componentrolespec). **Validation:** Optional: \{\} @@ -2689,6 +2704,12 @@ See [EPPConfig](#v1beta1-eppconfig). **Validation:** Optional: \{\} + +lpx holds LPX integration configuration. Only meaningful when `type` is `lpx`. +See [LPXConfig](#lpxconfig). +**Validation:** Optional: \{\} + + frontendSidecar optionally designates a container in `podTemplate.spec.containers` as the frontend sidecar. The value must match the `name` of a container in that list; the operator merges its frontend-sidecar defaults (auto-generated Dynamo env vars, ports, health probes) into that container the same way it merges into `"main"`. The full container definition (image, args, envFrom, env) lives in `podTemplate` -- this eliminates the redundant `image`, `args`, `envFromSecret`, and `envs` fields from v1alpha1's `FrontendSidecarSpec`. The validation webhook rejects values that do not match any container name in `podTemplate.spec.containers`. **Validation:** Optional: \{\} @@ -2721,7 +2742,7 @@ DynamoComponentDeploymentSpec defines the desired state of a DynamoComponentDepl **Appears in:** [DynamoComponentDeployment](#v1beta1-dynamocomponentdeployment) -backendFramework specifies the backend framework. +backendFramework specifies the GPU backend framework (for example, "sglang", "vllm", or "trtllm"). **Validation:** Enum: [sglang vllm trtllm] @@ -2733,7 +2754,7 @@ name is the stable logical identifier for this component within its DynamoGraphD type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. See [ComponentType](#componenttype). -**Validation:** Enum: [frontend worker prefill decode planner epp] Optional: \{\} +**Validation:** Enum: [frontend worker prefill decode planner epp lpx] Optional: \{\} @@ -2752,6 +2773,12 @@ See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes- **Validation:** Optional: \{\} + +extraPodSpecMergeStrategy controls how podTemplate is merged with the operator-generated pod and container defaults. When omitted, the operator uses its configured defaultExtraPodSpecMergeStrategy, which defaults to "override". +See [ExtraPodSpecMergeStrategy](#v1beta1-extrapodspecmergestrategy). +**Validation:** Enum: [override strategic] Optional: \{\} + + replicas is the desired number of Pods for this component. When `scalingAdapter` is set on this component, this field is managed by the DynamoGraphDeploymentScalingAdapter and should not be modified directly. **Validation:** Minimum: 0 Optional: \{\} @@ -2769,7 +2796,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} @@ -2893,6 +2920,27 @@ See [KvTransferPolicy](#v1beta1-kvtransferpolicy). + +DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. +Both served DGD versions use this type, so conversion preserves its live fields natively. + + +**Kind:** `type` +**Appears in:** [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) + + +modelDownload contains the model download status for remote builds. +See [ModelDownloadStatus](#modeldownloadstatus). +**Validation:** Optional: \{\} + + + +placement contains LPX scheduling progress and placement signals. +See [PlacementStatus](#placementstatus). +**Validation:** Optional: \{\} + + + DynamoGraphDeploymentRequest is the Schema for the dynamographdeploymentrequests API. It provides a simplified, SLA-driven interface for deploying inference models on Dynamo. @@ -3159,6 +3207,12 @@ See [ProviderOverride](#v1beta1-provideroverride). **Validation:** Optional: \{\} + +scheduling configures attempts for LPX components. Omission means no deadline. +See [SchedulingSpec](#schedulingspec). +**Validation:** Optional: \{\} + + annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). Component-level (`podTemplate`) values take precedence on conflict. **Validation:** Optional: \{\} @@ -3253,15 +3307,15 @@ See [ComponentCheckpointStatus](#componentcheckpointstatus). **Validation:** Optional: \{\} - -rollingUpdate tracks the progress of operator-managed rolling updates. Currently only supported for single-node, non-Grove deployments (DCD/Deployment). -See [RollingUpdateStatus](#v1beta1-rollingupdatestatus). + +lpx contains the status of the graph's LPX workload, when present. +See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). **Validation:** Optional: \{\} - -placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). -See [PlacementStatus](#v1beta1-placementstatus). + +rollingUpdate tracks the progress of operator-managed rolling updates. Currently only supported for single-node, non-Grove deployments (DCD/Deployment). +See [RollingUpdateStatus](#v1beta1-rollingupdatestatus). **Validation:** Optional: \{\} @@ -3328,6 +3382,19 @@ See [ComponentCheckpointConfig](#componentcheckpointconfig). + + +**Kind:** `enum` +**Underlying type:** `string` +**Validation:** Enum: [override strategic] +**Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#v1beta1-dynamocomponentdeploymentspec) + +**Allowed values** + +- override +- strategic + + FailoverSpec configures active-passive failover for a worker component. The main container is cloned into two engine containers (active + standby) @@ -3598,6 +3665,75 @@ preferredWeight is required and used only when enforcement is "preferred". Highe + +LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + + +**Kind:** `type` +**Appears in:** [LPXAttemptStatus](#lpxattemptstatus) + + + + + + + + +See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). + + + + +LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. + + +**Kind:** `type` +**Appears in:** [PlacementStatus](#placementstatus) + + + + + +See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [LPXAttemptRequestStatus](#lpxattemptrequeststatus). +**Validation:** MaxItems: 9 MinItems: 1 + + + + +LPXConfig identifies the component's compiled model and runtime settings. + + +**Kind:** `type` +**Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) + + +buildId references the immutable model build. +**Validation:** MinLength: 1 + + + +settings override the existing model configuration. +See [JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io). +**Validation:** Type: object Optional: \{\} + + + MockerSpec configures the simulated (mocker) backend. @@ -3634,6 +3770,25 @@ PVCMountPath is the mount path for the PVC inside the container. + +ModelDownloadStatus contains the status of remote LPU model downloads. + + +**Kind:** `type` +**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) + + +builds is the sorted set of resolved remote LPU build URLs whose artifacts were successfully downloaded into model-storage. +**Validation:** Optional: \{\} + + + +lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). +**Validation:** Optional: \{\} + + + ModelReference identifies a model served by a component. When specified, a headless service is created for endpoint discovery. @@ -3722,7 +3877,7 @@ See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api - + PlacementScoreState describes whether placement score is available and how complete the reported score is for a graph deployment. @@ -3738,7 +3893,7 @@ Every backend must set this field after the first reconciliation: **Kind:** `enum` **Underlying type:** `string` **Validation:** Enum: [Reported Partial Unsupported Unknown] -**Appears in:** [PlacementStatus](#v1beta1-placementstatus) +**Appears in:** [PlacementStatus](#placementstatus) **Allowed values** @@ -3748,7 +3903,7 @@ Every backend must set this field after the first reconciliation: - Unknown - + PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. @@ -3760,7 +3915,7 @@ conversion are landed here so downstream consumers can rely on the shape. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) +**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) score is the DGD-level scheduler placement score aggregated from relevant scheduler placement units. Normalized to [0.0, 1.0] where higher is better and 1.0 represents the best possible placement. Aggregation uses the minimum across placement units so the value is a worst-placement signal for the graph. Scores are only comparable across DGDs that share the same scheduler scoring contract and version. @@ -3769,9 +3924,15 @@ score is the DGD-level scheduler placement score aggregated from relevant schedu state indicates placement score reporting state. See PlacementScoreState for the semantics of each value. -See [PlacementScoreState](#v1beta1-placementscorestate). +See [PlacementScoreState](#placementscorestate). **Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} + + +lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. +See [LPXAttemptStatus](#lpxattemptstatus). +**Validation:** Optional: \{\} + @@ -4029,6 +4190,19 @@ Omit the field to opt out. **Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#v1beta1-dynamocomponentdeploymentspec) + +SchedulingSpec configures LPX scheduling attempts. + + +**Kind:** `type` +**Appears in:** [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) + + +attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. Omission means unlimited; the value is not a solver budget. +**Validation:** Maximum: 9.223372036e+09 Minimum: 1 Optional: \{\} + + + SearchStrategy controls the profiling search depth. @@ -4389,6 +4563,22 @@ Enabled overrides auto-detection. nil = auto-detect. + +LPXConfiguration holds LPX scheduler integration and model-registry settings. + + +**Kind:** `type` +**Appears in:** [OperatorConfiguration](#operatorconfiguration) + + +Enabled opts the operator into LPX request production. When true, startup verifies that the separately installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails otherwise. + + + +ModelRegistryURL configures the location of the LPU model registry used by LPX. Supported values are an absolute local path (or file:// URL) and gs:// URLs. + + + LWSConfiguration holds LWS orchestrator settings. @@ -4577,6 +4767,11 @@ Checkpoint/restore configuration See [CheckpointConfiguration](#checkpointconfiguration). + +Pod generation defaults +See [PodGenerationConfiguration](#podgenerationconfiguration). + + Discovery backend configuration See [DiscoveryConfiguration](#discoveryconfiguration). @@ -4596,6 +4791,11 @@ See [LoggingConfiguration](#loggingconfiguration). HTTP/2 and TLS settings See [SecurityConfiguration](#securityconfiguration). + + +LPX scheduler integration and model-registry configuration. +See [LPXConfiguration](#lpxconfiguration). + @@ -4626,6 +4826,20 @@ See [VolcanoSchedulerConfiguration](#volcanoschedulerconfiguration). + +PodGenerationConfiguration holds pod generation defaults applied by the operator. + + +**Kind:** `type` +**Appears in:** [OperatorConfiguration](#operatorconfiguration) + + +DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a component leaves spec.extraPodSpecMergeStrategy unset. +See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). +**Validation:** Enum: [override strategic] + + + RBACConfiguration holds RBAC settings for cluster-wide mode. diff --git a/docs/fern/scripts/kubernetes_api_rendering.py b/docs/fern/scripts/kubernetes_api_rendering.py index 3f3c75e0cde0..3257b5cd3d33 100644 --- a/docs/fern/scripts/kubernetes_api_rendering.py +++ b/docs/fern/scripts/kubernetes_api_rendering.py @@ -94,9 +94,12 @@ def render_mdx(reference: KubernetesReference) -> str: "", _package_index(reference), ] + valid_anchors = { + type_.anchor for package in reference.packages for type_ in package.types + } for package in reference.packages: parts.append("") - parts.append(_package_section(package)) + parts.append(_package_section(package, valid_anchors)) parts.append("") parts.append(_operator_defaults(reference.operator_defaults)) parts.append("") @@ -122,9 +125,8 @@ def _package_index(reference: KubernetesReference) -> str: return "\n".join(lines) -def _package_section(package: KubernetesPackage) -> str: +def _package_section(package: KubernetesPackage, valid_anchors: set[str]) -> str: """One package heading, its resource-type cards, and every typed section.""" - valid_anchors = {type_.anchor for type_ in package.types} # A bare
target, not an : package tables link to #anchor, but an # with no text is an empty link to assistive tech. lines = [f'
', "", f"## {package.name}"] @@ -174,7 +176,7 @@ def _type_metadata(type_: KubernetesType, valid_anchors: set[str]) -> list[str]: lines.append(f"**Underlying type:** `{_plain(type_.underlying_type)}`") if type_.validation: lines.append(f"**Validation:** {_prose(type_.validation)}") - refs = [ref for ref in type_.appears_in if ref.anchor in valid_anchors] + refs = dict.fromkeys(ref for ref in type_.appears_in if ref.anchor in valid_anchors) if refs: joined = ", ".join(f"[{ref.name}](#{ref.anchor})" for ref in refs) lines.append(f"**Appears in:** {joined}") diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index 5dc0f05b18bc..aa93ac016a6f 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -48,9 +48,9 @@ # (PodReference, PodSnapshotSource/Spec/Status, # PodSnapshotContentSource/Spec/Status, PodSnapshotReference), which are # owned by github.com/ai-dynamo/snapshot. - "nvidia.com/v1alpha1": 69, - "nvidia.com/v1beta1": 69, - "operator.config.dynamo.nvidia.com/v1alpha1": 28, + "nvidia.com/v1alpha1": 68, + "nvidia.com/v1beta1": 76, + "operator.config.dynamo.nvidia.com/v1alpha1": 30, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( "Pod Specification Defaults", @@ -394,6 +394,7 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( by_name = {type_.name: type_ for type_ in package.types} dcd = by_name["DynamoComponentDeploymentSpec"] assert "providerOverride" not in {field.name for field in dcd.fields} + assert "lpx" not in {field.name for field in dcd.fields} dcd_multinode = next(field for field in dcd.fields if field.name == "multinode") assert "MultinodeSpec" in dcd_multinode.type dcd_roles = next(field for field in dcd.fields if field.name == "roles") @@ -415,6 +416,17 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( } +def test_shared_lpx_type_links_across_api_versions( + reference: kubernetes_api_discovery.KubernetesReference, +) -> None: + rendered = kubernetes_api_rendering.render_mdx(reference) + alpha = rendered.split("## nvidia.com/v1beta1", 1)[0] + for name in ("LPXConfig", "SchedulingSpec", "DynamoGraphDeploymentLPXStatus"): + assert f"See [{name}](#{name.lower()})." in alpha + lpx = rendered.split('", 1)[0] + assert lpx.count("[DynamoComponentDeploymentSharedSpec]") == 1 + + def test_check_mode_returns_zero_on_fresh_outputs(workspace: Path) -> None: fern = workspace / "docs" / "fern" assert gen_kubernetes_api.main(["--fern-root", str(fern)]) == 0 From e58aad7137f00db7c069e47b350b2dd26c0b48ac Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 9 Sep 2026 12:10:52 +0000 Subject: [PATCH 002/196] fix(operator): separate LPX scaling-group replicas Syncs the complete lpu-dynamo delta between the recorded fork checkpoints. Fork-Source-Base: 8f44ed118d7f82289096bebb57bd215da7db9af4 Fork-Source-Head: bedd20140be7a5ed325b99a1ec4285420569c161 Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller_test.go | 2 +- .../dynamo/lpx/hybrid_cyborg_bindings.go | 2 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 6 +++--- .../internal/dynamo/lpx/materialization.go | 2 +- .../internal/dynamo/lpx/materialization_test.go | 2 +- .../operator/internal/dynamo/lpx/render_test.go | 6 +++--- .../dynamo/lpx/resolve_selected_workload.go | 17 ++++++++++------- .../lpx/resolve_selected_workload_test.go | 1 + .../internal/dynamo/lpx/selected_workload.go | 6 +++--- 9 files changed, 24 insertions(+), 20 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 29358cb5726c..ee1c31efec64 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -152,7 +152,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { }{ {name: "intent", reason: "LPXRejected", message: "providerOverride"}, {name: "name budget", reason: "LPXRejected", message: "spec.components[0].name: Invalid value"}, - {name: "selected workload", reason: "LPXRejected", message: "fixed engine replicas"}, + {name: "selected workload", reason: "LPXRejected", message: "scaling-group replicas"}, {name: "render", reason: "LPXReconciliationFailed", message: "model storage volume mount"}, {name: "invalid source", reason: "LPXRejected", message: "source"}, {name: "transient snapshot", reason: "LPXReconciliationFailed", message: "temporary snapshot timeout", preserve: true}, diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 358a841a48c3..589537785962 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -84,7 +84,7 @@ func ApplySelectedCyborgContainerDefaults( return err } hostsFile := lpuConfigMountPath + "/lpu_servers" - if workload.engineReplicas > 1 { + if workload.scalingGroupReplicas > 1 { // Resolve the replica through the downward API before Kubernetes expands // SERVER_HOSTS_FILE. The image entrypoint remains untouched. container.Env = append(container.Env, corev1.EnvVar{ diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index f203859eea55..195420b74acf 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -34,8 +34,8 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { projection := projectionBatch[0] projection.stage = testRenderComponentName workload := &SelectedWorkload{ - modelProjections: []*ModelProjection{projection}, - engineReplicas: 1, + modelProjections: []*ModelProjection{projection}, + scalingGroupReplicas: 1, } t.Log("Resolve the projected runtime path") @@ -47,7 +47,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Verify every engine replica addresses only its own Agents") for _, replicas := range []int32{1, 3, 9} { t.Run(strconv.Itoa(int(replicas)), func(t *testing.T) { - workload.engineReplicas = replicas + workload.scalingGroupReplicas = replicas plan, err := workload.PlanNodeLocalMaterialization("test-dgd-lpx") require.NoError(t, err) configMap, err := renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 75b4a40ea7cb..88cc638efeb9 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -118,7 +118,7 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi grovecommon.ResourceNameReplica{Name: pcsName, Replica: 0}, lpxScalingGroupTemplate, ), LPXScalingGroupTemplate: lpxScalingGroupTemplate, - Replicas: w.engineReplicas, + Replicas: w.scalingGroupReplicas, } plan = plan.ForReplica(0) diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index d65a00240ff8..b6c3c020d93e 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -43,7 +43,7 @@ func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { test.projection.stage = "longcomponent" - workload := &SelectedWorkload{modelProjections: []*ModelProjection{test.projection}, engineReplicas: 1} + workload := &SelectedWorkload{modelProjections: []*ModelProjection{test.projection}, scalingGroupReplicas: 1} pcsName := strings.Repeat("a", test.pcsLength) t.Log("Plan and verify materialization within the DNS hostname budget") plan, err := workload.PlanNodeLocalMaterialization(pcsName) diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 64546649385e..4a4df561183b 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -38,9 +38,9 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model return nil, err } workload := &SelectedWorkload{ - modelProjections: projections, - digest: digest, - engineReplicas: 1, + modelProjections: projections, + digest: digest, + scalingGroupReplicas: 1, } plan, err := workload.PlanNodeLocalMaterialization(pcs.Name) if err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index ad8ebafa366b..6aa432b89298 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -43,7 +43,6 @@ func ResolveSelectedWorkload( if err := allErrs.ToAggregate(); err != nil { return nil, err } - component := components[len(components)-1] projections := make([]*ModelProjection, 0, len(components)) var snapshot NormalizedBuildSnapshot var pipeline Pipeline @@ -118,9 +117,13 @@ func ResolveSelectedWorkload( } projections = projected } - replicas := ptr.Deref(component.Replicas, 1) - if pipeline != PipelineLPX && replicas != 1 { - return nil, fmt.Errorf("%w: fixed engine replicas above one currently require a hybrid engine", ErrUnsupportedRuntime) + scalingGroupReplicas := int32(1) + if len(components) == 1 { + // A sole component can flatten its replica axis into the outer scaling group. + scalingGroupReplicas = ptr.Deref(components[0].Replicas, 1) + } + if pipeline != PipelineLPX && scalingGroupReplicas != 1 { + return nil, fmt.Errorf("%w: scaling-group replicas above one currently require a hybrid engine", ErrUnsupportedRuntime) } // Canonical roles expand into default or draft0..draft7 followed by target. @@ -129,9 +132,9 @@ func ResolveSelectedWorkload( return nil, err } return &SelectedWorkload{ - modelProjections: projections, - digest: digest, - engineReplicas: replicas, + modelProjections: projections, + digest: digest, + scalingGroupReplicas: scalingGroupReplicas, }, nil } diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index 132137c8bf98..dc263657049e 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -172,6 +172,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { ) plan, err := selected.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) + require.EqualValues(t, 1, plan.Replicas, "draft fanout must not become the shared scaling-group axis") require.Equal(t, "small", projections[0].stage) require.Equal(t, "lpx", projections[2].stage) require.Equal( diff --git a/deploy/operator/internal/dynamo/lpx/selected_workload.go b/deploy/operator/internal/dynamo/lpx/selected_workload.go index 67b86e5d5ddd..ac4091d14b8b 100644 --- a/deploy/operator/internal/dynamo/lpx/selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/selected_workload.go @@ -11,9 +11,9 @@ import "strings" // LPX scheduling and runtime materialization. // Its methods require a successfully resolved, non-nil workload with projections. type SelectedWorkload struct { - modelProjections []*ModelProjection - digest WorkloadDigest - engineReplicas int32 + modelProjections []*ModelProjection + digest WorkloadDigest + scalingGroupReplicas int32 } // ModelProjections returns a read-only view of the selected projection list. From f785f204d6c4a2d1d8875ee93c79cccc01f65150 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 9 Sep 2026 13:20:35 +0000 Subject: [PATCH 003/196] fix(operator): let LPX own PodCliqueSet identity Signed-off-by: Andrew Paprotsky (cherry picked from commit f5da9fc2663bc7946d720006a42c7d3344cac1ca) --- .../internal/controller/dgd_lpx_handoff.go | 4 +- .../dgd_lpx_handoff_envtest_test.go | 2 +- .../controller/dgd_lpx_handoff_test.go | 2 +- .../controller/lpx/attempt_deadline.go | 17 ++-- .../controller/lpx/attempt_deadline_test.go | 12 +-- .../internal/controller/lpx/controller.go | 4 +- .../controller/lpx/controller_test.go | 5 +- .../operator/internal/controller/lpx/grove.go | 2 +- .../internal/controller/lpx/lifecycle.go | 82 ++++++++++--------- .../internal/controller/lpx/lifecycle_test.go | 17 ++-- .../internal/controller/lpx/preflight.go | 2 +- .../internal/controller/lpx/render_test.go | 2 +- deploy/operator/internal/dynamo/graph_lpx.go | 4 +- 13 files changed, 80 insertions(+), 75 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index 88091723b643..20d74d055c6b 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -49,8 +49,7 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra return nil, err } if exists && child.Spec.InputRevision == revision && - child.Annotations[dynamo.LPXRestartAnnotation] == restart && - child.Annotations[dynamo.LPXPCSNameAnnotation] == dynamo.PCSNameForLPX(source) { + child.Annotations[dynamo.LPXRestartAnnotation] == restart { return child, nil } if !exists { @@ -62,7 +61,6 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra } } child.Spec.InputRevision = revision - metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXPCSNameAnnotation, dynamo.PCSNameForLPX(source)) metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXRestartAnnotation, restart) metav1.SetMetaDataAnnotation(&child.ObjectMeta, lpx.DGDGenerationAnnotation, strconv.FormatInt(source.Generation, 10)) if exists { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 9fbe9c1977d6..37f1d83a2bc2 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -250,7 +250,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") require.NoError(t, env.Client().Delete(t.Context(), quota)) - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: child.Annotations[dynamo.LPXPCSNameAnnotation], Namespace: source.Namespace}} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(source), Namespace: source.Namespace}} configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: source.Name + "-lpu", Namespace: source.Namespace}} service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: dynamo.GetDCDResourceName(source, "lpx", ""), Namespace: source.Namespace}} require.EventuallyWithT(t, func(c *assert.CollectT) { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index f91a6c9dc122..967dc47ad813 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -74,7 +74,7 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { require.True(t, exactLPXSourceOwner(child, source)) require.NotEmpty(t, child.Spec.InputRevision) require.NoError(t, dynamo.ValidateLPXSource(child, source)) - require.NotEqual(t, dynamo.PCSNameForDGD(source.Name, source.Spec.Components), child.Annotations[dynamo.LPXPCSNameAnnotation]) + require.NotContains(t, child.Annotations, "lpx.nvidia.com/podcliqueset-name") pending := ReconcileResult{State: v1beta1.DGDStatePending} projectLPXChildStatus(source, child, &pending, &v1beta1.DynamoGraphDeploymentStatus{}) require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, pending.ComponentStatus["lpx"].ComponentKind) diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index 6d29614d7d9d..c247adfdac59 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -104,6 +104,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( if attempt == nil { return nil, time.Time{}, nil, nil } + pcsName := dynamo.PCSNameForLPX(source) deadlineAt := time.Time{} if attempt.DeadlineAt != nil { deadlineAt = attempt.DeadlineAt.Time @@ -121,7 +122,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( } if attempt.ExceededAt != nil { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, observed, lpxSchedulingDeadlineExceededReason, + ctx, deployment, pcsName, attempt, observed, lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; no retry is authorized", ) return state, time.Time{}, nil, err @@ -142,7 +143,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( } if missing { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, observed, lpxAttemptAuthorityLostReason, + ctx, deployment, pcsName, attempt, observed, lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring the aggregate without replacement", ) return state, time.Time{}, nil, err @@ -211,7 +212,7 @@ func (r *graphReconciler) reconcileLPXAttemptPreparation( // revalidateLPXAttemptPublication fences live authority after preparation has // established the current nonnil attempt and an unpublished desired request. -func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment) error { +func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, pcsName string) error { attempt := currentLPXAttemptStatus(deployment) if attempt.ExceededAt != nil || attempt.DisarmedAt != nil { return fmt.Errorf("LPX publication is forbidden after a durable attempt decision") @@ -224,7 +225,7 @@ func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, d return fmt.Errorf("recorded LPX request authority changed before sibling publication") } pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs); err != nil { + if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { return err } if pcs.UID != attempt.PodCliqueSetUID || !pcs.DeletionTimestamp.IsZero() { @@ -382,6 +383,7 @@ func exactLPXAttemptState( func (r *graphReconciler) continueTerminalLPXAttempt( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, attempt *nvidiacomv1beta1.LPXAttemptStatus, observed map[string]*lpxv1alpha1.LPUPipelineRequest, reason string, @@ -409,14 +411,14 @@ func (r *graphReconciler) continueTerminalLPXAttempt( } } if pending { - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, attempt.PodCliqueSetUID); err != nil { + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID); err != nil { return nil, err } return newLPXDeadlineTransition( attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, lpxRetirementRequeueAfter, ), nil } - pending, err := r.retireLPXAttemptPodCliqueSet(ctx, deployment, attempt.PodCliqueSetUID) + pending, err := r.retireLPXAttemptPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID) if err != nil { return nil, err } @@ -432,10 +434,11 @@ func (r *graphReconciler) continueTerminalLPXAttempt( func (r *graphReconciler) retireLPXAttemptPodCliqueSet( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, uid types.UID, ) (bool, error) { pcs := &grovev1alpha1.PodCliqueSet{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName} if err := r.apiReader.Get(ctx, key, pcs); err != nil { return false, client.IgnoreNotFound(err) } diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 696a2cd778c8..382ec4b876c0 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -125,10 +125,10 @@ func TestLPXAttemptPartialPublicationLossFencesEverySibling(t *testing.T) { require.Len(t, requests, 2, "preparation plus one persisted transition per successful create") dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Second))) - require.ErrorContains(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd), "deadline") + require.ErrorContains(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd, desired.plan.PodCliqueSetName), "deadline") dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Minute))) require.NoError(t, reconciler.Delete(ctx, &requests[0])) - require.Error(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd)) + require.Error(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd, desired.plan.PodCliqueSetName)) classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) require.NoError(t, err) require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) @@ -149,7 +149,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { source.Spec.Scheduling = deadlineTestScheduling() request := deadlineTestRequest(dgd, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} - pcs := deadlineTestPCS(dgd, "pcs-uid") + pcs := deadlineTestPCS(dgd, source, "pcs-uid") attempt := deadlineTestAttempt(dgd, request, time.Now().Add(time.Minute)) attempt.DeadlineAt, attempt.PodCliqueSetUID = nil, "" dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} @@ -214,7 +214,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { replacement.UID = "replacement-uid" replacement.ResourceVersion = "2" replacement.Annotations[lpxPCSUIDAnnotation] = "replacement-pcs-uid" - pcs = deadlineTestPCS(dgd, "replacement-pcs-uid") + pcs = deadlineTestPCS(dgd, source, "replacement-pcs-uid") reconciler = newLPXTestReconciler(t, nil, dgd, source, replacement, pcs) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) @@ -281,10 +281,10 @@ func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, return request } -func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { +func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { return &grovev1alpha1.PodCliqueSet{ ObjectMeta: metav1.ObjectMeta{ - Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation], Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", + Name: dynamo.PCSNameForLPX(source), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 8629820d9ee4..a51b6775ef22 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -224,7 +224,7 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs); err != nil { + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { if !apierrors.IsNotFound(err) { return state, ctrl.Result{}, err } @@ -351,7 +351,7 @@ func (r *graphReconciler) validateLPXDeploymentAuthority(ctx context.Context, de if current.UID != deployment.UID || current.Generation != deployment.Generation || !apiequality.Semantic.DeepEqual(current.Spec, deployment.Spec) || !apiequality.Semantic.DeepEqual(current.OwnerReferences, deployment.OwnerReferences) || !current.DeletionTimestamp.IsZero() { return fmt.Errorf("LPXGraphDeployment authority changed before LPX resource mutation") } - for _, key := range []string{dynamo.LPXPCSNameAnnotation, dynamo.LPXRestartAnnotation, lpx.DGDGenerationAnnotation} { + for _, key := range []string{dynamo.LPXRestartAnnotation, lpx.DGDGenerationAnnotation} { if current.Annotations[key] != deployment.Annotations[key] { return fmt.Errorf("LPXGraphDeployment publication metadata changed before LPX resource mutation") } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index ee1c31efec64..f1bc52f493fa 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -167,7 +167,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { _, err := r.Reconcile(t.Context(), request) require.NoError(t, err) pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: child.Annotations[dynamo.LPXPCSNameAnnotation]} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(source)} require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) pcs.UID = "staged-pcs" require.NoError(t, r.Update(t.Context(), pcs)) @@ -198,7 +198,6 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { source.Spec.ProviderOverride = &v1beta1.ProviderOverride{Target: "PodCliqueSet"} case "name budget": component.ComponentName = "serving-engines" - child.Annotations[dynamo.LPXPCSNameAnnotation] = dynamo.PCSNameForLPX(source) case "selected workload": component.Replicas = ptr.To(int32(2)) case "render": @@ -271,7 +270,7 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T }} foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" - foreignPCS.Name = child.Annotations[dynamo.LPXPCSNameAnnotation] + foreignPCS.Name = dynamo.PCSNameForLPX(source) foreignEndpoint.Name = dynamo.GetDCDResourceName(source, "lpx", "") foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences if scenario == "pending finalizer" { diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index 9732740c0965..c35fe483bb5a 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -43,7 +43,7 @@ func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( ) (*grovev1alpha1.PodCliqueSet, bool, error) { // A missing observation permits only creation after the publication fence. if existingPodCliqueSet == nil { - if err := r.fenceLPXPublicationBeforeGroveSpecWrite(ctx, deployment); err != nil { + if err := r.fenceLPXPublicationBeforeGroveSpecWrite(ctx, deployment, desired.Name); err != nil { return nil, false, err } modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired) diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index e70617757457..edfe5054557e 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -318,7 +318,7 @@ func (r *graphReconciler) reconcileLPXKnownIntentFence( if selected { reason = "The DGD incarnation, generation, model, or selected input changed" } - retiring, retireErr := r.retireLPXRequest(ctx, deployment, request, reason) + retiring, retireErr := r.retireLPXRequest(ctx, deployment, "", request, reason) return retiring, nil, retireErr } return nil, requests, nil @@ -337,7 +337,7 @@ func (r *graphReconciler) prepareLPXMaterializing( return nil, &lpxRejected{reason: err.Error()}, nil } projections := workload.ModelProjections() - plan, err := workload.PlanNodeLocalMaterialization(deployment.Annotations[dynamo.LPXPCSNameAnnotation]) + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) if err != nil { return nil, &lpxRejected{reason: err.Error()}, nil } @@ -384,7 +384,7 @@ func (r *graphReconciler) retireInvalidLPXWorkload( return nil, err } if len(requests) > 0 { - return r.retireLPXRequest(ctx, deployment, &requests[0], reason) + return r.retireLPXRequest(ctx, deployment, "", &requests[0], reason) } // Source edits can rename both resources; discover only this exact child's old objects. @@ -523,7 +523,7 @@ func (r *graphReconciler) reconcileLPXAttemptFence( return nil, nil, err } if stale != nil { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, stale, "The immutable build snapshot or complete attempt identity changed") + retiring, retireErr := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, stale, "The immutable build snapshot or complete attempt identity changed") return nil, retiring, retireErr } // A deterministic-name collision must never be adopted or deleted. @@ -613,18 +613,20 @@ func (r *graphReconciler) listOwnedLPXRequestsIfAvailable( func (r *graphReconciler) retireFirstOwnedLPXRequest( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, reason string, ) (*lpxRetiring, error) { requests, err := r.listOwnedLPXRequests(ctx, deployment) if err != nil || len(requests) == 0 { return nil, err } - return r.retireLPXRequest(ctx, deployment, &requests[0], reason) + return r.retireLPXRequest(ctx, deployment, pcsName, &requests[0], reason) } func (r *graphReconciler) retireLPXRequest( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, request *lpxv1alpha1.LPUPipelineRequest, reason string, ) (*lpxRetiring, error) { @@ -678,7 +680,7 @@ func (r *graphReconciler) retireLPXRequest( } // Hold the shared Grove attempt at zero only after every publication is retiring. - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, ""); err != nil { + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, pcsName, ""); err != nil { return nil, fmt.Errorf("scale down the LPX PodCliqueSet after retiring all attempt requests: %w", err) } return &lpxRetiring{ @@ -694,30 +696,36 @@ func (r *graphReconciler) retireLPXRequest( func (r *graphReconciler) scaleDownLPXPodCliqueSet( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, pcsUID types.UID, ) error { - key := types.NamespacedName{ - Namespace: deployment.Namespace, - Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation], - } - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, key, pcs); err != nil { - if apierrors.IsNotFound(err) { + scale := func(pcs *grovev1alpha1.PodCliqueSet) error { + if pcsUID != "" && pcs.UID != pcsUID { return nil } - return err - } - if pcsUID != "" && pcs.UID != pcsUID { - return nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return fmt.Errorf("refusing to scale down PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + if !metav1.IsControlledBy(pcs, deployment) { + return fmt.Errorf("refusing to scale down PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + } + if !pcs.DeletionTimestamp.IsZero() || pcs.Spec.Replicas == 0 { + return nil + } + pcs.Spec.Replicas = 0 + return r.Update(ctx, pcs) } - if !pcs.DeletionTimestamp.IsZero() || pcs.Spec.Replicas == 0 { - return nil + if pcsName != "" { + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { + return client.IgnoreNotFound(err) + } + return scale(pcs) } - pcs.Spec.Replicas = 0 - return r.Update(ctx, pcs) + return visitLifecycleObjectPages(ctx, r.apiReader, &grovev1alpha1.PodCliqueSetList{}, func(object k8sruntime.Object) error { + pcs := object.(*grovev1alpha1.PodCliqueSet) + if !metav1.IsControlledBy(pcs, deployment) { + return nil + } + return scale(pcs) + }, client.InNamespace(deployment.Namespace)) } func (r *graphReconciler) reconcileSelectedLPX( @@ -778,6 +786,7 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return r.retireLPXRequest( ctx, deployment, + desired.plan.PodCliqueSetName, live, fmt.Sprintf("LPX request exceeds the scheduler size budget: %v", sizeErr), ) @@ -786,6 +795,7 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return r.retireLPXRequest( ctx, deployment, + desired.plan.PodCliqueSetName, live, "Waiting for the current LPX request finalizer to finish cleanup", ) @@ -838,7 +848,7 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( continue } if attemptProjection != nil { - if err := r.revalidateLPXAttemptPublication(ctx, deployment); err != nil { + if err := r.revalidateLPXAttemptPublication(ctx, deployment, desired.plan.PodCliqueSetName); err != nil { return nil, err } } @@ -903,7 +913,7 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( firstName = name } } - retiring, err := r.retireLPXRequest(ctx, deployment, currents[firstName], reason) + retiring, err := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, currents[firstName], reason) return nil, nil, retiring, err } @@ -947,11 +957,11 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( if pending != nil && pending.incomplete != "" { reason = fmt.Sprintf("%s: %s", reason, pending.incomplete) } - retiring, err := r.retireLPXRequest(ctx, deployment, live, reason) + retiring, err := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, live, reason) return nil, nil, retiring, err } if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, live, err.Error()) + retiring, retireErr := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, live, err.Error()) return nil, nil, retiring, retireErr } } @@ -984,11 +994,12 @@ func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassifica func (r *graphReconciler) fenceLPXPublicationBeforeGroveSpecWrite( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, ) error { if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { return err } - retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") + retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, pcsName, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") if err == nil && retiring != nil { err = retiring } @@ -1014,7 +1025,7 @@ func (r *graphReconciler) recreateStaleLPXPodCliqueSet( desired *lpxMaterializing, ) error { // Retire all exactly-owned requests before changing shared Grove state. - retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, "LPX publication was retired before recreating its Grove PodCliqueSet") + retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, desiredPCS.Name, "LPX publication was retired before recreating its Grove PodCliqueSet") if err != nil { return err } @@ -1022,10 +1033,7 @@ func (r *graphReconciler) recreateStaleLPXPodCliqueSet( return retiring } - key := types.NamespacedName{ - Namespace: deployment.Namespace, - Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation], - } + key := types.NamespacedName{Namespace: deployment.Namespace, Name: desiredPCS.Name} live := &grovev1alpha1.PodCliqueSet{} if err := r.apiReader.Get(ctx, key, live); err != nil { if apierrors.IsNotFound(err) { @@ -1048,7 +1056,7 @@ func (r *graphReconciler) recreateStaleLPXPodCliqueSet( return fmt.Errorf("%w is deleted", errLPXGrovePodCliqueSetRecreating) } if live.Spec.Replicas != 0 { - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, ""); err != nil { + if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, live.Name, ""); err != nil { return err } return fmt.Errorf("%w is scaled to zero", errLPXGrovePodCliqueSetRecreating) @@ -1291,7 +1299,7 @@ func (r *graphReconciler) observeLPXPodCliqueSet( noCurrentRequests bool, ) (*grovev1alpha1.PodCliqueSet, string, error) { pcs := &grovev1alpha1.PodCliqueSet{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Annotations[dynamo.LPXPCSNameAnnotation]} + key := types.NamespacedName{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName} if err := r.apiReader.Get(ctx, key, pcs); err != nil { if apierrors.IsNotFound(err) { return nil, "Waiting for the selected Grove PodCliqueSet", nil @@ -1591,7 +1599,7 @@ func (r *graphReconciler) finalizeLPXRequests( } // Start or continue retirement from the authoritative first request. - if _, err := r.retireLPXRequest(ctx, deployment, &requests[0], "LPXGraphDeployment is being deleted"); err != nil { + if _, err := r.retireLPXRequest(ctx, deployment, "", &requests[0], "LPXGraphDeployment is being deleted"); err != nil { return err } return fmt.Errorf("waiting for LPX request %q to finish fail-closed retirement", requests[0].Name) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index e63d27d7e218..3b0bf8f05036 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -416,7 +416,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { requests[index].Finalizers = []string{"scheduling.lpu.nvidia.com/test-cleanup"} require.NoError(t, reconciler.Update(ctx, &requests[index])) } - _, err = reconciler.retireLPXRequest(ctx, dgd, &requests[0], "test group retirement") + _, err = reconciler.retireLPXRequest(ctx, dgd, desired.plan.PodCliqueSetName, &requests[0], "test group retirement") require.NoError(t, err) requests, err = reconciler.listOwnedLPXRequests(ctx, dgd) require.NoError(t, err) @@ -427,7 +427,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { pcs := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, types.NamespacedName{ Namespace: dgd.Namespace, - Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation], + Name: desired.plan.PodCliqueSetName, }, pcs)) require.Zero(t, pcs.Spec.Replicas) @@ -517,7 +517,7 @@ func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { require.IsType(t, &lpxOpen{}, classification) requestBefore := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) key := client.ObjectKeyFromObject(dgd) - pcsKey := client.ObjectKey{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]} + pcsKey := client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} pcsBefore := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, pcsBefore)) @@ -857,7 +857,7 @@ func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) - pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]} + pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} drifted := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) require.NotEmpty(t, drifted.Spec.Template.Cliques) @@ -970,7 +970,7 @@ func TestGroveSpecSyncRecreatesStaleLPXAttemptBeforeSuccessor(t *testing.T) { pcs := &grovev1alpha1.PodCliqueSet{} pcsKey := types.NamespacedName{ Namespace: successorDGD.Namespace, - Name: successorDGD.Annotations[dynamo.LPXPCSNameAnnotation], + Name: successorDesired.plan.PodCliqueSetName, } require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) oldCliqueCount := len(pcs.Spec.Template.Cliques) @@ -1152,7 +1152,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { require.False(t, request.DeletionTimestamp.IsZero()) require.Equal(t, []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"}, request.Finalizers) pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: dgd.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs)) + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) require.Zero(t, pcs.Spec.Replicas, "child finalization must stop the workload before scheduler cleanup finishes") t.Log("Keep the child finalizer until the scheduler releases the exact request") @@ -1547,7 +1547,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { request := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) require.False(t, request.DeletionTimestamp.IsZero()) pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: child.Annotations[dynamo.LPXPCSNameAnnotation]}, pcs)) + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(source)}, pcs)) require.Zero(t, pcs.Spec.Replicas) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.ErrorContains(t, err, "waiting for LPX request") @@ -2113,7 +2113,7 @@ func newLPXTestDeployment(t *testing.T, source *nvidiacomv1beta1.DynamoGraphDepl return &nvidiacomv1alpha1.LPXGraphDeployment{ TypeMeta: metav1.TypeMeta{APIVersion: nvidiacomv1alpha1.GroupVersion.String(), Kind: "LPXGraphDeployment"}, ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: types.UID("lpx-" + string(source.UID)), Generation: 1, - Annotations: map[string]string{dynamo.LPXPCSNameAnnotation: dynamo.PCSNameForLPX(source), lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, + Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, Finalizers: []string{lpxGraphDeploymentFinalizer}, }, @@ -2285,7 +2285,6 @@ func newLPXTestReconciler( revision, err := dynamo.LPXInputRevision(source, "") require.NoError(t, err) dgd.Spec.InputRevision = revision - dgd.Annotations[dynamo.LPXPCSNameAnnotation] = dynamo.PCSNameForLPX(source) if dgd.ResourceVersion == "" { dgd.ResourceVersion = "1" } diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index 4d6f425a57d3..8aa9a8bfb9c7 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -103,7 +103,7 @@ func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( return nil, nil, fmt.Errorf("failed to list LPX requests after workload resolution: %w", listErr) } if len(requests) > 0 { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, &requests[0], rejected.reason) + retiring, retireErr := r.retireLPXRequest(ctx, deployment, dynamo.PCSNameForLPX(source), &requests[0], rejected.reason) if retireErr != nil { return nil, nil, fmt.Errorf("failed to retire LPX attempt after workload resolution: %w", retireErr) } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 28f0cca02ec5..2659cbd8f66c 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -45,7 +45,7 @@ func newLPXRenderDeployment(t *testing.T, source *v1beta1.DynamoGraphDeployment) require.NoError(t, err) return &v1alpha1.LPXGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: "lpx-render-uid", Generation: 1, - Annotations: map[string]string{dynamo.LPXPCSNameAnnotation: dynamo.PCSNameForLPX(source), lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, + Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, }, Spec: v1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index d840be168a04..e31880d47a6e 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -89,7 +89,6 @@ const ( LPXDeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" LPXDeploymentGenerationAnnotation = "lpx.nvidia.com/deployment-generation" LPXInputRevisionAnnotation = "lpx.nvidia.com/input-revision" - LPXPCSNameAnnotation = "lpx.nvidia.com/podcliqueset-name" LPXRestartAnnotation = "lpx.nvidia.com/restart-id" LPXServingLabel = "lpx.nvidia.com/serving" lpxGPUExecutionRole = "gpu" @@ -133,8 +132,7 @@ func ValidateLPXSource(deployment *v1alpha1.LPXGraphDeployment, source *v1beta1. if err != nil { return err } - if deployment.Spec.InputRevision != revision || deployment.Annotations[LPXRestartAnnotation] != restart || - deployment.Annotations[LPXPCSNameAnnotation] != PCSNameForLPX(source) { + if deployment.Spec.InputRevision != revision || deployment.Annotations[LPXRestartAnnotation] != restart { return fmt.Errorf("LPXGraphDeployment is waiting for the current source input revision") } return nil From ff93d1c0f39c4e283620fee679e9fb333ddd6157 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 9 Sep 2026 14:06:08 +0000 Subject: [PATCH 004/196] fix(operator): use conductor image for SSH init Signed-off-by: Andrew Paprotsky (cherry picked from commit dbdd330feec73058704974646544d995134864f8) --- .../internal/dynamo/lpx/pod_roles_test.go | 19 +++++++++++++++++++ .../internal/dynamo/lpx/runtime_agent.go | 2 +- .../internal/dynamo/lpx/runtime_ssh.go | 7 +++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 543e842ccf2e..af87d310ae25 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -314,6 +314,25 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { require.Contains(t, testVolumeMountPaths(initContainer.VolumeMounts), "/sys") } +func TestConfigureNodeLocalXTConductorSSHInitUsesMainImage(t *testing.T) { + t.Log("Place an unrelated sidecar before the XT conductor main container") + podSpec := corev1.PodSpec{Containers: []corev1.Container{ + {Name: "metrics", Image: "metrics-sidecar", ImagePullPolicy: corev1.PullAlways}, + {Name: commonconsts.MainContainerName, Image: "lpu-runtime", ImagePullPolicy: corev1.PullIfNotPresent}, + }} + + t.Log("Configure the conductor without deriving its runtime identity from list position") + require.NoError(t, configureNodeLocalConductorRuntime(&podSpec, BuildFamilyXT, "lpu-wkr-m-0", "ssh-secret")) + + t.Log("Run SSH initialization with the resolved conductor image and preserve the sidecar") + require.Equal(t, "metrics", podSpec.Containers[0].Name) + require.Equal(t, "metrics-sidecar", podSpec.Containers[0].Image) + require.Equal(t, "conductor", podSpec.Containers[1].Name) + require.Len(t, podSpec.InitContainers, 1) + require.Equal(t, "lpu-runtime", podSpec.InitContainers[0].Image) + require.Equal(t, corev1.PullIfNotPresent, podSpec.InitContainers[0].ImagePullPolicy) +} + func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T) { t.Log("Define custom entrypoint and probe preservation cases") customStartup := testExecProbe("custom-startup") diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 6044e9e76ddd..ba93ed990968 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -209,7 +209,7 @@ func configureNodeLocalConductorRuntime( if isXT { sshVolumeName = sshSecretName } - addConductorSSHKey(conductorPodSpec, sshSecretName, sshVolumeName) + addConductorSSHKey(conductorPodSpec, conductor, sshSecretName, sshVolumeName) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index 580a0bcab0b3..37140327dd3f 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -42,7 +42,7 @@ func sshVolumeMount(volumeName string) corev1.VolumeMount { } } -func addConductorSSHKey(podSpec *corev1.PodSpec, secretName, volumeName string) { +func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName, volumeName string) { // Mount the source Secret and the writable destination used by OpenMPI. addSSHVolumeWithDefaultMode(podSpec, secretName, volumeName, 0600) podSpec.Volumes = setVolumeByName(podSpec.Volumes, corev1.Volume{ @@ -53,11 +53,10 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, secretName, volumeName string) }) // Copy the key with OpenSSH's required mode before the conductor starts. - mainContainer := podSpec.Containers[0] initContainer := corev1.Container{ Name: conductorSSHKeyInitContainerName, - Image: mainContainer.Image, - ImagePullPolicy: mainContainer.ImagePullPolicy, + Image: conductor.Image, + ImagePullPolicy: conductor.ImagePullPolicy, Command: []string{"/bin/bash"}, Args: []string{"-c", conductorSSHKeyInitCommand}, SecurityContext: &corev1.SecurityContext{ From 5b2d46a1fc4a9920cf0ccd393816e937c944e356 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:28:43 +0300 Subject: [PATCH 005/196] fix(operator): preserve DGD placement status (#424) Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 10 ++ .../dynamographdeployment_conversion.go | 86 +++++++++++ ...amographdeployment_conversion_bugs_test.go | 28 ++++ .../dynamographdeployment_conversion_test.go | 4 + .../v1alpha1/dynamographdeployment_types.go | 57 +++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 77 +++++++++ deploy/operator/api/v1beta1/common.go | 8 +- .../v1beta1/dynamographdeployment_types.go | 5 + .../api/v1beta1/zz_generated.deepcopy.go | 5 + deploy/operator/cmd/crd-apply/main_test.go | 17 +- .../nvidia.com_dynamographdeployments.yaml | 146 ++++++++++++++++++ .../internal/controller/dgd_lpx_handoff.go | 8 + .../controller/dgd_lpx_handoff_test.go | 30 ++++ .../additional-resources/api-reference-k8s.md | 92 ++++++++++- .../dynamo-graph-deployment.mdx | 4 + .../kubernetes-api/full-api-reference.mdx | 144 ++++++++++++++--- 16 files changed, 688 insertions(+), 33 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 1c5faa52b50a..c6eda7f7503c 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -206,6 +206,16 @@ DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.uid DynamoGraphDeploymentStatus.lpx.placement.score DynamoGraphDeploymentStatus.lpx.placement.state DynamoGraphDeploymentStatus.observedGeneration +DynamoGraphDeploymentStatus.placement.lpxAttempt.deadlineAt +DynamoGraphDeploymentStatus.placement.lpxAttempt.disarmedAt +DynamoGraphDeploymentStatus.placement.lpxAttempt.exceededAt +DynamoGraphDeploymentStatus.placement.lpxAttempt.observedGeneration +DynamoGraphDeploymentStatus.placement.lpxAttempt.podCliqueSetUID +DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.attemptDigest +DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.name +DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.uid +DynamoGraphDeploymentStatus.placement.score +DynamoGraphDeploymentStatus.placement.state DynamoGraphDeploymentStatus.restart.inProgress DynamoGraphDeploymentStatus.restart.observedID DynamoGraphDeploymentStatus.restart.phase diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 6f533cb244f1..2452284a9606 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -654,6 +654,12 @@ func ConvertToSpecTopologyConstraint(src *v1beta1.SpecTopologyConstraint, dst *S func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, dst *v1beta1.DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = v1beta1.DGDState(src.State) + if src.Placement != nil { + dst.Placement = &v1beta1.PlacementStatus{} + ConvertFromPlacementStatus(src.Placement, dst.Placement) + } else { + dst.Placement = nil + } dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) @@ -692,6 +698,12 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStatus, dst *DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = DGDState(src.State) + if src.Placement != nil { + dst.Placement = &PlacementStatus{} + ConvertToPlacementStatus(src.Placement, dst.Placement) + } else { + dst.Placement = nil + } dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) @@ -725,6 +737,80 @@ func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStat } } +// ConvertFromPlacementStatus converts placement status from v1alpha1 to v1beta1. +func ConvertFromPlacementStatus(src *PlacementStatus, dst *v1beta1.PlacementStatus) { + *dst = v1beta1.PlacementStatus{State: v1beta1.PlacementScoreState(src.State)} + if src.Score != nil { + dst.Score = ptr.To(*src.Score) + } + if src.LPXAttempt != nil { + dst.LPXAttempt = &v1beta1.LPXAttemptStatus{} + ConvertFromLPXAttemptStatus(src.LPXAttempt, dst.LPXAttempt) + } +} + +// ConvertToPlacementStatus converts placement status from v1beta1 to v1alpha1. +func ConvertToPlacementStatus(src *v1beta1.PlacementStatus, dst *PlacementStatus) { + *dst = PlacementStatus{State: PlacementScoreState(src.State)} + if src.Score != nil { + dst.Score = ptr.To(*src.Score) + } + if src.LPXAttempt != nil { + dst.LPXAttempt = &LPXAttemptStatus{} + ConvertToLPXAttemptStatus(src.LPXAttempt, dst.LPXAttempt) + } +} + +// ConvertFromLPXAttemptStatus converts LPX attempt status from v1alpha1 to v1beta1. +func ConvertFromLPXAttemptStatus(src *LPXAttemptStatus, dst *v1beta1.LPXAttemptStatus) { + *dst = v1beta1.LPXAttemptStatus{ + ObservedGeneration: src.ObservedGeneration, + PodCliqueSetUID: src.PodCliqueSetUID, + Requests: make([]v1beta1.LPXAttemptRequestStatus, len(src.Requests)), + } + // Copy timestamp pointers before converting each request structurally. + dst.DeadlineAt = src.DeadlineAt.DeepCopy() + dst.ExceededAt = src.ExceededAt.DeepCopy() + dst.DisarmedAt = src.DisarmedAt.DeepCopy() + for i := range src.Requests { + ConvertFromLPXAttemptRequestStatus(&src.Requests[i], &dst.Requests[i]) + } +} + +// ConvertToLPXAttemptStatus converts LPX attempt status from v1beta1 to v1alpha1. +func ConvertToLPXAttemptStatus(src *v1beta1.LPXAttemptStatus, dst *LPXAttemptStatus) { + *dst = LPXAttemptStatus{ + ObservedGeneration: src.ObservedGeneration, + PodCliqueSetUID: src.PodCliqueSetUID, + Requests: make([]LPXAttemptRequestStatus, len(src.Requests)), + } + // Copy timestamp pointers before converting each request structurally. + dst.DeadlineAt = src.DeadlineAt.DeepCopy() + dst.ExceededAt = src.ExceededAt.DeepCopy() + dst.DisarmedAt = src.DisarmedAt.DeepCopy() + for i := range src.Requests { + ConvertToLPXAttemptRequestStatus(&src.Requests[i], &dst.Requests[i]) + } +} + +// ConvertFromLPXAttemptRequestStatus converts one request from v1alpha1 to v1beta1. +func ConvertFromLPXAttemptRequestStatus(src *LPXAttemptRequestStatus, dst *v1beta1.LPXAttemptRequestStatus) { + *dst = v1beta1.LPXAttemptRequestStatus{ + Name: src.Name, + AttemptDigest: src.AttemptDigest, + UID: src.UID, + } +} + +// ConvertToLPXAttemptRequestStatus converts one request from v1beta1 to v1alpha1. +func ConvertToLPXAttemptRequestStatus(src *v1beta1.LPXAttemptRequestStatus, dst *LPXAttemptRequestStatus) { + *dst = LPXAttemptRequestStatus{ + Name: src.Name, + AttemptDigest: src.AttemptDigest, + UID: src.UID, + } +} + // ConvertFromRestartStatus converts restart status from v1alpha1 to v1beta1. func ConvertFromRestartStatus(src *RestartStatus, dst *v1beta1.RestartStatus) { *dst = v1beta1.RestartStatus{ diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go index d83afecf0b65..94e1a1a3fb3e 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go @@ -24,10 +24,38 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" ) +func TestDGDPlacementStatusSurvivesConversionAlongsideLPX(t *testing.T) { + t.Log("Define distinct public and LPX-owned placement values on the hub") + src := &v1beta1.DynamoGraphDeployment{ + Status: v1beta1.DynamoGraphDeploymentStatus{ + Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, + LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ + Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStatePartial}, + }, + }, + } + + t.Log("Convert through the served alpha version and back") + alpha := &DynamoGraphDeployment{} + if err := alpha.ConvertFrom(src); err != nil { + t.Fatal(err) + } + got := &v1beta1.DynamoGraphDeployment{} + if err := alpha.ConvertTo(got); err != nil { + t.Fatal(err) + } + + t.Log("Preserve both paths independently without dropping or conflating the public field") + if diff := cmp.Diff(src.Status, got.Status); diff != "" { + t.Fatalf("status round-trip mismatch (-want +got):\n%s", diff) + } +} + func TestBugDGD_IntermediateHubAddsMainOnlyPodTemplateRoundTrips(t *testing.T) { alpha := &DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "main-only-pod-template", Namespace: "ns"}, diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index 2a5a8c9a1e6b..664e0aeda274 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -958,6 +958,10 @@ func TestDGD_RoundTrip_Status(t *testing.T) { Status: v1beta1.DynamoGraphDeploymentStatus{ ObservedGeneration: 7, State: v1beta1.DGDStateSuccessful, + Placement: &v1beta1.PlacementStatus{ + Score: ptr.To(0.92), + State: v1beta1.PlacementScoreStateReported, + }, LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ Placement: &v1beta1.PlacementStatus{ Score: ptr.To(0.87), diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index c49bcd08410b..fd8050b75259 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -22,6 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -55,6 +56,58 @@ const ( DGDStateFailed DGDState = "failed" ) +// PlacementScoreState describes whether placement score is available and how +// complete the reported score is for a graph deployment. See the v1beta1 +// PlacementScoreState for the authoritative semantics of each value. +// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;Unknown +type PlacementScoreState string + +const ( + PlacementScoreStateReported PlacementScoreState = "Reported" + PlacementScoreStatePartial PlacementScoreState = "Partial" + PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" + PlacementScoreStateUnknown PlacementScoreState = "Unknown" +) + +// PlacementStatus groups DGD-level scheduler placement fields under a single +// status object so future placement signals can be added without a schema +// break. See the v1beta1 PlacementStatus for the authoritative field docs. +type PlacementStatus struct { + // Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] + // where higher is better and 1.0 is the best possible placement. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + Score *float64 `json:"score,omitempty"` + // State indicates placement score reporting state. + // +optional + State PlacementScoreState `json:"state,omitempty"` + // LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + // +optional + LPXAttempt *LPXAttemptStatus `json:"lpxAttempt,omitempty"` +} + +// LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. +type LPXAttemptStatus struct { + ObservedGeneration int64 `json:"observedGeneration"` + PodCliqueSetUID types.UID `json:"podCliqueSetUID,omitempty"` + DeadlineAt *metav1.Time `json:"deadlineAt,omitempty"` + ExceededAt *metav1.Time `json:"exceededAt,omitempty"` + DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=9 + Requests []LPXAttemptRequestStatus `json:"requests"` +} + +// LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. +type LPXAttemptRequestStatus struct { + Name string `json:"name"` + AttemptDigest string `json:"attemptDigest"` + UID types.UID `json:"uid,omitempty"` +} + // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. // +kubebuilder:validation:XValidation:rule="oldSelf.hasValue() || !has(self.restart)",message="spec.restart must be unset on create; set spec.restart.id after creation to request a restart",optionalOldSelf=true type DynamoGraphDeploymentSpec struct { @@ -177,6 +230,10 @@ type DynamoGraphDeploymentStatus struct { // The map key is the service name from spec.services. // +optional Checkpoints map[string]ServiceCheckpointStatus `json:"checkpoints,omitempty"` + // Placement groups DGD-level scheduler placement signals (score, reporting + // state, and any future placement fields). + // +optional + Placement *PlacementStatus `json:"placement,omitempty"` // LPX contains the status of the graph's LPX workload, when present. // +optional LPX *v1beta1.DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 10736812e665..bc8d798ff6b5 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -962,6 +962,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementStatus) + (*in).DeepCopyInto(*out) + } if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(v1beta1.DynamoGraphDeploymentLPXStatus) @@ -1363,6 +1368,53 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXAttemptRequestStatus) DeepCopyInto(out *LPXAttemptRequestStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptRequestStatus. +func (in *LPXAttemptRequestStatus) DeepCopy() *LPXAttemptRequestStatus { + if in == nil { + return nil + } + out := new(LPXAttemptRequestStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXAttemptStatus) DeepCopyInto(out *LPXAttemptStatus) { + *out = *in + if in.DeadlineAt != nil { + in, out := &in.DeadlineAt, &out.DeadlineAt + *out = (*in).DeepCopy() + } + if in.ExceededAt != nil { + in, out := &in.ExceededAt, &out.ExceededAt + *out = (*in).DeepCopy() + } + if in.DisarmedAt != nil { + in, out := &in.DisarmedAt, &out.DisarmedAt + *out = (*in).DeepCopy() + } + if in.Requests != nil { + in, out := &in.Requests, &out.Requests + *out = make([]LPXAttemptRequestStatus, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptStatus. +func (in *LPXAttemptStatus) DeepCopy() *LPXAttemptStatus { + if in == nil { + return nil + } + out := new(LPXAttemptStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXGraphDeployment) DeepCopyInto(out *LPXGraphDeployment) { *out = *in @@ -1562,6 +1614,31 @@ func (in *PVC) DeepCopy() *PVC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { + *out = *in + if in.Score != nil { + in, out := &in.Score, &out.Score + *out = new(float64) + **out = **in + } + if in.LPXAttempt != nil { + in, out := &in.LPXAttempt, &out.LPXAttempt + *out = new(LPXAttemptStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. +func (in *PlacementStatus) DeepCopy() *PlacementStatus { + if in == nil { + return nil + } + out := new(PlacementStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProfilingConfigSpec) DeepCopyInto(out *ProfilingConfigSpec) { *out = *in diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index 6bdf8e9319ec..a797186ca80e 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -707,10 +707,10 @@ const ( // status object so future placement signals (e.g. scheduler contract version, // last-report timestamp, per-unit reports) can be added without a schema break. // -// The score source is an open question in DEP #10064 (Grove mirror, typed Grove -// scheduler API, or unstructured provider). Until a source is selected and -// implemented, the DGD controller does not write this field; the schema and -// conversion are landed here so downstream consumers can rely on the shape. +// The generic score source is an open question in DEP #10064 (Grove mirror, +// typed Grove scheduler API, or unstructured provider). Until one is selected, +// the DGD controller writes this field only as a compatibility projection of +// current LPX placement. type PlacementStatus struct { // score is the DGD-level scheduler placement score aggregated from // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 6dbd2204c80b..5ada213f9a08 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -133,6 +133,11 @@ type DynamoGraphDeploymentStatus struct { // +optional Checkpoints map[string]ComponentCheckpointStatus `json:"checkpoints,omitempty"` + // placement groups DGD-level scheduler placement signals (score, reporting + // state, and any future placement fields). + // +optional + Placement *PlacementStatus `json:"placement,omitempty"` + // lpx contains the status of the graph's LPX workload, when present. // +optional LPX *DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 295593c91078..81e3a38f05d9 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -898,6 +898,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementStatus) + (*in).DeepCopyInto(*out) + } if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(DynamoGraphDeploymentLPXStatus) diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index fe6c3fb65d0a..24c72fe53a4f 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -22,6 +22,8 @@ import ( ) func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { + const objectType = "object" + t.Log("Start a local API server with its default request and storage limits") env := &envtest.Environment{UseExistingCluster: ptr.To(false)} config, err := env.Start() @@ -74,7 +76,7 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { } } - t.Log("Verify installed schemas keep public LPX status grouped and private child status flat") + t.Log("Verify installed schemas preserve DGD placement, keep LPX status grouped, and keep private child status flat") client, err := apiextensionsclient.NewForConfig(config) if err != nil { t.Fatal(err) @@ -109,7 +111,7 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { templateField = "extraPodSpec" } template := version.Schema.OpenAPIV3Schema.Properties["spec"].Properties[templateField] - if template.Type != "object" || template.Description == "" { + if template.Type != objectType || template.Description == "" { t.Errorf("%s %s is missing the documented %s schema", name, version.Name, templateField) } } @@ -118,17 +120,18 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { for _, version := range crd.Spec.Versions { status := version.Schema.OpenAPIV3Schema.Properties["status"] if name == "dynamographdeployments.nvidia.com" { - for _, field := range []string{"modelDownload", "placement"} { - if _, exists := status.Properties[field]; exists { - t.Errorf("%s %s retains status.%s", name, version.Name, field) - } + if _, exists := status.Properties["modelDownload"]; exists { + t.Errorf("%s %s retains status.modelDownload", name, version.Name) + } + if status.Properties["placement"].Type != objectType { + t.Errorf("%s %s is missing the established status.placement projection", name, version.Name) } status = status.Properties["lpx"] } else if _, exists := status.Properties["lpx"]; exists { t.Errorf("%s %s unexpectedly groups private child status", name, version.Name) } for _, field := range []string{"modelDownload", "placement"} { - if status.Properties[field].Type != "object" { + if status.Properties[field].Type != objectType { t.Errorf("%s %s is missing the typed %s projection", name, version.Name, field) } } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index f1de4853be29..b88695862331 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -16067,6 +16067,76 @@ spec: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placement: + description: |- + Placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + lpxAttempt: + description: LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + properties: + deadlineAt: + format: date-time + type: string + disarmedAt: + format: date-time + type: string + exceededAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + podCliqueSetUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + requests: + items: + description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + properties: + attemptDigest: + type: string + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - attemptDigest + - name + type: object + maxItems: 9 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - observedGeneration + - requests + type: object + score: + description: |- + Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] + where higher is better and 1.0 is the best possible placement. + maximum: 1 + minimum: 0 + type: number + state: + description: State indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -25855,6 +25925,82 @@ spec: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placement: + description: |- + placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + lpxAttempt: + description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. + properties: + deadlineAt: + format: date-time + type: string + disarmedAt: + format: date-time + type: string + exceededAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + podCliqueSetUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + requests: + items: + description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + properties: + attemptDigest: + type: string + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - attemptDigest + - name + type: object + maxItems: 9 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - observedGeneration + - requests + type: object + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: restart contains the status of a graph-level restart. properties: diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index 20d74d055c6b..7d905565a493 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -12,6 +12,7 @@ import ( v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -164,11 +165,16 @@ func observeLPXRestart(ctx context.Context, reader client.Reader, source *v1beta // Its identity is already reconciled; results and conditions still require current generations. // child may be nil or deleting; all other pointer inputs must be non-nil. func projectLPXChildStatus(source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, result *ReconcileResult, status *v1beta1.DynamoGraphDeploymentStatus) { + previousLPX := status.LPX status.LPX = nil components := lpx.Components(source) if len(components) == 0 && child == nil { + if previousLPX != nil && apiequality.Semantic.DeepEqual(status.Placement, previousLPX.Placement) { + status.Placement = nil + } return } + status.Placement = nil if result.ComponentStatus == nil { result.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) } @@ -194,6 +200,8 @@ func projectLPXChildStatus(source *v1beta1.DynamoGraphDeployment, child *v1alpha if status.LPX.Placement != nil && status.LPX.Placement.LPXAttempt != nil { status.LPX.Placement.LPXAttempt.ObservedGeneration = source.Generation } + // Keep the established DGD-level field as a compatibility projection of LPX-owned placement. + status.Placement = status.LPX.Placement // Project only current authored members from the single observed child. for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 967dc47ad813..c199809fb9a4 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -123,6 +123,7 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Equal(t, v1beta1.DGDStateSuccessful, result.State) require.Len(t, result.ComponentStatus, 2) require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) + require.Equal(t, status.LPX.Placement, status.Placement) status.LPX.ModelDownload.Builds[0] = "parent-copy" status.LPX.Placement.LPXAttempt.Requests[0].UID = "parent-copy" require.Equal(t, "build", child.Status.ModelDownload.Builds[0]) @@ -145,8 +146,10 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Equal(t, v1beta1.DGDStatePending, pending.State, failure) if failure == "missing-child" || failure == "generation" { require.Nil(t, previous.LPX, failure) + require.Nil(t, previous.Placement, failure) } else { require.NotNil(t, previous.LPX, failure) + require.Equal(t, previous.LPX.Placement, previous.Placement, failure) } } @@ -161,6 +164,7 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Equal(t, v1beta1.DGDStatePending, result.State) require.Equal(t, Reason("LPXRetiring"), result.Reason) require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{}, status.LPX) + require.Nil(t, status.Placement) require.Equal(t, partial, result.ComponentStatus["lpx"]) source.Status.Components = result.ComponentStatus request := &v1beta1.DynamoGraphDeploymentRequest{} @@ -183,6 +187,7 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { source.Spec.Components = nil projectLPXChildStatus(source, child, &result, &status) require.Nil(t, status.LPX) + require.Nil(t, status.Placement) } func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { @@ -662,6 +667,7 @@ func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "download-status", Namespace: "default", Generation: 1}, Status: v1beta1.DynamoGraphDeploymentStatus{ State: v1beta1.DGDStateSuccessful, ObservedGeneration: 1, + Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}, LastCheckedAt: &checkedAt}, Placement: &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: 1}}, @@ -686,4 +692,28 @@ func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { updated := &v1beta1.DynamoGraphDeployment{} require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), updated)) require.Nil(t, updated.Status.LPX) + require.Equal(t, source.Status.Placement, updated.Status.Placement) +} + +func TestOrdinaryDGDClearsMirroredLPXPlacement(t *testing.T) { + t.Log("Seed an ordinary DGD with the same compatibility projection at both placement paths") + placement := &v1beta1.PlacementStatus{ + Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported, + LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: 1}, + } + source := &v1beta1.DynamoGraphDeployment{ + Status: v1beta1.DynamoGraphDeploymentStatus{ + Placement: placement.DeepCopy(), + LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{Placement: placement.DeepCopy()}, + }, + } + status := source.Status.DeepCopy() + result := &ReconcileResult{} + + t.Log("Project status after the obsolete LPX child is absent") + projectLPXChildStatus(source, nil, result, status) + + t.Log("Clear both the obsolete nested status and its matching top-level projection") + require.Nil(t, status.LPX) + require.Nil(t, status.Placement) } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 019e5cb90661..956dd78fdf57 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -641,6 +641,7 @@ _Appears in:_ | `services` _object (keys:string, values:[ServiceReplicaStatus](#servicereplicastatus))_ | Services contains per-service replica status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| +| `placement` _[PlacementStatus](#placementstatus)_ | Placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| | `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | LPX contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| @@ -982,6 +983,45 @@ _Appears in:_ | `preferredWeight` _float_ | PreferredWeight is required and used only when enforcement is
"preferred". Higher values create a stronger same-domain routing
preference, but do not guarantee same-domain selection. The value is not
a probability; worker selection still depends on load and other routing
inputs. A value of 0 disables the topology preference; 1 is the strongest
supported preference. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +#### LPXAttemptRequestStatus + + + +LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + + + +_Appears in:_ +- [LPXAttemptStatus](#lpxattemptstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | | | | +| `attemptDigest` _string_ | | | | +| `uid` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | + + +#### LPXAttemptStatus + + + +LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. + + + +_Appears in:_ +- [PlacementStatus](#placementstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | | | | +| `podCliqueSetUID` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | +| `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | +| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | MaxItems: 9
MinItems: 1
| + + #### ModelReference @@ -1055,6 +1095,48 @@ _Appears in:_ | `volumeAccessMode` _[PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#persistentvolumeaccessmode-v1-core)_ | VolumeAccessMode is the volume access mode of the PVC. Required when create is true. | | | +#### PlacementScoreState + +_Underlying type:_ _string_ + +PlacementScoreState describes whether placement score is available and how +complete the reported score is for a graph deployment. See the v1beta1 +PlacementScoreState for the authoritative semantics of each value. + +_Validation:_ +- Enum: [Reported Partial Unsupported Unknown] + +_Appears in:_ +- [PlacementStatus](#placementstatus) + +| Field | Description | +| --- | --- | +| `Reported` | | +| `Partial` | | +| `Unsupported` | | +| `Unknown` | | + + +#### PlacementStatus + + + +PlacementStatus groups DGD-level scheduler placement fields under a single +status object so future placement signals can be added without a schema +break. See the v1beta1 PlacementStatus for the authoritative field docs. + + + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `score` _float_ | Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0]
where higher is better and 1.0 is the best possible placement. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +| `state` _[PlacementScoreState](#placementscorestate)_ | State indicates placement score reporting state. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| +| `lpxAttempt` _[LPXAttemptStatus](#lpxattemptstatus)_ | LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. | | Optional: \{\}
| + + #### ProfilingConfigSpec @@ -2204,6 +2286,7 @@ _Appears in:_ | `components` _object (keys:string, values:[ComponentReplicaStatus](#componentreplicastatus))_ | components contains per-component replica status information, keyed by component name. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| +| `placement` _[PlacementStatus](#placementstatus)_ | placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| | `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | lpx contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| @@ -2741,15 +2824,16 @@ PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. -The score source is an open question in DEP #10064 (Grove mirror, typed Grove -scheduler API, or unstructured provider). Until a source is selected and -implemented, the DGD controller does not write this field; the schema and -conversion are landed here so downstream consumers can rely on the shape. +The generic score source is an open question in DEP #10064 (Grove mirror, +typed Grove scheduler API, or unstructured provider). Until one is selected, +the DGD controller writes this field only as a compatibility projection of +current LPX placement. _Appears in:_ - [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 92aac3752ed4..4296c3b5e292 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -168,6 +168,10 @@ The operator maintains observed state under `status`. Most recent `metadata.generation` observed by the controller. + + Public deployment-level placement summary. For LPX workloads, this field mirrors `status.lpx.placement` for compatibility with existing clients. + + LPX download and placement observations for the current configuration. Omitted when no current LPX result exists, including ordinary-only deployments. LPX component entries and aggregate readiness remain under `status.components`, `status.state`, and `status.conditions`. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 1ff0ab5512cf..ed83a723d65e 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,10 +10,10 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 3 Kubernetes API packages with 174 typed CRD and config sections. +Dynamo publishes 3 Kubernetes API packages with 178 typed CRD and config sections. -68 types +72 types 76 types 30 types @@ -1104,6 +1104,12 @@ See [ServiceCheckpointStatus](#servicecheckpointstatus). **Validation:** Optional: \{\} + +Placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). +See [PlacementStatus](#placementstatus). +**Validation:** Optional: \{\} + + LPX contains the status of the graph's LPX workload, when present. See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). @@ -1525,6 +1531,56 @@ PreferredWeight is required and used only when enforcement is "preferred". Highe + +LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. + + +**Kind:** `type` +**Appears in:** [LPXAttemptStatus](#lpxattemptstatus) + + + + + + + + +See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). + + + + +LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. + + +**Kind:** `type` +**Appears in:** [PlacementStatus](#placementstatus) + + + + + +See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). + + + +See [LPXAttemptRequestStatus](#lpxattemptrequeststatus). +**Validation:** MaxItems: 9 MinItems: 1 + + + ModelReference identifies a model served by this component @@ -1596,6 +1652,52 @@ See [PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/ + +PlacementScoreState describes whether placement score is available and how +complete the reported score is for a graph deployment. See the v1beta1 +PlacementScoreState for the authoritative semantics of each value. + + +**Kind:** `enum` +**Underlying type:** `string` +**Validation:** Enum: [Reported Partial Unsupported Unknown] +**Appears in:** [PlacementStatus](#placementstatus) + +**Allowed values** + +- Reported +- Partial +- Unsupported +- Unknown + + + +PlacementStatus groups DGD-level scheduler placement fields under a single +status object so future placement signals can be added without a schema +break. See the v1beta1 PlacementStatus for the authoritative field docs. + + +**Kind:** `type` +**Appears in:** [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + + +Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] where higher is better and 1.0 is the best possible placement. +**Validation:** Maximum: 1 Minimum: 0 Optional: \{\} + + + +State indicates placement score reporting state. +See [PlacementScoreState](#placementscorestate). +**Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} + + + +LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. +See [LPXAttemptStatus](#lpxattemptstatus). +**Validation:** Optional: \{\} + + + ProfilingConfigSpec defines configuration for the profiling process. This structure maps directly to the profile_sla.py config format. @@ -2936,7 +3038,7 @@ See [ModelDownloadStatus](#modeldownloadstatus). placement contains LPX scheduling progress and placement signals. -See [PlacementStatus](#placementstatus). +See [PlacementStatus](#v1beta1-placementstatus). **Validation:** Optional: \{\} @@ -3307,6 +3409,12 @@ See [ComponentCheckpointStatus](#componentcheckpointstatus). **Validation:** Optional: \{\} + +placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). +See [PlacementStatus](#v1beta1-placementstatus). +**Validation:** Optional: \{\} + + lpx contains the status of the graph's LPX workload, when present. See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). @@ -3665,12 +3773,12 @@ preferredWeight is required and used only when enforcement is "preferred". Highe - + LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. **Kind:** `type` -**Appears in:** [LPXAttemptStatus](#lpxattemptstatus) +**Appears in:** [LPXAttemptStatus](#v1beta1-lpxattemptstatus) @@ -3683,12 +3791,12 @@ See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#u - + LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. **Kind:** `type` -**Appears in:** [PlacementStatus](#placementstatus) +**Appears in:** [PlacementStatus](#v1beta1-placementstatus) @@ -3710,7 +3818,7 @@ See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/# -See [LPXAttemptRequestStatus](#lpxattemptrequeststatus). +See [LPXAttemptRequestStatus](#v1beta1-lpxattemptrequeststatus). **Validation:** MaxItems: 9 MinItems: 1 @@ -3877,7 +3985,7 @@ See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api - + PlacementScoreState describes whether placement score is available and how complete the reported score is for a graph deployment. @@ -3893,7 +4001,7 @@ Every backend must set this field after the first reconciliation: **Kind:** `enum` **Underlying type:** `string` **Validation:** Enum: [Reported Partial Unsupported Unknown] -**Appears in:** [PlacementStatus](#placementstatus) +**Appears in:** [PlacementStatus](#v1beta1-placementstatus) **Allowed values** @@ -3903,19 +4011,19 @@ Every backend must set this field after the first reconciliation: - Unknown - + PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. -The score source is an open question in DEP #10064 (Grove mirror, typed Grove -scheduler API, or unstructured provider). Until a source is selected and -implemented, the DGD controller does not write this field; the schema and -conversion are landed here so downstream consumers can rely on the shape. +The generic score source is an open question in DEP #10064 (Grove mirror, +typed Grove scheduler API, or unstructured provider). Until one is selected, +the DGD controller writes this field only as a compatibility projection of +current LPX placement. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) +**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus), [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) score is the DGD-level scheduler placement score aggregated from relevant scheduler placement units. Normalized to [0.0, 1.0] where higher is better and 1.0 represents the best possible placement. Aggregation uses the minimum across placement units so the value is a worst-placement signal for the graph. Scores are only comparable across DGDs that share the same scheduler scoring contract and version. @@ -3924,13 +4032,13 @@ score is the DGD-level scheduler placement score aggregated from relevant schedu state indicates placement score reporting state. See PlacementScoreState for the semantics of each value. -See [PlacementScoreState](#placementscorestate). +See [PlacementScoreState](#v1beta1-placementscorestate). **Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. -See [LPXAttemptStatus](#lpxattemptstatus). +See [LPXAttemptStatus](#v1beta1-lpxattemptstatus). **Validation:** Optional: \{\} From 9da41666bf7b57ca2de02080149620031332de1f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:48:23 +0300 Subject: [PATCH 006/196] fix(operator): ignore foreign LPX child during finalization (#425) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_lpx_handoff.go | 3 ++- deploy/operator/internal/controller/dgd_lpx_handoff_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index 7d905565a493..c75a67466499 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -93,7 +93,8 @@ func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGrap return client.IgnoreNotFound(err) } if !exactLPXSourceOwner(child, source) { - return fmt.Errorf("refusing to delete a foreign LPXGraphDeployment %q", child.Name) + // It is not our dependent; do not delete it and do not wait for it. + return nil } if err := r.deleteChild(ctx, child); err != nil { return err diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index c199809fb9a4..a9821be92874 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -93,11 +93,13 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { require.ErrorContains(t, err, "adoption is not supported", invalidIdentity) } - t.Log("Refuse adoption and deletion after the source name is reused with a different UID") + t.Log("Refuse adoption after the source name is reused with a different UID") source.UID = "replacement-source" _, err = handoff.Reconcile(t.Context(), source) require.ErrorContains(t, err, "adoption is not supported") - require.ErrorContains(t, handoff.Finalize(t.Context(), source), "foreign") + + t.Log("Do not block finalization on or delete the child owned by the prior source") + require.NoError(t, handoff.Finalize(t.Context(), source)) stored := &v1alpha1.LPXGraphDeployment{} require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) require.True(t, stored.DeletionTimestamp.IsZero()) From c60c3c454dd23d6ebd164c40034e5e2589c8aa45 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:26:49 +0300 Subject: [PATCH 007/196] refactor(operator): narrow the LPX Grove seam (#426) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_projection.go | 23 +++ .../controller/dgd_grove_lpx_restart.go | 77 ++++++++++ .../controller/dgd_grove_lpx_status.go | 102 +++++++++++++ .../internal/controller/dgd_grove_program.go | 22 ++- .../controller/dgd_grove_watch_setup.go | 4 +- .../controller/dgd_grove_workload_renderer.go | 27 ++-- .../dgd_grove_workloads_reconciler.go | 22 +-- .../dgd_grove_workloads_reconciler_test.go | 17 ++- .../internal/controller/dgd_lpx_handoff.go | 140 ------------------ .../controller/dgd_lpx_handoff_test.go | 60 +++++++- .../dgd_worker_rollout_reconciler_test.go | 2 +- .../controller/dgd_workload_program_test.go | 4 +- .../dynamographdeployment_controller_test.go | 7 +- .../operator/internal/controller/lpx/watch.go | 31 ---- .../internal/controller/upgrade_test.go | 4 +- deploy/operator/internal/dynamo/graph.go | 4 +- deploy/operator/internal/dynamo/graph_lpx.go | 2 +- deploy/operator/internal/dynamo/graph_test.go | 27 ---- .../internal/dynamo/lpx/grove_ownership.go | 54 +++++++ .../validation/dynamographdeployment.go | 2 +- .../validation/dynamographdeployment_lpx.go | 19 +++ 21 files changed, 394 insertions(+), 256 deletions(-) create mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_projection.go create mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_restart.go create mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_status.go create mode 100644 deploy/operator/internal/dynamo/lpx/grove_ownership.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeployment_lpx.go diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_projection.go b/deploy/operator/internal/controller/dgd_grove_lpx_projection.go new file mode 100644 index 000000000000..befb794b536a --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_lpx_projection.go @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "slices" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// projectOrdinaryGroveDeployment returns the component subset materialized by +// the DGD-owned Grove PodCliqueSet. The source is not mutated. +func projectOrdinaryGroveDeployment(source *v1beta1.DynamoGraphDeployment) *v1beta1.DynamoGraphDeployment { + ordinary := source.DeepCopy() + ordinary.Spec.Components = slices.DeleteFunc( + ordinary.Spec.Components, + func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.ManagedByExternalController() + }, + ) + return ordinary +} diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go new file mode 100644 index 000000000000..9111ec56deeb --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// resolveRestartProgress composes child-owned LPX and ordinary Grove observations. +func (p *groveProgram) resolveRestartProgress( + ctx context.Context, + source *v1beta1.DynamoGraphDeployment, + ordinaryDGD *v1beta1.DynamoGraphDeployment, + inProgress []string, +) []string { + ordinary := make([]string, 0, len(inProgress)) + pending := make(map[string]bool, len(inProgress)) + var childComponents map[string]v1beta1.ComponentReplicaStatus + childObserved := false + + for _, name := range inProgress { + component := source.GetComponentByName(name) + if component == nil { + continue + } + if !component.IsLPX() { + ordinary = append(ordinary, name) + continue + } + if !childObserved { + childComponents = observeLPXRestart(ctx, p.lpx.Client, source) + childObserved = true + } + if !childComponents[name].Ready { + pending[name] = true + } + } + + if len(ordinary) > 0 { + for _, name := range p.restartProgress.Resolve(ctx, ordinaryDGD, ordinary) { + pending[name] = true + } + } + + remaining := make([]string, 0, len(inProgress)) + for _, name := range inProgress { + if pending[name] { + remaining = append(remaining, name) + } + } + return remaining +} + +// observeLPXRestart returns component status from one current, ready restart observation. +// A failed read or incomplete child leaves every requested member pending. +func observeLPXRestart(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment) map[string]v1beta1.ComponentReplicaStatus { + child := &v1alpha1.LPXGraphDeployment{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || + child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || + dynamo.ValidateLPXSource(child, source) != nil || source.Spec.Restart == nil || + child.Annotations[dynamo.LPXRestartAnnotation] != source.Spec.Restart.ID { + return nil + } + ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { + return nil + } + return child.Status.Components +} diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go new file mode 100644 index 000000000000..52b7b9b1ae08 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// mergeLPXChildStatus composes a current LGD observation into the ordinary +// Grove result. The outer DGD controller remains the only status writer. +func mergeLPXChildStatus( + source *v1beta1.DynamoGraphDeployment, + child *v1alpha1.LPXGraphDeployment, + ordinary ReconcileResult, +) (ReconcileResult, *v1beta1.DynamoGraphDeploymentLPXStatus) { + components := lpx.Components(source) + if len(components) == 0 && child == nil { + return ordinary, nil + } + if ordinary.ComponentStatus == nil { + ordinary.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) + } + serving := lpx.ServingComponent(source) + for _, component := range components { + kind := v1beta1.ComponentKindPodCliqueScalingGroup + if component != serving { + kind = v1beta1.ComponentKindPodClique + } + ordinary.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ComponentKind: kind} + } + + current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() + observed := current && child.Status.ObservedGeneration == child.Generation + var status *v1beta1.DynamoGraphDeploymentLPXStatus + if observed { + status = &v1beta1.DynamoGraphDeploymentLPXStatus{ + ModelDownload: child.Status.ModelDownload.DeepCopy(), + Placement: child.Status.Placement.DeepCopy(), + } + if status.Placement != nil && status.Placement.LPXAttempt != nil { + status.Placement.LPXAttempt.ObservedGeneration = source.Generation + } + for _, component := range components { + if observedStatus, found := child.Status.Components[component.ComponentName]; found { + ordinary.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() + } + } + } + + if ordinary.State != v1beta1.DGDStateFailed && current { + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + if failed != nil && failed.Status == metav1.ConditionTrue && failed.ObservedGeneration == child.Generation { + ordinary.State = v1beta1.DGDStateFailed + ordinary.Reason, ordinary.Message = Reason(failed.Reason), Message(failed.Message) + return ordinary, status + } + } + if observed { + ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + if ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { + return ordinary, status + } + if ordinary.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { + ordinary.State = v1beta1.DGDStatePending + ordinary.Reason, ordinary.Message = Reason(ready.Reason), Message(ready.Message) + return ordinary, status + } + } + if ordinary.State != v1beta1.DGDStateFailed { + ordinary.State = v1beta1.DGDStatePending + ordinary.Reason = "LPXChildPending" + ordinary.Message = "Waiting for the current LPX engine revision and readiness" + } + return ordinary, status +} + +// lpxPlacementProjection retains an independent ordinary-DGD placement value, +// but mirrors current LPX placement and clears stale mirrors. +func lpxPlacementProjection( + source *v1beta1.DynamoGraphDeployment, + child *v1alpha1.LPXGraphDeployment, + current *v1beta1.PlacementStatus, + previousLPX *v1beta1.DynamoGraphDeploymentLPXStatus, + nextLPX *v1beta1.DynamoGraphDeploymentLPXStatus, +) *v1beta1.PlacementStatus { + if nextLPX != nil { + return nextLPX.Placement + } + if len(lpx.Components(source)) > 0 || child != nil { + return nil + } + if previousLPX != nil && apiequality.Semantic.DeepEqual(current, previousLPX.Placement) { + return nil + } + return current +} diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 77f0d1102805..6f383506da60 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -92,6 +92,7 @@ func (p *groveProgram) Reconcile( programResult.Fail(req.DGD.Generation, reasonSelectedWorkloadProviderUnavailable, err) return programResult, reconcile.TerminalError(err) } + var ordinaryDGD *nvidiacomv1beta1.DynamoGraphDeployment defer func() { if retErr != nil { @@ -101,7 +102,10 @@ func (p *groveProgram) Reconcile( } programResult.Fail(req.DGD.Generation, reason, retErr) } - p.topology.Reconcile(ctx, req.DGD, &programResult) + if ordinaryDGD == nil { + ordinaryDGD = projectOrdinaryGroveDeployment(req.DGD) + } + p.topology.Reconcile(ctx, ordinaryDGD, &programResult) }() log.FromContext(ctx).Info( "Reconciling Grove resources", @@ -119,13 +123,16 @@ func (p *groveProgram) Reconcile( if err != nil { return programResult, err } + ordinaryDGD = projectOrdinaryGroveDeployment(req.DGD) previousRestart := programResult.Status.Restart restart := p.restart.Resolve( ctx, req.DGD, &programResult.Status, - p.resolveRestartProgress, + func(ctx context.Context, source *nvidiacomv1beta1.DynamoGraphDeployment, inProgress []string) []string { + return p.resolveRestartProgress(ctx, source, ordinaryDGD, inProgress) + }, ) recordRestartTransition(previousRestart, restart.Status, &programResult) programResult.Status.Restart = restart.Status @@ -138,6 +145,7 @@ func (p *groveProgram) Reconcile( result, err := p.workloads.Reconcile( ctx, req.DGD, + ordinaryDGD, restart.State, checkpoints.Infos, ) @@ -153,7 +161,15 @@ func (p *groveProgram) Reconcile( if err != nil { return programResult, fmt.Errorf("reconcile LPX child: %w", err) } - projectLPXChildStatus(req.DGD, child, &result, &programResult.Status) + previousLPX := programResult.Status.LPX + result, programResult.Status.LPX = mergeLPXChildStatus(req.DGD, child, result) + programResult.Status.Placement = lpxPlacementProjection( + req.DGD, + child, + programResult.Status.Placement, + previousLPX, + programResult.Status.LPX, + ) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) if child != nil && !child.DeletionTimestamp.IsZero() { programResult.RequeueAfter = 5 * time.Second diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index 8b91cc9e3124..814ccba31409 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -21,8 +21,8 @@ import ( "context" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -105,7 +105,7 @@ func (s *groveWatchSetup) mapPodCliqueToRequests( ) []ctrl.Request { podClique, ok := obj.(*grovev1alpha1.PodClique) // LPX owns its role observations; the DGD watches the aggregate child status. - if !ok || podClique == nil || lpxcontroller.OwnsPodClique(ctx, s.reader, podClique) { + if !ok || podClique == nil || dynamolpx.OwnsPodClique(ctx, s.reader, podClique) { return nil } diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index 2ebe0c95fa08..90075338ff30 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -20,7 +20,6 @@ package controller import ( "context" "fmt" - "slices" "strings" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -72,12 +71,13 @@ func newGroveWorkloadRenderer( func (r *groveWorkloadRenderer) Render( ctx context.Context, - dgd *nvidiacomv1beta1.DynamoGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + ordinary *nvidiacomv1beta1.DynamoGraphDeployment, restartState *dynamo.RestartState, checkpointInfos map[string]*checkpoint.CheckpointInfo, workerGenerationChanged bool, ) (*grovePodCliqueSetRender, error) { - if dgd == nil { + if source == nil || ordinary == nil { return nil, fmt.Errorf("cannot render Grove PodCliqueSet without a DynamoGraphDeployment") } if r.reader == nil { @@ -85,8 +85,8 @@ func (r *groveWorkloadRenderer) Render( } existingPodCliqueSet := &grovev1alpha1.PodCliqueSet{} key := types.NamespacedName{ - Name: dynamo.PCSNameForDGD(dgd.Name, dgd.Spec.Components), - Namespace: dgd.Namespace, + Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), + Namespace: ordinary.Namespace, } if err := r.reader.Get(ctx, key, existingPodCliqueSet); err != nil { if !apierrors.IsNotFound(err) { @@ -95,8 +95,8 @@ func (r *groveWorkloadRenderer) Render( existingPodCliqueSet = nil } - workerHashSuffixNeeded := shouldRenderGroveWorkerHashSuffix(dgd, existingPodCliqueSet, workerGenerationChanged) - renderDeployment, err := groveRenderDeployment(dgd, existingPodCliqueSet, workerHashSuffixNeeded) + workerHashSuffixNeeded := shouldRenderGroveWorkerHashSuffix(ordinary, existingPodCliqueSet, workerGenerationChanged) + renderDeployment, err := groveRenderDeployment(ordinary, source, existingPodCliqueSet, workerHashSuffixNeeded) if err != nil { return nil, err } @@ -131,18 +131,11 @@ func (r *groveWorkloadRenderer) Render( } func groveRenderDeployment( - dgd *nvidiacomv1beta1.DynamoGraphDeployment, + ordinary *nvidiacomv1beta1.DynamoGraphDeployment, + hashSource *nvidiacomv1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, workerHashSuffix bool, ) (*nvidiacomv1beta1.DynamoGraphDeployment, error) { - // Exclude externally managed templates before copying the ordinary render inputs. - ordinary := *dgd - externallyManaged := func(component nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) bool { - return component.ManagedByExternalController() - } - if slices.ContainsFunc(dgd.Spec.Components, externallyManaged) { - ordinary.Spec.Components = slices.DeleteFunc(slices.Clone(dgd.Spec.Components), externallyManaged) - } renderDeployment := ordinary.DeepCopy() // Compatibility and worker labels mutate only the independently owned copy. @@ -150,7 +143,7 @@ func groveRenderDeployment( if !workerHashSuffix { return renderDeployment, nil } - if err := applyGroveWorkerHashSuffix(renderDeployment, dgd); err != nil { + if err := applyGroveWorkerHashSuffix(renderDeployment, hashSource); err != nil { return nil, err } return renderDeployment, nil diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 9ce745f1da48..09f95481a684 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -72,19 +72,21 @@ func newGroveWorkloadsReconciler( func (r *groveWorkloadsReconciler) Reconcile( ctx context.Context, - dgd *nvidiacomv1beta1.DynamoGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + ordinary *nvidiacomv1beta1.DynamoGraphDeployment, restartState *dynamo.RestartState, checkpointInfos map[string]*checkpoint.CheckpointInfo, ) (ReconcileResult, error) { logger := log.FromContext(ctx) - workerHashTransition, err := r.rollout.planUnsupportedWorkerHashTransition(dgd) + workerHashTransition, err := r.rollout.planUnsupportedWorkerHashTransition(source) if err != nil { return ReconcileResult{}, failWorkloadProgram(reasonRollingUpdateFailed, err) } renderedPodCliqueSet, err := r.renderer.Render( ctx, - dgd, + source, + ordinary, restartState, checkpointInfos, workerHashTransition.hashChanged, @@ -95,7 +97,7 @@ func (r *groveWorkloadsReconciler) Reconcile( } if len(renderedPodCliqueSet.desired.Spec.Template.Cliques) == 0 { if existing := renderedPodCliqueSet.existing; existing != nil { - if !metav1.IsControlledBy(existing, dgd) { + if !metav1.IsControlledBy(existing, source) { return ReconcileResult{}, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) } if existing.DeletionTimestamp.IsZero() { @@ -106,13 +108,13 @@ func (r *groveWorkloadsReconciler) Reconcile( } return ReconcileResult{State: nvidiacomv1beta1.DGDStatePending, Reason: "RemovingEmptyPodCliqueSet", Message: "Waiting for the removed ordinary workload"}, nil } - stableResources, err := r.stableResources.Reconcile(ctx, dgd, renderedPodCliqueSet.renderDeployment) + stableResources, err := r.stableResources.Reconcile(ctx, source, renderedPodCliqueSet.renderDeployment) if err != nil { return ReconcileResult{}, err } return checkResourcesReadiness(stableResources), nil } - syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, dgd, renderedPodCliqueSet) + syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) if err != nil { return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) } @@ -121,8 +123,8 @@ func (r *groveWorkloadsReconciler) Reconcile( // Pre-existing legacy PCS without a suffix is the only case where unstamped is valid. isLegacyUnsuffixed := workerHashTransition.noCurrentAnnotation && renderedPodCliqueSet.existing != nil && - !podCliqueSetUsesGroveWorkerHashSuffix(dgd, renderedPodCliqueSet.existing) - observed, err := podCliqueSetObservesWorkerHash(dgd, renderedPodCliqueSet.existing, isLegacyUnsuffixed) + !podCliqueSetUsesGroveWorkerHashSuffix(source, renderedPodCliqueSet.existing) + observed, err := podCliqueSetObservesWorkerHash(source, renderedPodCliqueSet.existing, isLegacyUnsuffixed) if err != nil { return ReconcileResult{}, failWorkloadProgram( reasonRollingUpdateFailed, @@ -130,7 +132,7 @@ func (r *groveWorkloadsReconciler) Reconcile( ) } if observed { - if err := r.rollout.commitUnsupportedWorkerHashTransition(ctx, dgd, workerHashTransition, true); err != nil { + if err := r.rollout.commitUnsupportedWorkerHashTransition(ctx, source, workerHashTransition, true); err != nil { return ReconcileResult{}, failWorkloadProgram( reasonRollingUpdateFailed, fmt.Errorf("project observed Grove worker hash: %w", err), @@ -144,7 +146,7 @@ func (r *groveWorkloadsReconciler) Reconcile( return ReconcileResult{}, fmt.Errorf("failed to reconcile Grove scaling: %w", err) } - stableResources, err := r.stableResources.Reconcile(ctx, dgd, renderedPodCliqueSet.renderDeployment) + stableResources, err := r.stableResources.Reconcile(ctx, source, renderedPodCliqueSet.renderDeployment) if err != nil { return ReconcileResult{}, err } diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index c296907406f7..3ccf0e612bf6 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -120,6 +120,7 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { result, err := reconciler.newGroveProgram().workloads.Reconcile( context.Background(), dgd, + projectOrdinaryGroveDeployment(dgd), nil, nil, ) @@ -236,7 +237,7 @@ func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFai t.Log("Reconcile the full workload transition") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) t.Log("Verify the failed PCS sync leaves the persisted DGD hash unchanged") require.Error(t, err) @@ -323,7 +324,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Persist the PCS suffix without projecting the DGD hash from the write receipt") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) require.NoError(t, err) t.Log("Verify the write receipt leaves the parent hash unchanged") @@ -342,7 +343,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Observe the suffix on a later reconcile, then reject the parent projection") freshDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) require.Error(t, err) assert.Equal(t, 1, pcsUpdateCalls) assert.Equal(t, 1, dgdUpdateCalls) @@ -351,7 +352,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( failDGDUpdate = false freshDGD = &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) require.NoError(t, err) t.Log("Verify the retry commits the target hash without rewriting the PCS") @@ -364,7 +365,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Verify the completed transition is idempotent") idempotentDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), idempotentDGD)) - _, err = workloads.Reconcile(context.Background(), idempotentDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), idempotentDGD, projectOrdinaryGroveDeployment(idempotentDGD), nil, nil) require.NoError(t, err) assert.Equal(t, 1, pcsUpdateCalls) assert.Equal(t, 2, dgdUpdateCalls) @@ -648,7 +649,7 @@ func TestGroveWorkloadsReconciler_SkipsHashObservationWhenHashIsCurrent(t *testi t.Log("Reconcile: hash observation block must be skipped entirely when needsCommit is false") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) require.NoError(t, err) assert.Zero(t, dgdUpdateCalls, "DGD must not be updated when the hash annotation is already current") @@ -727,14 +728,14 @@ func TestGroveWorkloadsReconciler_DefersHashCommitUntilPCSWriteObserved(t *testi t.Log("First reconcile writes the PCS; commit must be deferred") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) require.NoError(t, err) assert.Zero(t, dgdUpdateCalls, "hash annotation must not be committed on the reconcile that writes the PCS") t.Log("Second reconcile is a no-op PCS sync; commit must proceed") freshDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) require.NoError(t, err) assert.Equal(t, 1, dgdUpdateCalls, "hash annotation must be committed once the PCS write is observed") diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index c75a67466499..e86bcafc2546 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -12,9 +12,7 @@ import ( v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -101,141 +99,3 @@ func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGrap } return fmt.Errorf("waiting for LPXGraphDeployment %q to finish cleanup", child.Name) } - -// resolveRestartProgress composes child-owned LPX and ordinary Grove observations. -func (p *groveProgram) resolveRestartProgress(ctx context.Context, dgd *v1beta1.DynamoGraphDeployment, inProgress []string) []string { - ordinary := make([]string, 0, len(inProgress)) - pending := make(map[string]bool, len(inProgress)) - var childComponents map[string]v1beta1.ComponentReplicaStatus - childObserved := false - - // Partition live members and observe the shared LPX child only when requested. - for _, name := range inProgress { - component := dgd.GetComponentByName(name) - if component == nil { - continue - } - if !component.IsLPX() { - ordinary = append(ordinary, name) - continue - } - if !childObserved { - childComponents = observeLPXRestart(ctx, p.lpx.Client, dgd) - childObserved = true - } - if !childComponents[name].Ready { - pending[name] = true - } - } - - // Native pending or failed observations cannot prevent LPX progress. - if len(ordinary) > 0 { - for _, name := range p.restartProgress.Resolve(ctx, dgd, ordinary) { - pending[name] = true - } - } - - // Retain the original authored restart order across the two observations. - remaining := make([]string, 0, len(inProgress)) - for _, name := range inProgress { - if pending[name] { - remaining = append(remaining, name) - } - } - return remaining -} - -// observeLPXRestart returns component status from one current, ready restart observation. -// A failed read or incomplete child leaves every requested member pending. -func observeLPXRestart(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment) map[string]v1beta1.ComponentReplicaStatus { - child := &v1alpha1.LPXGraphDeployment{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || - child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || - dynamo.ValidateLPXSource(child, source) != nil || source.Spec.Restart == nil || - child.Annotations[dynamo.LPXRestartAnnotation] != source.Spec.Restart.ID { - return nil - } - ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") - if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { - return nil - } - return child.Status.Components -} - -// projectLPXChildStatus projects the child returned by a successful handoff of source. -// Its identity is already reconciled; results and conditions still require current generations. -// child may be nil or deleting; all other pointer inputs must be non-nil. -func projectLPXChildStatus(source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, result *ReconcileResult, status *v1beta1.DynamoGraphDeploymentStatus) { - previousLPX := status.LPX - status.LPX = nil - components := lpx.Components(source) - if len(components) == 0 && child == nil { - if previousLPX != nil && apiequality.Semantic.DeepEqual(status.Placement, previousLPX.Placement) { - status.Placement = nil - } - return - } - status.Placement = nil - if result.ComponentStatus == nil { - result.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) - } - serving := lpx.ServingComponent(source) - for _, component := range components { - kind := v1beta1.ComponentKindPodCliqueScalingGroup - if component != serving { - kind = v1beta1.ComponentKindPodClique - } - result.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ - ComponentKind: kind, - } - } - // Result payloads belong only to a completely observed revision. - current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() - observed := current && child.Status.ObservedGeneration == child.Generation - if observed { - status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: child.Status.ModelDownload.DeepCopy(), - Placement: child.Status.Placement.DeepCopy(), - } - // Public attempt observation is relative to the DGD, not its private child. - if status.LPX.Placement != nil && status.LPX.Placement.LPXAttempt != nil { - status.LPX.Placement.LPXAttempt.ObservedGeneration = source.Generation - } - // Keep the established DGD-level field as a compatibility projection of LPX-owned placement. - status.Placement = status.LPX.Placement - // Project only current authored members from the single observed child. - for _, component := range components { - if observedStatus, found := child.Status.Components[component.ComponentName]; found { - result.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() - } - } - } - - // Reconciliation errors advance their condition, not the complete result. - // Surface that current failure without exposing stale payloads or hiding an ordinary failure. - if result.State != v1beta1.DGDStateFailed && current { - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") - if failed != nil && failed.Status == metav1.ConditionTrue && failed.ObservedGeneration == child.Generation { - result.State = v1beta1.DGDStateFailed - result.Reason, result.Message = Reason(failed.Reason), Message(failed.Message) - return - } - } - if observed { - // The child publishes aggregate readiness and member status atomically. - ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") - if ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { - return - } - if result.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { - result.State = v1beta1.DGDStatePending - result.Reason, result.Message = Reason(ready.Reason), Message(ready.Message) - return - } - } - if result.State != v1beta1.DGDStateFailed { - result.State = v1beta1.DGDStatePending - result.Reason = "LPXChildPending" - result.Message = "Waiting for the current LPX engine revision and readiness" - } -} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index a9821be92874..020308b05c52 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -64,6 +64,37 @@ func newLPXHandoffFixture(t *testing.T, fixture string) (*v1alpha1.LPXGraphDeplo return child, source, kube } +// projectLPXChildStatus applies the production call-site composition for focused tests. +func projectLPXChildStatus( + source *v1beta1.DynamoGraphDeployment, + child *v1alpha1.LPXGraphDeployment, + result *ReconcileResult, + status *v1beta1.DynamoGraphDeploymentStatus, +) { + previousLPX := status.LPX + *result, status.LPX = mergeLPXChildStatus(source, child, *result) + status.Placement = lpxPlacementProjection(source, child, status.Placement, previousLPX, status.LPX) +} + +func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { + t.Log("Project the exact DGD-owned component subset from a hybrid graph") + source := newLPXHandoffSource(t, "node-local-v2-hybrid") + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", + ComponentType: v1beta1.ComponentTypePrefill, + }) + before := source.DeepCopy() + ordinary := projectOrdinaryGroveDeployment(source) + + require.Equal(t, before, source) + require.Nil(t, ordinary.GetComponentByName("lpx")) + require.Len(t, ordinary.Spec.Components, len(source.Spec.Components)-1) + + t.Log("Keep the projected value independently mutable by later Grove preparation") + ordinary.Spec.Components[0].ComponentName = "ordinary-copy" + require.Equal(t, before, source) +} + func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { t.Log("Create the generated handoff from a real source DGD") source := newLPXHandoffSource(t, "node-local-v2-lpu-only") @@ -276,9 +307,13 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { unchanged, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.Empty(t, unchanged.Annotations[dynamo.LPXRestartAnnotation]) - progress := (&DynamoGraphDeploymentReconciler{ + resolveProgress := (&DynamoGraphDeploymentReconciler{ Client: kube, RuntimeConfig: &commoncontroller.RuntimeConfig{}, }).newGroveProgram().resolveRestartProgress + ordinaryDGD := projectOrdinaryGroveDeployment(source) + progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { + return resolveProgress(ctx, source, ordinaryDGD, inProgress) + } restart := newDGDRestartReconciler().Resolve(t.Context(), source, &source.Status, progress) require.Equal(t, []string{"lpx"}, restart.Status.InProgress) @@ -423,9 +458,13 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { return reader.Get(ctx, key, obj, opts...) }, }) - progress := (&DynamoGraphDeploymentReconciler{ + resolveProgress := (&DynamoGraphDeploymentReconciler{ Client: observed, RuntimeConfig: &commoncontroller.RuntimeConfig{}, }).newGroveProgram().resolveRestartProgress + ordinaryDGD := projectOrdinaryGroveDeployment(source) + progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { + return resolveProgress(ctx, source, ordinaryDGD, inProgress) + } restarter := newDGDRestartReconciler() source.Status.Restart = restarter.Resolve(t.Context(), source, &source.Status, progress).Status require.NotEmpty(t, source.Status.Restart.InProgress) @@ -476,6 +515,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1))}, v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, ) + ordinaryDGD = projectOrdinaryGroveDeployment(source) child.Status.Components["draft"] = v1beta1.ComponentReplicaStatus{Ready: true, Replicas: 2} require.NoError(t, kube.Status().Update(t.Context(), child)) requested := []string{"draft", "frontend", "lpx", "prefill", "removed"} @@ -487,7 +527,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { t.Log("An unobserved ordinary PCS cannot hold back either ready LPX member") pcs := &grovev1alpha1.PodCliqueSet{ - ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForDGD(source.Name, source.Spec.Components), Namespace: source.Namespace, Generation: 2}, + ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForDGD(ordinaryDGD.Name, ordinaryDGD.Spec.Components), Namespace: source.Namespace, Generation: 2}, Status: grovev1alpha1.PodCliqueSetStatus{ObservedGeneration: ptr.To(int64(1))}, } require.NoError(t, kube.Create(t.Context(), pcs)) @@ -500,7 +540,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) require.NoError(t, kube.Update(t.Context(), pcs)) prefill := &grovev1alpha1.PodClique{ - ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(source, "prefill"), Namespace: source.Namespace, Generation: 1}, + ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(ordinaryDGD, "prefill"), Namespace: source.Namespace, Generation: 1}, Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, } @@ -587,7 +627,8 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.NoError(t, err) require.Equal(t, v1beta1.DGDStatePending, result.Status.State) pcs := &grovev1alpha1.PodCliqueSet{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(source.Name, source.Spec.Components)} + ordinaryDGD := projectOrdinaryGroveDeployment(source) + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinaryDGD.Name, ordinaryDGD.Spec.Components)} require.NoError(t, kube.Get(t.Context(), key, pcs)) require.True(t, metav1.IsControlledBy(pcs, source)) require.Len(t, pcs.Spec.Template.Cliques, 1) @@ -607,6 +648,13 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.Equal(t, "enabled", service.Annotations["example.com/model-discovery"]) require.True(t, metav1.IsControlledBy(service, source)) + t.Log("Observe the created PCS before committing its worker hash") + require.Empty(t, source.Annotations[consts.AnnotationCurrentWorkerHashV2]) + _, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + require.NotEmpty(t, source.Annotations[consts.AnnotationCurrentWorkerHashV2]) + t.Log("A pending ordinary checkpoint retains its startup and scaling gates alongside the pending child") prefill := source.GetComponentByName("prefill") prefill.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ @@ -653,7 +701,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { t.Log("Removing the last ordinary component deletes its PCS without disturbing the child") source.Spec.Components = source.Spec.Components[:1] for range 2 { - _, err = program.workloads.Reconcile(t.Context(), source, nil, nil) + _, err = program.workloads.Reconcile(t.Context(), source, projectOrdinaryGroveDeployment(source), nil, nil) require.NoError(t, err) require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), key, pcs))) } diff --git a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go index aa5508903f2c..6a2295d11fd9 100644 --- a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go @@ -270,7 +270,7 @@ func TestGroveRenderDeploymentWorkerHashSuffix(t *testing.T) { before := dgd.DeepCopy() t.Log("Render the Grove deployment") - rendered, err := groveRenderDeployment(dgd, nil, tt.workerHashSuffix) + rendered, err := groveRenderDeployment(projectOrdinaryGroveDeployment(dgd), dgd, nil, tt.workerHashSuffix) require.NoError(t, err) worker := rendered.GetComponentByName("worker") require.NotNil(t, worker) diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index 1eefa5525147..cc4327649f1c 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -460,14 +460,14 @@ func TestGroveRendererFailsWhenResolvedDRADependencyDisappears(t *testing.T) { ) t.Log("Verify the renderer initially publishes the full multinode shape") - rendered, err := renderer.Render(t.Context(), dgd, nil, nil, false) + rendered, err := renderer.Render(t.Context(), dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) require.NoError(t, err) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerEngine) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerReplica) t.Log("Delete the dependency without changing DGD generation and render again") require.NoError(t, kubeClient.Delete(t.Context(), claimTemplate)) - _, err = renderer.Render(t.Context(), dgd, nil, nil, false) + _, err = renderer.Render(t.Context(), dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) require.ErrorContains(t, err, "ResourceClaimTemplate default/gpu-template") } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 426cdf89212d..4598d1cebc89 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -1124,6 +1124,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { result, err := reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, + projectOrdinaryGroveDeployment(dgd), nil, nil, ) @@ -1154,6 +1155,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { result, err = reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, + projectOrdinaryGroveDeployment(dgd), nil, nil, ) @@ -1250,6 +1252,7 @@ func TestGroveWorkloadsReconciler_UsesPreservedAlphaServiceIngress(t *testing.T) _, err := reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, + projectOrdinaryGroveDeployment(dgd), nil, nil, ) @@ -1336,7 +1339,7 @@ func TestGroveWorkloadRendererRenderPreservesLegacyWorkerSelectors(t *testing.T) nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) generatedPCS := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -1634,7 +1637,7 @@ func TestGroveWorkloadRendererRenderKeepsNativeWorkerSelectors(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) renderDGD := renderedPCS.renderDeployment prefill := renderDGD.GetComponentByName("prefill") diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index f27d1e42e69f..f3d7d2a6a282 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -14,13 +14,11 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" - groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -177,35 +175,6 @@ func lpxPodGangPredicate() predicate.Predicate { } } -// OwnsPodClique verifies the LPX owner chain before suppressing ordinary DGD readiness events. -// Annotations are authorable hints; missing cached owners must not hide updates. -func OwnsPodClique(ctx context.Context, reader client.Reader, clique *grovev1alpha1.PodClique) bool { - if clique == nil || reader == nil || !lpxPodCliqueMaterializationEvent(clique) { - return false - } - owner := metav1.GetControllerOf(clique) - if owner == nil || owner.Kind != groveconstants.KindPodCliqueScalingGroup || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { - return false - } - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, group); err != nil || group.UID != owner.UID { - return false - } - owner = metav1.GetControllerOf(group) - if owner == nil || owner.Kind != groveconstants.KindPodCliqueSet || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { - return false - } - pcs := &grovev1alpha1.PodCliqueSet{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, pcs); err != nil || pcs.UID != owner.UID { - return false - } - owner = metav1.GetControllerOf(pcs) - return owner != nil && owner.Kind == nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind && - owner.APIVersion == nvidiacomv1alpha1.GroupVersion.String() && owner.UID != "" && - owner.Name == clique.Labels[consts.KubeLabelDynamoGraphDeploymentName] && - string(owner.UID) == clique.Annotations[dynamo.LPXDeploymentUIDAnnotation] -} - func lpxPodCliqueMaterializationEvent(obj client.Object) bool { clique := obj.(*grovev1alpha1.PodClique) role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] diff --git a/deploy/operator/internal/controller/upgrade_test.go b/deploy/operator/internal/controller/upgrade_test.go index f4e68f515fe5..b6636cb7bd35 100644 --- a/deploy/operator/internal/controller/upgrade_test.go +++ b/deploy/operator/internal/controller/upgrade_test.go @@ -811,7 +811,7 @@ spec: &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) require.NoError(t, err) pcs := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -937,7 +937,7 @@ func TestGroveNativeWorkerIdentityLabelsStayNative(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) require.NoError(t, err) desired := renderedPCS.desired renderDGD := renderedPCS.renderDeployment diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 1e919e6468e4..9e227b569a72 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -1367,12 +1367,10 @@ func expandMultinodeGMSRoles(componentName string, numberOfNodes int32, totalEng // For short DGD names the PCS name equals the DGD name (backwards compatible). // For long names, the PCS name is truncated with a deterministic 4-char hash // suffix to guarantee uniqueness and reconcile-loop stability. +// Components must be the exact subset materialized by this PodCliqueSet. func PCSNameForDGD(dgdName string, components []v1beta1.DynamoComponentDeploymentSharedSpec) string { maxComponentBudget := 0 for i := range components { - if components[i].ManagedByExternalController() { - continue - } budget := ComponentNameBudget(&components[i]) if budget > maxComponentBudget { maxComponentBudget = budget diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index e31880d47a6e..692d61184df0 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -86,7 +86,7 @@ func RenderLPXBasePodCliqueSet( } const ( - LPXDeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" + LPXDeploymentUIDAnnotation = dynamolpx.DeploymentUIDAnnotation LPXDeploymentGenerationAnnotation = "lpx.nvidia.com/deployment-generation" LPXInputRevisionAnnotation = "lpx.nvidia.com/input-revision" LPXRestartAnnotation = "lpx.nvidia.com/restart-id" diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index cb9288507c5e..f211d15bf668 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -10936,13 +10936,6 @@ func TestPCSNameForDGD(t *testing.T) { Multinode: &v1beta1.MultinodeSpec{NodeCount: 2}, } } - lpxComponent := func(name string) v1beta1.DynamoComponentDeploymentSharedSpec { - return v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: name, - ComponentType: v1beta1.ComponentTypeLPX, - LPX: &v1beta1.LPXConfig{BuildID: "build-id"}, - } - } interPodGMSComponent := func(name string) v1beta1.DynamoComponentDeploymentSharedSpec { return v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: name, @@ -10994,15 +10987,6 @@ func TestPCSNameForDGD(t *testing.T) { // pcsBudget=45-19=26; dgdName is 28 chars → needs truncation to 26. wantLen: 26, }, - { - name: "normal PCS name does not budget LPX worker cliques", - dgdName: "abcdefghijklmnopqrstuvwxyzabcdef", - components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - lpxComponent("lpu"), - }, - // LPX has its own PCS name budget, so this name stays unchanged. - wantLen: 32, - }, { name: "deterministic - same input always produces same output", dgdName: "deepseek-v32-fp4-trtllm-dgd1", @@ -11022,14 +11006,6 @@ func TestPCSNameForDGD(t *testing.T) { // 7 < minPCSNameLength(8), so clamped to 8 wantLen: 8, }, - { - name: "normal PCS name is independent of LPX role expansion", - dgdName: "deepseek-v32-fp4-lpu-dgd", - components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - lpxComponent("lpu"), - }, - want: "deepseek-v32-fp4-lpu-dgd", - }, { name: "empty components - no truncation needed", dgdName: "my-dgd", @@ -11063,9 +11039,6 @@ func TestPCSNameForDGD(t *testing.T) { maxComponentBudget := 0 for i := range tt.components { component := &tt.components[i] - if component.IsLPX() { - continue - } budget := ComponentNameBudget(component) if budget > maxComponentBudget { maxComponentBudget = budget diff --git a/deploy/operator/internal/dynamo/lpx/grove_ownership.go b/deploy/operator/internal/dynamo/lpx/grove_ownership.go new file mode 100644 index 000000000000..6c3b8efe2b3f --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/grove_ownership.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. +const DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" + +// OwnsPodClique verifies the LPX owner chain before ordinary DGD watches +// suppress a role-readiness event. Authorable annotations alone are not proof. +func OwnsPodClique(ctx context.Context, reader client.Reader, clique *grovev1alpha1.PodClique) bool { + if clique == nil || reader == nil || !isMaterializedRole(clique) { + return false + } + owner := metav1.GetControllerOf(clique) + if owner == nil || owner.Kind != groveconstants.KindPodCliqueScalingGroup || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { + return false + } + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, group); err != nil || group.UID != owner.UID { + return false + } + owner = metav1.GetControllerOf(group) + if owner == nil || owner.Kind != groveconstants.KindPodCliqueSet || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { + return false + } + pcs := &grovev1alpha1.PodCliqueSet{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, pcs); err != nil || pcs.UID != owner.UID { + return false + } + owner = metav1.GetControllerOf(pcs) + return owner != nil && owner.Kind == nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind && + owner.APIVersion == nvidiacomv1alpha1.GroupVersion.String() && owner.UID != "" && + owner.Name == clique.Labels[consts.KubeLabelDynamoGraphDeploymentName] && + string(owner.UID) == clique.Annotations[DeploymentUIDAnnotation] +} + +func isMaterializedRole(clique *grovev1alpha1.PodClique) bool { + role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] + return clique.Annotations[WorkloadDigestAnnotation] != "" && + (role == lpxv1alpha1.PodRoleAgent || role == lpxv1alpha1.PodRoleConductor || role == lpxv1alpha1.PodRoleCyborgWorker) +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index cb4b7982a82a..2bb5a81cfae2 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -208,7 +208,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( oldComponents = componentsByName(oldDGD.Spec.Components) } specOpts := dynamoGraphDeploymentSpecValidationOptions{ - pcsName: dynamo.PCSNameForDGD(dgd.Name, dgd.Spec.Components), + pcsName: dynamo.PCSNameForDGD(dgd.Name, ordinaryGroveComponents(dgd.Spec.Components)), generation: dgd.Generation, workloadProvider: workloadProvider, grovePathway: grovePathway, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx.go new file mode 100644 index 000000000000..30e1769d98d2 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx.go @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "slices" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +func ordinaryGroveComponents(components []v1beta1.DynamoComponentDeploymentSharedSpec) []v1beta1.DynamoComponentDeploymentSharedSpec { + return slices.DeleteFunc( + slices.Clone(components), + func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.ManagedByExternalController() + }, + ) +} From b75fc189aada39177c96f944cb72d8e6ccd4be43 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:39 +0300 Subject: [PATCH 008/196] fix(operator): scope fsGroup policy to LPX pods (#427) Signed-off-by: Andrew Paprotsky (cherry picked from commit 6f4276c447a5f0244b46f82d251387c3b4a95cae) Signed-off-by: Andrew Paprotsky --- ...namocomponentdeployment_controller_test.go | 6 +- .../internal/controller/upgrade_test.go | 5 -- deploy/operator/internal/dynamo/graph.go | 1 - deploy/operator/internal/dynamo/graph_lpx.go | 6 ++ .../internal/dynamo/graph_lpx_test.go | 71 +++++++++++++++++++ deploy/operator/internal/dynamo/graph_test.go | 36 ++++------ .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 2 - 7 files changed, 91 insertions(+), 36 deletions(-) diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go index a2c8190d64ee..86cfaf1ff4f0 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go @@ -1314,8 +1314,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing. Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -1456,8 +1455,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing. Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { diff --git a/deploy/operator/internal/controller/upgrade_test.go b/deploy/operator/internal/controller/upgrade_test.go index b6636cb7bd35..5d1844a1e5c7 100644 --- a/deploy/operator/internal/controller/upgrade_test.go +++ b/deploy/operator/internal/controller/upgrade_test.go @@ -231,7 +231,6 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch strategy: type: RollingUpdate rollingUpdate: @@ -427,7 +426,6 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch workerTemplate: metadata: labels: @@ -514,7 +512,6 @@ spec: serviceAccountName: qwen-decode-db6b6891-k8s-service-discovery securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch size: 2 rolloutStrategy: type: "" @@ -687,7 +684,6 @@ spec: terminationGracePeriodSeconds: 60 securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch replicas: 1 minAvailable: 1 - name: vllmdecodeworker @@ -789,7 +785,6 @@ spec: terminationGracePeriodSeconds: 60 securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch replicas: 1 minAvailable: 1 cliqueStartupType: CliqueStartupTypeAnyOrder diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 9e227b569a72..3501352aeceb 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -1610,7 +1610,6 @@ func applyDefaultSecurityContext(podSpec *corev1.PodSpec) { // which maintains compatibility with both root and non-root images if podSpec.SecurityContext.FSGroup == nil { podSpec.SecurityContext.FSGroup = ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)) - podSpec.SecurityContext.FSGroupChangePolicy = ptr.To(corev1.FSGroupChangeOnRootMismatch) } } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 692d61184df0..fac0cb7897e1 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -484,6 +484,12 @@ func renderSelectedLPXRole( if err != nil { return nil, err } + + // Keep the LPX volume-permission default without changing authored security contexts. + if component.PodTemplate == nil || component.PodTemplate.Spec.SecurityContext == nil { + basePodSpec.SecurityContext.FSGroupChangePolicy = ptr.To(corev1.FSGroupChangeOnRootMismatch) + } + if component.IsLPX() && selectedWorkload.BuildFamily() == dynamolpx.BuildFamilyXT { explicitVolumes := component.PodTemplate.Spec.Volumes runtimeVolumes := []string{"config"} diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index ad2e865622e0..55b927fd86c8 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -35,6 +36,76 @@ func TestBackendDetectionSkipsNonWorkers(t *testing.T) { } } +func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { + tests := []struct { + name string + authored *corev1.PodSecurityContext + expected *corev1.PodSecurityContext + }{ + { + name: "no authored security context", + expected: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + }, + }, + { + name: "authored fsGroup without policy", + authored: &corev1.PodSecurityContext{FSGroup: ptr.To(int64(2000))}, + expected: &corev1.PodSecurityContext{FSGroup: ptr.To(int64(2000))}, + }, + { + name: "authored always policy", + authored: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(2000)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), + }, + expected: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(2000)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), + }, + }, + { + name: "authored empty security context", + authored: &corev1.PodSecurityContext{}, + expected: &corev1.PodSecurityContext{}, + }, + { + name: "authored root security context", + authored: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), + }, + expected: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Author a Cyborg role using the shared LPX role base renderer") + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "hybrid", Namespace: "test"}, + } + component := &v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "engine", ComponentType: v1beta1.ComponentTypeDecode, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main", Image: "cyborg:test"}}, + SecurityContext: test.authored, + }}, + } + + t.Log("Render the role before runtime-specific lowering") + template, err := renderSelectedLPXRole(component, source, nil, + &configv1alpha1.OperatorConfiguration{}, &mockSecretsRetriever{}, DiscoveryContext{}, + &lpx.SelectedWorkload{}, &imageEntrypointComponentDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) + require.NoError(t, err) + + t.Log("Retain authored security context and default only an absent one") + require.Equal(t, test.expected, template.Spec.SecurityContext) + }) + } +} + func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "chat-uid"}, diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index f211d15bf668..fa0a4d380125 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -2041,8 +2041,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, ImagePullSecrets: []corev1.LocalObjectReference{ { @@ -2237,8 +2236,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { ServiceAccountName: commonconsts.PlannerServiceAccountName, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, RestartPolicy: corev1.RestartPolicyAlways, @@ -2658,8 +2656,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { RestartPolicy: corev1.RestartPolicyAlways, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -2873,8 +2870,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { RestartPolicy: corev1.RestartPolicyAlways, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -3060,8 +3056,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -3224,8 +3219,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), ServiceAccountName: commonconsts.PlannerServiceAccountName, SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, RestartPolicy: corev1.RestartPolicyAlways, Volumes: []corev1.Volume{ @@ -3695,8 +3689,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -3886,8 +3879,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { PodSpec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -4074,8 +4066,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { }, TerminationGracePeriodSeconds: ptr.To(int64(10)), SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ @@ -4238,8 +4229,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), ServiceAccountName: commonconsts.PlannerServiceAccountName, SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -6696,8 +6686,7 @@ func TestGenerateBasePodSpec_Worker(t *testing.T) { TerminationGracePeriodSeconds: ptr.To(int64(60)), SecurityContext: &corev1.PodSecurityContext{ // Only fsGroup is injected by default for volume permissions - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, Volumes: []corev1.Volume{ { @@ -8010,8 +7999,7 @@ func TestGenerateBasePodSpec_SecurityContext(t *testing.T) { ComponentType: commonconsts.ComponentTypeFrontend, }, expectedSecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), }, description: "Operator should only inject fsGroup for volume permissions, not UID/GID (backward compatible)", }, diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index f1d6b1ca4d1f..3756dd777754 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -771,7 +771,6 @@ spec: restartPolicy: Always securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 volumes: - name: model-storage @@ -948,7 +947,6 @@ spec: restartPolicy: Always securityContext: fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 tolerations: - key: nvidia.com/gpu From 6260265dee6e5be5360ee0e73959bbf652e58bf0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:40 +0300 Subject: [PATCH 009/196] fix(operator): preserve args-only LPX entrypoints (#428) Signed-off-by: Andrew Paprotsky (cherry picked from commit 2d69c3fcd50d24d0cec9fd7b123d7ae0b4b1dec6) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 3 --- .../operator/internal/dynamo/lpx/pod_roles_test.go | 4 ++-- .../operator/internal/dynamo/lpx/runtime_agent.go | 14 +++++++------- .../dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 2 -- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 2659cbd8f66c..56066b753a4f 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1449,9 +1449,6 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin require.Contains(t, main.Args[1], "GROVE_PCLQ_POD_INDEX") } else { wantCommand := intent.command - if mode.hybrid && len(wantCommand) == 0 { - wantCommand = []string{"/bin/bash"} - } require.Equal(t, wantCommand, main.Command) require.True(t, slices.Equal(intent.args, main.Args)) } diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index af87d310ae25..ec7f3115c9df 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -348,7 +348,7 @@ func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T { name: "args", container: corev1.Container{Name: commonconsts.MainContainerName, Args: []string{"custom-agent"}}, - wantCommand: []string{"/bin/bash"}, + wantCommand: nil, wantArgs: []string{"custom-agent"}, wantStartup: lpuV2StartupProbe(true), wantReadiness: lpuV2ReadinessProbe(true), @@ -368,7 +368,7 @@ func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T StartupProbe: customStartup, ReadinessProbe: customReadiness, }, - wantCommand: []string{"/bin/bash"}, + wantCommand: nil, wantArgs: []string{"custom-agent"}, wantStartup: customStartup, wantReadiness: customReadiness, diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index ba93ed990968..8304e927dd79 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -118,18 +118,18 @@ func configureDirectHybridAgentRuntime( agent := findMainContainer(agentPodSpec.Containers) // Drop Nova-only flags before applying the direct Agent identity and worker defaults. agent.Args, _ = stripNovaOnlyArgs(agent.Args) - hasCustomCommand := len(agent.Command) != 0 || len(agent.Args) != 0 + hasCustomStartup := len(agent.Command) != 0 || len(agent.Args) != 0 agent.Name = lpuAgentContainerName - applyLPUWorkerContainerBase(agent, len(agent.Command) != 0) - if len(agent.Args) == 0 && !hasCustomCommand { + applyLPUWorkerContainerBase(agent, hasCustomStartup) + if len(agent.Args) == 0 && !hasCustomStartup { agent.Args = []string{"-c", lpuPartitionRunCommand} } - if !hasCustomCommand || agent.StartupProbe == nil { - agent.StartupProbe = lpuV2StartupProbe(hasCustomCommand) + if !hasCustomStartup || agent.StartupProbe == nil { + agent.StartupProbe = lpuV2StartupProbe(hasCustomStartup) } - if !hasCustomCommand || agent.ReadinessProbe == nil { - agent.ReadinessProbe = lpuV2ReadinessProbe(hasCustomCommand) + if !hasCustomStartup || agent.ReadinessProbe == nil { + agent.ReadinessProbe = lpuV2ReadinessProbe(hasCustomStartup) } agent.Env = append(agent.Env, diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 3756dd777754..9adce524b437 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -494,8 +494,6 @@ spec: --nic "${NIC_NAME}" \ --metadata-file "$(mktemp)" \ --readiness-port "${READINESS_PORT}" - command: - - /bin/bash env: - name: DYN_COMPONENT value: lpx From 66ec78de13236c8871cdc1a199d24a28032ed225 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:40 +0300 Subject: [PATCH 010/196] fix(operator): clamp LPX CPU request to limit (#429) Signed-off-by: Andrew Paprotsky (cherry picked from commit 9f054ffd4087fcec23affa44ebf628f401b3c955) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 2 +- .../operator/internal/dynamo/lpx/lpu_pod.go | 11 +++++-- .../internal/dynamo/lpx/pod_roles_test.go | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 56066b753a4f..1084bd20a6b0 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1230,7 +1230,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom", ReadOnly: true}) require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) require.Contains(t, agent.Spec.PodSpec.Volumes, customConfigVolume) - require.Equal(t, resource.MustParse("62"), main.Resources.Requests[corev1.ResourceCPU]) + require.True(t, main.Resources.Requests.Cpu().Equal(resource.MustParse("62"))) require.Equal(t, resource.MustParse("4096Mi"), main.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"]) t.Log("Verify Cyborg receives its generated config and InfiniBand bindings instead of fallback PVCs") diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index c9709efb0832..32875f80f6d3 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -13,7 +13,10 @@ import ( "k8s.io/utils/ptr" ) -const lpuAgentContainerName = "agent" +const ( + lpuAgentContainerName = "agent" + defaultLPUAgentCPUs int64 = 62 +) func applyLPUHostDeviceVolumes(podSpec *corev1.PodSpec, replaceExisting bool) { apply := appendVolumeIfMissing @@ -69,7 +72,11 @@ func applyLPUWorkerContainerBase(container *corev1.Container, preserveCommand bo }) // Agent scheduling already initialized device resources; add runtime CPU and hugepage requirements. - container.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("62") + cpuRequest := *resource.NewQuantity(defaultLPUAgentCPUs, resource.DecimalSI) + if cpuLimit, ok := container.Resources.Limits[corev1.ResourceCPU]; ok && cpuLimit.Cmp(cpuRequest) < 0 { + cpuRequest = cpuLimit.DeepCopy() + } + container.Resources.Requests[corev1.ResourceCPU] = cpuRequest container.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") container.Resources.Limits[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") } diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index ec7f3115c9df..50b736fe244f 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -111,6 +111,35 @@ func TestSelectedRolePodSpecsMaterializeFamilyResourceOnlyOnAgentMainContainer(t } } +func TestApplyLPUWorkerContainerBaseClampsCPURequestToLimit(t *testing.T) { + for _, test := range []struct { + name string + limit string + want string + }{ + {name: "lower limit", limit: "16", want: "16"}, + {name: "higher limit", limit: "64", want: "62"}, + {name: "no limit", want: "62"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Configure the worker CPU limit") + container := corev1.Container{Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, + Limits: corev1.ResourceList{}, + }} + if test.limit != "" { + container.Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(test.limit)} + } + + t.Log("Apply the LPX worker runtime defaults") + applyLPUWorkerContainerBase(&container, false) + + t.Log("Clamp the default request to the configured limit") + require.True(t, container.Resources.Requests.Cpu().Equal(resource.MustParse(test.want))) + }) + } +} + func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { lists := []corev1.ResourceList{ spec.InitContainers[0].Resources.Limits, From 38c525595761c5651f8e5b367f796b0fdf3191d9 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:40 +0300 Subject: [PATCH 011/196] fix(operator): bound LPX auxiliary names (#430) Signed-off-by: Andrew Paprotsky (cherry picked from commit 52b050a592234fb774ed7ea4468a91b4354a3ab3) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/eviction.go | 4 ++-- .../internal/controller/lpx/eviction_test.go | 24 +++++++++++++++++++ .../dynamo/lpx/hybrid_cyborg_bindings.go | 2 +- .../internal/dynamo/lpx/lpu_configmap.go | 21 +++++++++++++++- .../internal/dynamo/lpx/lpu_configmap_test.go | 19 +++++++++++++++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index c35334f1099a..814dcd981518 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -128,9 +128,9 @@ func (r *lpuEvictionReconciler) runtimePartitionByPodIndex(ctx context.Context, return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap hash", trigger.Namespace, trigger.Name) } - // Load the same runtime partition table mounted by every Agent Pod. + // Read the generated runtime partition table independently of authored volume overrides. + configName := dynamolpx.LPUConfigMapName(trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName]) var config corev1.ConfigMap - configName := trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] + "-lpu" if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: configName}, &config); err != nil { return nil, fmt.Errorf("get LPU runtime ConfigMap %s/%s: %w", trigger.Namespace, configName, err) } diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 13a58bf6831f..ca0c739d7803 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -8,6 +8,7 @@ package lpx import ( "context" "sort" + "strings" "testing" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" @@ -18,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -200,6 +202,28 @@ func TestLPUEviction_PodsForTrigger(t *testing.T) { }) } +func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) { + t.Log("Create the generated runtime table for a maximum-length DGD name") + dgdName := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) + config := newLPUEvictionConfigMap() + config.Name = dynamolpx.LPUConfigMapName(dgdName) + + t.Log("Override the Agent config volume without replacing its generated partition table") + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dgdName + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = dynamolpx.LPUConfigMapHash(config) + trigger.Spec.Volumes = []corev1.Volume{{ + Name: "config", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }} + r, _ := newLPUEvictionReconciler(trigger, config) + + t.Log("Read the bounded generated ConfigMap despite the non-ConfigMap volume override") + partitions, err := r.runtimePartitionByPodIndex(t.Context(), trigger) + require.NoError(t, err) + require.Equal(t, map[string]int{"0": 0, "1": 0, "2": 1}, partitions) +} + func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { tests := []struct { name string diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 589537785962..058aca384c5b 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -29,7 +29,7 @@ const ( ) func selectedCyborgConfigMapName(dgdName string) string { - return dgdName + selectedCyborgConfigSuffix + return boundedAuxiliaryName(dgdName, selectedCyborgConfigSuffix) } // ApplySelectedCyborgContainerDefaults installs the operator-owned V2 Cyborg diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 9920996346c4..59014726d043 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" ) const ( @@ -33,6 +34,24 @@ type lpuModelStorage struct { mount corev1.VolumeMount } +func boundedAuxiliaryName(root, suffix string) string { + candidate := root + suffix + if len(candidate) <= validation.DNS1123SubdomainMaxLength { + return candidate + } + + // Distinguish overlong resource names using the full name, including its suffix. + digest := sha256.Sum256([]byte(candidate)) + hashSuffix := fmt.Sprintf("-%x", digest[:4]) + prefix := strings.TrimRight(candidate[:validation.DNS1123SubdomainMaxLength-len(hashSuffix)], "-.") + return prefix + hashSuffix +} + +// LPUConfigMapName returns the generated runtime ConfigMap name within the Kubernetes name limit. +func LPUConfigMapName(root string) string { + return boundedAuxiliaryName(root, "-lpu") +} + func renderLPUConfigMap( namespace string, dgdName string, @@ -79,7 +98,7 @@ func renderLPUConfigMap( data["datacenter.toml"] = datacenterTOML.String() return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: dgdName + "-lpu", + Name: LPUConfigMapName(dgdName), Namespace: namespace, }, Data: data, diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index e4ac6bbcebec..4cbb969ac149 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -7,11 +7,30 @@ package lpx import ( "encoding/json" + "strings" "testing" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/validation" ) +func TestLPXAuxiliaryNamesAreBounded(t *testing.T) { + t.Log("Use a source name already at the Kubernetes length limit") + root := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) + + t.Log("Render the LPU and decode ConfigMap names") + lpuName := LPUConfigMapName(root) + decodeName := selectedCyborgConfigMapName(root) + + t.Log("Keep names valid and distinct while leaving short names unchanged") + require.Len(t, lpuName, validation.DNS1123SubdomainMaxLength) + require.Len(t, decodeName, validation.DNS1123SubdomainMaxLength) + require.Empty(t, validation.IsDNS1123Subdomain(lpuName)) + require.Empty(t, validation.IsDNS1123Subdomain(decodeName)) + require.NotEqual(t, lpuName, decodeName) + require.Equal(t, "short-lpu", LPUConfigMapName("short")) +} + func TestLPURuntimeBuildRef(t *testing.T) { t.Parallel() From c065d371ca164ab8c315d65c8bbea0f0ea44ea3b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:41 +0300 Subject: [PATCH 012/196] refactor(operator): give LPX roles semantic names (#431) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7d7c2e66c4994b81a56fc0889af289fcb72eb274) Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1alpha1/common.go | 4 +++ .../dynamocomponentdeployment_types.go | 2 +- .../api/v1alpha1/lpx_conversion_test.go | 22 +++++++-------- deploy/operator/api/v1beta1/common.go | 4 +++ .../dynamocomponentdeployment_types.go | 2 +- ...nvidia.com_dynamocomponentdeployments.yaml | 4 +-- .../nvidia.com_dynamographdeployments.yaml | 4 +-- .../controller/dgd_lpx_envtest_test.go | 2 +- .../dgd_lpx_handoff_envtest_test.go | 10 +++---- .../controller/lpx/component_roles_test.go | 8 +++--- .../controller/lpx/controller_test.go | 10 +++---- .../internal/controller/lpx/lifecycle_test.go | 18 ++++++------ .../controller/lpx/modeldownload_test.go | 4 +-- .../internal/controller/lpx/render_test.go | 22 +++++++-------- .../internal/controller/lpx/watch_test.go | 10 +++---- deploy/operator/internal/dynamo/graph_lpx.go | 4 +-- .../internal/dynamo/graph_lpx_test.go | 28 +++++++++---------- .../internal/dynamo/lpx/render_test.go | 8 +++--- .../dynamo/lpx/resolve_selected_workload.go | 10 ++++--- .../lpx/resolve_selected_workload_test.go | 28 +++++++++---------- .../operator/internal/dynamo/lpx/selection.go | 6 ++-- .../internal/dynamo/lpx/selection_test.go | 10 +++---- .../dynamo/lpx/selection_validation.go | 8 +++--- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 4 +-- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 12 ++++---- .../node-local-v2-hybrid.input.yaml | 4 +-- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 12 ++++---- .../node-local-v2-lpu-only.input.yaml | 2 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 10 +++---- .../node-local-v2-specdecode.input.yaml | 6 ++-- .../node-local-v2-specdecode.yaml | 14 +++++----- .../node-local-v3-hx-hybrid.input.yaml | 4 +-- .../node-local-v3-hx-hybrid.yaml | 10 +++---- .../node-local-v3-hx-lpu-only.input.yaml | 2 +- .../node-local-v3-hx-lpu-only.yaml | 10 +++---- .../node-local-v3-hx-specdecode.input.yaml | 6 ++-- .../node-local-v3-hx-specdecode.yaml | 14 +++++----- .../from_dgd_yaml/single_v2.input.yaml | 4 +-- .../testdata/from_dgd_yaml/single_v2.yaml | 10 +++---- .../single_v2_manifest_defaults.input.yaml | 4 +-- ...hdeployment_lpx_validation_envtest_test.go | 24 ++++++++-------- .../webhook/validation/shared_v1beta1.go | 11 ++++++-- .../model-deployment/deploy-with-dgd.md | 6 ++-- .../additional-resources/api-reference-k8s.md | 8 +++--- .../dynamo-component-deployment.mdx | 12 ++++---- .../kubernetes-api/full-api-reference.mdx | 8 +++--- 46 files changed, 220 insertions(+), 205 deletions(-) diff --git a/deploy/operator/api/v1alpha1/common.go b/deploy/operator/api/v1alpha1/common.go index 0525fad35502..4d58e5317559 100644 --- a/deploy/operator/api/v1alpha1/common.go +++ b/deploy/operator/api/v1alpha1/common.go @@ -63,6 +63,10 @@ const ( ComponentRoleLeader = "leader" // ComponentRoleWorker identifies the worker Pod-producing role of a multinode component. ComponentRoleWorker = "worker" + // ComponentRoleLPXConductor identifies the launcher role of an LPX component. + ComponentRoleLPXConductor = "conductor" + // ComponentRoleLPXAgent identifies the LPU-serving role of an LPX component. + ComponentRoleLPXAgent = "agent" ) // ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index ea8d2d33c319..7b9a53923904 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -210,7 +210,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components require a worker role and may declare a leader role. + // LPX components require an agent role and may declare a conductor role. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index e7db570b6040..5645edfccad1 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -47,17 +47,17 @@ func TestDynamoGraphDeploymentRoleShapesRoundTrip(t *testing.T) { { name: "native fields", alpha: []ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleLeader}, + {Name: v1beta1.ComponentRoleLPXConductor}, { - Name: v1beta1.ComponentRoleWorker, + Name: v1beta1.ComponentRoleLPXAgent, Replicas: ptr.To[int32](3), PodTemplate: ptr.To(testLPXRoleTemplate("agent")), }, }, beta: []v1beta1.ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleLeader}, + {Name: v1beta1.ComponentRoleLPXConductor}, { - Name: v1beta1.ComponentRoleWorker, + Name: v1beta1.ComponentRoleLPXAgent, Replicas: ptr.To[int32](3), PodTemplate: ptr.To(testLPXRoleTemplate("agent")), }, @@ -108,8 +108,8 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { ComponentType: v1beta1.ComponentTypeLPX, LPX: testCanonicalLPXConfig(), Roles: []v1beta1.ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleLeader, Replicas: ptr.To[int32](2)}, - {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + {Name: v1beta1.ComponentRoleLPXConductor, Replicas: ptr.To[int32](2)}, + {Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, }, }}, }, @@ -145,8 +145,8 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { t.Fatalf("LPX config conversion mismatch (-want +got):\n%s", diff) } wantAlphaRoles := []ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleLeader, Replicas: ptr.To[int32](2)}, - {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + {Name: v1beta1.ComponentRoleLPXConductor, Replicas: ptr.To[int32](2)}, + {Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, } if diff := cmp.Diff(wantAlphaRoles, service.Roles); diff != "" { t.Fatalf("LPX role order or optional fields changed in alpha (-want +got):\n%s", diff) @@ -178,7 +178,7 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { service.LPX.BuildID = editedThroughAlphaBuildID service.Roles = []ComponentRoleSpec{ *service.Roles[1].DeepCopy(), - {Name: v1beta1.ComponentRoleLeader, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, + {Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, } got := &v1beta1.DynamoGraphDeployment{} @@ -190,8 +190,8 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { t.Fatalf("alpha LPX edit was lost: %#v", component) } wantBetaRoles := []v1beta1.ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleWorker, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, - {Name: v1beta1.ComponentRoleLeader, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, + {Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: ptr.To(testLPXRoleTemplate("agent"))}, + {Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, } if diff := cmp.Diff(wantBetaRoles, component.Roles); diff != "" { t.Fatalf("authored alpha role edits did not survive conversion (-want +got):\n%s", diff) diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index a797186ca80e..09c45e073fb3 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -108,6 +108,10 @@ const ( ComponentRoleLeader = "leader" // ComponentRoleWorker identifies the worker Pod-producing role of a multinode component. ComponentRoleWorker = "worker" + // ComponentRoleLPXConductor identifies the launcher role of an LPX component. + ComponentRoleLPXConductor = "conductor" + // ComponentRoleLPXAgent identifies the LPU-serving role of an LPX component. + ComponentRoleLPXAgent = "agent" ) // ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 28cdb65dfa2c..3baf88d89b97 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -199,7 +199,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components require a worker role and may declare a leader role. + // LPX components require an agent role and may declare a conductor role. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 8657d375c1a4..252e671a8f37 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -11476,7 +11476,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require a worker role and may declare a leader role. + LPX components require an agent role and may declare a conductor role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -28679,7 +28679,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require a worker role and may declare a leader role. + LPX components require an agent role and may declare a conductor role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index b88695862331..37a8d18120ed 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11866,7 +11866,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require a worker role and may declare a leader role. + LPX components require an agent role and may declare a conductor role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -25150,7 +25150,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require a worker role and may declare a leader role. + LPX components require an agent role and may declare a conductor role. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go index 4f9a45741b3a..4742aee565d4 100644 --- a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go @@ -74,7 +74,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) t.Log("An LPX template edit advances the real child generation and invalidates the old observation") - lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" + lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" require.NoError(t, env.Client().Update(t.Context(), source)) updated, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 37f1d83a2bc2..4a697bea563e 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -163,8 +163,8 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "lpx", ComponentType: v1beta1.ComponentTypeLPX, LPX: &v1beta1.LPXConfig{BuildID: (&url.URL{Scheme: "file", Path: buildDir}).String()}, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLeader}, { - Name: v1beta1.ComponentRoleWorker, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXConductor}, { + Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "example/lpu-runtime:1.4.0", @@ -188,7 +188,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { }, 20*time.Second, 50*time.Millisecond) t.Log("Change LPX intent so the failing child generation is newer than its completed observation") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "example/lpu-runtime:1.4.1" + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "example/lpu-runtime:1.4.1" require.NoError(t, env.Client().Update(t.Context(), source)) child := &v1alpha1.LPXGraphDeployment{} require.EventuallyWithT(t, func(c *assert.CollectT) { @@ -222,7 +222,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { require.Empty(t, requests.Items) t.Log("Accept an LPX runtime-invalid edit and report its exact rejection on the public DGD") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeName = "manual-placement" + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "manual-placement" require.NoError(t, env.Client().Update(t.Context(), source)) require.EventuallyWithT(t, func(c *assert.CollectT) { if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { @@ -245,7 +245,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { }, 20*time.Second, 50*time.Millisecond) t.Log("Repair the accepted configuration so the same DGD and child can reconcile again") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeName = "" + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "" require.NoError(t, env.Client().Update(t.Context(), source)) t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go index 0ecd19d6f199..efe37165340c 100644 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -22,18 +22,18 @@ func TestLPXRoleDependenciesIncludeDraftAndConductorFallback(t *testing.T) { t.Log("Give both components consumed Agent claims while the target conductor uses its Agent fallback") source := newLPXSpecDecodeTestSource() for _, component := range lpx.Components(source) { - pod := &component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec + pod := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(component.ComponentName + "-gpu")}} pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} } - require.Nil(t, lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate) + require.Nil(t, lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate) before := source.DeepCopy() require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu"}, lpxDRAClaimReferences(true)(source)) require.Equal(t, before, source) t.Log("An independent conductor template contributes its own consumed dependency") - conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader) - conductor.PodTemplate = lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.DeepCopy() + conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor) + conductor.PodTemplate = lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.DeepCopy() conductor.PodTemplate.Spec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("conductor-gpu") require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu", "conductor-gpu"}, lpxDRAClaimReferences(true)(source)) } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index f1bc52f493fa..6440f3b0ae74 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -125,8 +125,8 @@ func TestLPXGPUCapacityReportsTheCompleteEngine(t *testing.T) { case "hybrid": child, source, registry = newLPXTestDGD(t, lpx.PipelineLPX) source.Spec.Components[0].Replicas = ptr.To(int32(3)) - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) - pod := &lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) + pod := &lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec pod.ResourceClaims = nil pod.Containers[0].Resources.Limits = corev1.ResourceList{consts.KubeResourceGPUNvidia: resource.MustParse("4")} want = 8 @@ -201,7 +201,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { case "selected workload": component.Replicas = ptr.To(int32(2)) case "render": - component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].VolumeMounts = nil + component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].VolumeMounts = nil case "invalid source": child.OwnerReferences[0].UID = "replaced-source" case "transient snapshot", "inconsistent snapshot": @@ -498,7 +498,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { {name: "invalid intent", messages: []string{ "spec.providerOverride: Forbidden:", "spec.components[0].providerOverride: Forbidden:", - `LPX worker component requires a "main" runtime container`, + `LPX agent component requires a "main" runtime container`, "does not support checkpointing", "spec.topologyConstraint: Forbidden:", "spec.components[0].topologyConstraint: Forbidden:", @@ -541,7 +541,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueTemplateSpec", } component.ProviderOverride.Value.Raw = []byte(`{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}`) - component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate = &corev1.PodTemplateSpec{} + component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate = &corev1.PodTemplateSpec{} component.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{Enabled: true}} source.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} component.TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 3b0bf8f05036..cac677a21f7d 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -278,7 +278,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) source.Spec.Components[0].Replicas = ptr.To(int32(2)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) require.Equal(t, int32(2), desired.plan.Replicas) require.Len(t, desired.requests, 2) @@ -449,7 +449,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { func TestSelectedLPXObservesLimitsOnlyCyborgGPUIntent(t *testing.T) { t.Log("Materialize a hybrid Cyborg whose classic GPU intent exists only in limits") dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - cyborgPodSpec := &source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + cyborgPodSpec := &source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec cyborgPodSpec.ResourceClaims = nil cyborgPodSpec.Containers[0].Resources = corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -1177,7 +1177,7 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { deployment, source, registry := newLPXTestDGD(t, pipeline) if pipeline == lpx.PipelineLPX { source.Spec.Components[0].Replicas = ptr.To(int32(1)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) } reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) @@ -1808,7 +1808,7 @@ func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) if test.name == crossReplicaAgent { source.Spec.Components[0].Replicas = ptr.To(int32(2)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(1)) + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(1)) } reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) @@ -2136,8 +2136,8 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D BuildID: buildID, Settings: &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":true}`)}, }, - Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLeader}, { - Name: nvidiacomv1beta1.ComponentRoleWorker, + Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, { + Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "lpu-runtime", @@ -2157,8 +2157,8 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D }, } if pipeline == lpx.PipelineLPX { - *source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLeader) = nvidiacomv1beta1.ComponentRoleSpec{ - Name: nvidiacomv1beta1.ComponentRoleLeader, + *source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) = nvidiacomv1beta1.ComponentRoleSpec{ + Name: nvidiacomv1beta1.ComponentRoleLPXConductor, Replicas: &one, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ ResourceClaims: []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("gpu")}}, @@ -2186,7 +2186,7 @@ func newLPXSpecDecodeTestSource() *nvidiacomv1beta1.DynamoGraphDeployment { target := &source.Spec.Components[0] draft := target.DeepCopy() draft.ComponentName, draft.LPX.BuildID, draft.Replicas = "draft", "draft-build", ptr.To(int32(2)) - draft.Roles = []nvidiacomv1beta1.ComponentRoleSpec{*draft.ComponentRole(nvidiacomv1beta1.ComponentRoleWorker)} + draft.Roles = []nvidiacomv1beta1.ComponentRoleSpec{*draft.ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent)} source.Spec.Components = append(source.Spec.Components, *draft) return source } diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 88e814ee851c..a70f098bbf1e 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -340,12 +340,12 @@ func newModelDownloadDGD(buildIDs ...string) *v1beta1.DynamoGraphDeployment { component := v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "lpx-worker", ComponentType: v1beta1.ComponentTypeLPX, Replicas: ptr.To(int32(1)), LPX: &v1beta1.LPXConfig{BuildID: buildID}, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime"}}, }}}}, } if i == len(buildIDs)-1 { - component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLeader}) + component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}) } else { component.ComponentName = "draft" } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 1084bd20a6b0..b87b6b773d38 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -217,11 +217,11 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { if hybrid { component.Replicas = ptr.To(int32(3)) component.MinAvailable = ptr.To(int32(3)) - component.ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + component.ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) } if singleXT { t.Log("Keep XT's authored readonly mount at the canonical config path") - main := &component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0] + main := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0] main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs", ReadOnly: true}) } @@ -426,14 +426,14 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) for _, component := range lpx.Components(source) { - agent := component.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate + agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate agent.Spec.Containers[0].Image = component.ComponentName + "-runtime" agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} } target := lpx.ServingComponent(source) wantImage := target.ComponentName + "-runtime" if explicit { - template := target.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.DeepCopy() + template := target.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.DeepCopy() wantImage = "independent-conductor-runtime" template.Spec.Containers[0].Image = wantImage template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} @@ -444,7 +444,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"conductor"}}}, }}}, }} - target.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate = template + target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = template } before := source.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) @@ -472,7 +472,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { require.Contains(t, container.Args[1], `; exec /bin/nova "$@"`) require.Equal(t, "--", container.Args[2]) if explicit { - template := target.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate + template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate require.Equal(t, "explicit-conductor", clique.Labels["owner"]) require.Equal(t, template.Spec.NodeSelector, clique.Spec.PodSpec.NodeSelector) require.Equal(t, template.Spec.Affinity, clique.Spec.PodSpec.Affinity) @@ -1075,7 +1075,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test lpxComponent.ModelRef = &v1beta1.ModelReference{Name: "test/model"} t.Log("Customize the independent Cyborg batch while keeping its sidecar separate") - cyborgRole := lpxComponent.ComponentRole(v1beta1.ComponentRoleLeader) + cyborgRole := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXConductor) require.NotNil(t, cyborgRole) authoredCyborgMain := &cyborgRole.PodTemplate.Spec.Containers[0] require.Equal(t, commonconsts.MainContainerName, authoredCyborgMain.Name) @@ -1090,7 +1090,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test cyborgRole.PodTemplate.Spec.Containers = append(cyborgRole.PodTemplate.Spec.Containers, sidecar) t.Log("Customize Agent metadata and placement while leaving its launch to the runtime") - agentTemplate := lpxComponent.ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate + agentTemplate := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate agentMain := &agentTemplate.Spec.Containers[0] require.NotEmpty(t, agentMain.Command) require.NotEmpty(t, agentMain.Args) @@ -1390,12 +1390,12 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin component := dgd.GetComponentByName("lpu") require.NotNil(t, component) component.ExtraPodSpecMergeStrategy = strategy.componentStrategy - agent := component.ComponentRole(v1beta1.ComponentRoleWorker) + agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) lpxMain := &agent.PodTemplate.Spec.Containers[0] lpxMain.VolumeMounts = append(lpxMain.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) templates := []*corev1.PodTemplateSpec{agent.PodTemplate} if mode.hybrid { - templates = append(templates, component.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate) + templates = append(templates, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate) } for roleIndex, template := range templates { main := &template.Spec.Containers[0] @@ -1530,7 +1530,7 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) - leader := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate + leader := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ {Name: "config", MountPath: "/custom-config", ReadOnly: true}, {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 68dd190dabc1..73ee02a4abaf 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -258,7 +258,7 @@ func TestLPXSourceWatchIgnoresOrdinaryChurn(t *testing.T) { require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: source})) t.Log("LPX input changes and persisted restart selection wake the child") - changed.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers[0].Image = "new-runtime" + changed.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "new-runtime" require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) changed = source.DeepCopy() changed.Spec.Restart = &nvidiacomv1beta1.Restart{ID: "restart"} @@ -285,8 +285,8 @@ func BenchmarkLPXSourceWatchStatusChurn(b *testing.B) { Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "model/build"}, - Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLeader}, { - Name: nvidiacomv1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{ + Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, { + Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{ Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime:test"}}}, }, }}, @@ -309,7 +309,7 @@ func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { t.Log("Index consumed regular and init-container references, deduplicating shared aliases") source := newLPXTestSource(dynamolpx.PipelineLPX, "build-v2") child := newLPXTestDeployment(t, source) - pod := &dynamolpx.ServingComponent(source).ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + pod := &dynamolpx.ServingComponent(source).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "direct"}, {Name: "template"}} pod.Containers = append(pod.Containers, corev1.Container{Name: "sidecar", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "direct"}}}}) pod.InitContainers = []corev1.Container{ @@ -331,7 +331,7 @@ func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { foreign.Name, foreign.Namespace = "foreign", "other-namespace" unused := source.DeepCopy() unused.Name = "unused" - unusedPod := &dynamolpx.ServingComponent(unused).ComponentRole(nvidiacomv1beta1.ComponentRoleLeader).PodTemplate.Spec + unusedPod := &dynamolpx.ServingComponent(unused).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec unusedPod.Containers = unusedPod.Containers[:1] unusedPod.Containers[0].Resources.Claims, unusedPod.InitContainers = nil, nil claim := &resourcev1.ResourceClaim{ diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index fac0cb7897e1..775d18757e79 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -305,7 +305,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload) ( gpuCliques := make([]*grovev1alpha1.PodCliqueTemplateSpec, 0, 1) for _, component := range dynamolpx.Components(p.dynamoDeployment) { alphaComponent := alphaComponents[component.ComponentName] - agent := component.ComponentRole(v1beta1.ComponentRoleWorker) + agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) lpuDefaults := &imageEntrypointComponentDefaults{ComponentDefaults: &BaseComponentDefaults{}} lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, @@ -316,7 +316,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload) ( lpuTemplate.Labels[dynamolpx.StageLabel] = component.ComponentName lpuTemplate.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" input.Stages[component.ComponentName] = *lpuTemplate - conductor := component.ComponentRole(v1beta1.ComponentRoleLeader) + conductor := component.ComponentRole(v1beta1.ComponentRoleLPXConductor) if component != p.component { continue } diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 55b927fd86c8..fa908631d65b 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -111,7 +111,7 @@ func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "chat-uid"}, Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ {ComponentName: "engine", ComponentType: v1beta1.ComponentTypeLPX, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLeader}, {Name: v1beta1.ComponentRoleWorker}}}, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXConductor}, {Name: v1beta1.ComponentRoleLPXAgent}}}, }}, } name := PCSNameForLPX(source) @@ -141,7 +141,7 @@ func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { t.Log("A draft name cannot change the target-owned resource name budget") source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: strings.Repeat("d", 30), ComponentType: v1beta1.ComponentTypeLPX, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleWorker}}, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent}}, }) require.Equal(t, name, PCSNameForLPX(source)) @@ -172,8 +172,8 @@ func TestLPXInputRevision(t *testing.T) { {ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeLPX, Replicas: ptr.To(int32(2)), LPX: &v1beta1.LPXConfig{BuildID: "hybrid-build"}, Roles: []v1beta1.ComponentRoleSpec{ - {Name: v1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{}}, - {Name: v1beta1.ComponentRoleLeader, PodTemplate: &corev1.PodTemplateSpec{}}, + {Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{}}, + {Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: &corev1.PodTemplateSpec{}}, }}, }}, } @@ -227,28 +227,28 @@ func TestLPXInputRevision(t *testing.T) { lpx.ServingComponent(d).LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":false}`)} }}, {"agent/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).Replicas = ptr.To(int32(4)) + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(int32(4)) }}, {"conductor/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).Replicas = ptr.To(int32(2)) + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) }}, {"agent/image", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "agent:next"}} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "agent:next"}} }}, {"conductor/image", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "conductor:next"}} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "conductor:next"}} }}, {"agent/placement", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Spec.NodeSelector = map[string]string{"lpu": "new"} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeSelector = map[string]string{"lpu": "new"} }}, {"conductor/placement", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Spec.NodeSelector = map[string]string{"gpu": "new"} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.NodeSelector = map[string]string{"gpu": "new"} }}, {"agent/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleWorker).PodTemplate.Labels = map[string]string{"role": "new"} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Labels = map[string]string{"role": "new"} }}, {"conductor/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate.Annotations = map[string]string{"role": "new"} + lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Annotations = map[string]string{"role": "new"} }}, {"scheduling/deadline", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To(int64(60)) @@ -331,7 +331,7 @@ func TestLPXInputRevision(t *testing.T) { target.Replicas = ptr.To(int32(1)) draft := target.DeepCopy() draft.ComponentName = "small-model" - draft.Roles = []v1beta1.ComponentRoleSpec{*draft.ComponentRole(v1beta1.ComponentRoleWorker)} + draft.Roles = []v1beta1.ComponentRoleSpec{*draft.ComponentRole(v1beta1.ComponentRoleLPXAgent)} pair.Spec.Components = append(pair.Spec.Components, *draft) pairRevision, err := LPXInputRevision(pair, "") require.NoError(t, err) @@ -356,7 +356,7 @@ func TestLPXInputRevisionTracksIndirectRenderMetadata(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) component := lpx.ServingComponent(source) - role := lpxRoleComponent(component, component.ComponentRole(v1beta1.ComponentRoleLeader).PodTemplate, source, "") + role := lpxRoleComponent(component, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate, source, "") role.ComponentType = v1beta1.ComponentTypeDecode metadata := generatePodMetadata(role, source, getDGDAlphaComponent(source, component.ComponentName), component.ComponentName, DiscoveryContext{}) before, err := LPXInputRevision(source, "") diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 4a4df561183b..9944f1adfc7e 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -220,7 +220,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { t.Log("Keep independent draft and target templates, with the conductor seeded by target") draft := testLPXComponent(draftStageName, "draft-build", - v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) draft.Replicas = ptr.To(int32(2)) draft.LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{ @@ -228,13 +228,13 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { "setup":{"agent_connect_timeout":"30s","agent_setup_timeout":"180s"} }`)} target := testLPXComponent(testTargetStageName, "target-build", - v1beta1.ComponentRoleSpec{Name: "leader"}, - v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) source := newSelectedTestDGD(t, "specdecode", draft, target) stages := make(map[string]corev1.PodTemplateSpec) for _, component := range source.Spec.Components { - template := component.ComponentRole("worker").PodTemplate + template := component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate template.Labels = map[string]string{"owner": component.ComponentName} template.Annotations = map[string]string{"owner": component.ComponentName} template.Spec.Containers[0].Image = component.ComponentName + "-runtime" diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index 6aa432b89298..d250d0829c26 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -107,7 +107,7 @@ func ResolveSelectedWorkload( // Component geometry fixes the Agent count independently of draft fanout. componentProjections := projected[len(projections):] - if count := stage.ComponentRole(dynamov1beta1.ComponentRoleWorker).Replicas; count != nil && int(*count) != componentProjections[0].agentReplicas { + if count := stage.ComponentRole(dynamov1beta1.ComponentRoleLPXAgent).Replicas; count != nil && int(*count) != componentProjections[0].agentReplicas { return nil, fmt.Errorf("component %q agent replicas %d must match the compiled count %d", configuredModel, *count, componentProjections[0].agentReplicas) } @@ -145,13 +145,13 @@ func validateSelectedConductor( pipeline Pipeline, compilationMode BuildCompilationMode, ) error { - conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLeader) + conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) // Hybrid execution is selected by immutable build metadata, not template presence. if compilationMode == BuildCompilationModeHybrid { if pipeline == PipelineSpecDecode { return fmt.Errorf("%w: the shared speculative runtime requires LPU-only builds", ErrUnsupportedRuntime) } - template := component.ComponentRole(dynamov1beta1.ComponentRoleWorker).PodTemplate + template := component.ComponentRole(dynamov1beta1.ComponentRoleLPXAgent).PodTemplate if conductor != nil && conductor.PodTemplate != nil { template = conductor.PodTemplate } @@ -178,7 +178,9 @@ func validateSelectedConductor( componentIndex := slices.IndexFunc(dgd.Spec.Components, func(candidate dynamov1beta1.DynamoComponentDeploymentSharedSpec) bool { return candidate.ComponentName == component.ComponentName }) - roleIndex := slices.IndexFunc(component.Roles, func(role dynamov1beta1.ComponentRoleSpec) bool { return role.Name == dynamov1beta1.ComponentRoleLeader }) + roleIndex := slices.IndexFunc(component.Roles, func(role dynamov1beta1.ComponentRoleSpec) bool { + return role.Name == dynamov1beta1.ComponentRoleLPXConductor + }) for containerIndex := range conductor.PodTemplate.Spec.Containers { container := &conductor.PodTemplate.Spec.Containers[containerIndex] if container.Name != commonconsts.MainContainerName { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index dc263657049e..6eeeb7fa521a 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -22,7 +22,7 @@ import ( func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { t.Log("Create one LPX component beside an unrelated conventional decode") - dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")})) + dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")})) dgd.Annotations = nil dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "ordinary-decode", ComponentType: v1beta1.ComponentTypeDecode, @@ -54,8 +54,8 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin source := staticBuildSnapshotSource{"build": snapshot} _, err = ResolveSelectedWorkload(t.Context(), dgd, source) require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") - dgd.Spec.Components[0].Roles = append(dgd.Spec.Components[0].Roles, v1beta1.ComponentRoleSpec{Name: "leader"}) - conductor := dgd.Spec.Components[0].ComponentRole("leader") + dgd.Spec.Components[0].Roles = append(dgd.Spec.Components[0].Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}) + conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) conductor.PodTemplate = testLPXPodTemplate("cyborg-runtime") _, err = ResolveSelectedWorkload(t.Context(), dgd, source) require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") @@ -90,7 +90,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Empty(t, scheduledPlan.ConductorTemplate) t.Log("Use the same agent template when the standalone conductor role is omitted") - agent := *dgd.Spec.Components[0].ComponentRole("worker") + agent := *dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent) agent.PodTemplate = conductor.PodTemplate.DeepCopy() dgd.Spec.Components[0].Roles = []v1beta1.ComponentRoleSpec{agent} before := dgd.DeepCopy() @@ -134,8 +134,8 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Build a selected SpecDecode DGD for the fixture's manifest generation") dgd := newSelectedTestDGD(t, "specdecode", - testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: "leader"}, v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")}), - testLPXComponent("small", "draft-build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("lpu-runtime")}), + testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), + testLPXComponent("small", "draft-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), ) dgd.Spec.Components[0].LPX.Settings = settings dgd.Spec.Components[1].LPX.Settings = settings.DeepCopy() @@ -144,7 +144,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { if test.family == BuildFamilyXT { compiledAgentCount = 4 } - dgd.Spec.Components[1].ComponentRole("worker").Replicas = ptr.To(compiledAgentCount) + dgd.Spec.Components[1].ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(compiledAgentCount) source := staticBuildSnapshotSource{ "draft-build": draftSnapshot, "target-build": targetSnapshot, @@ -198,7 +198,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Agent replica assertions count one compiled model instance, not draft fanout") invalidCount := dgd.DeepCopy() - invalidCount.Spec.Components[1].ComponentRole("worker").Replicas = ptr.To(compiledAgentCount * 2) + invalidCount.Spec.Components[1].ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(compiledAgentCount * 2) _, err = ResolveSelectedWorkload(t.Context(), invalidCount, source) require.ErrorContains(t, err, "must match the compiled count") @@ -272,8 +272,8 @@ func TestResolveConductorLaunchErrorUsesAuthoredRolePath(t *testing.T) { conductor := testLPXPodTemplate("conductor-runtime") conductor.Spec.Containers[0].Args = []string{"--allocation=forged"} dgd := newSelectedTestDGD(t, "specdecode", - testLPXComponent("large", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("target-runtime")}, v1beta1.ComponentRoleSpec{Name: "leader", PodTemplate: conductor}), - testLPXComponent("small", "build", v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("draft-runtime")}), + testLPXComponent("large", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: conductor}), + testLPXComponent("small", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft-runtime")}), ) snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) before := dgd.DeepCopy() @@ -296,8 +296,8 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule, }} dgd := newSelectedTestDGD(t, "selected", testLPXComponent("lpx", "build", - v1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: agent}, - v1beta1.ComponentRoleSpec{Name: "leader", PodTemplate: conductor}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: agent}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: conductor}, )) t.Log("Aggregate every actionable error at its authored path before acquiring a build") @@ -316,6 +316,6 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes t.Log("Reject missing main containers for both roles before acquiring a build") agent.Spec.Containers, conductor.Spec.Containers = nil, nil _, err = ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) - require.ErrorContains(t, err, `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX worker component requires a "main" runtime container`) - require.ErrorContains(t, err, `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX leader component requires a "main" runtime container`) + require.ErrorContains(t, err, `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX agent component requires a "main" runtime container`) + require.ErrorContains(t, err, `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX conductor component requires a "main" runtime container`) } diff --git a/deploy/operator/internal/dynamo/lpx/selection.go b/deploy/operator/internal/dynamo/lpx/selection.go index be208148f72a..7708ee5a5db4 100644 --- a/deploy/operator/internal/dynamo/lpx/selection.go +++ b/deploy/operator/internal/dynamo/lpx/selection.go @@ -29,14 +29,14 @@ func Components(dgd *dynamov1beta1.DynamoGraphDeployment) []*dynamov1beta1.Dynam components = append(components, component) } } - if len(components) == 2 && components[0].ComponentRole(dynamov1beta1.ComponentRoleLeader) != nil { + if len(components) == 2 && components[0].ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { components[0], components[1] = components[1], components[0] } return components } // ServingComponent returns the sole LPX component or the component declaring -// the shared leader role. It returns nil when a pair has no leader owner. +// the shared conductor role. It returns nil when a pair has no conductor owner. // dgd is non-nil and the returned component is read-only. func ServingComponent(dgd *dynamov1beta1.DynamoGraphDeployment) *dynamov1beta1.DynamoComponentDeploymentSharedSpec { var soleComponent *dynamov1beta1.DynamoComponentDeploymentSharedSpec @@ -46,7 +46,7 @@ func ServingComponent(dgd *dynamov1beta1.DynamoGraphDeployment) *dynamov1beta1.D if !component.IsLPX() { continue } - if component.ComponentRole(dynamov1beta1.ComponentRoleLeader) != nil { + if component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { return component } soleComponent = component diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go index 10810bf283f5..8b66040d9060 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_test.go +++ b/deploy/operator/internal/dynamo/lpx/selection_test.go @@ -42,7 +42,7 @@ func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { components: canonicalLPX, want: true, }, - {name: "non-LPX", components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: "worker"}}}, + {name: "non-LPX", components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: dynamov1beta1.ComponentRoleLPXAgent}}}, } for _, test := range tests { @@ -68,8 +68,8 @@ func TestSelectedModelNames(t *testing.T) { draftReplicas := int32(3) dgd := &dynamov1beta1.DynamoGraphDeployment{Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ Components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{ - testLPXComponent("small", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("draft")}), - testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: "leader"}, dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("target")}), + testLPXComponent("small", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}), + testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor}, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}), }, }} dgd.Spec.Components[0].Replicas = &draftReplicas @@ -103,8 +103,8 @@ func TestSelectedModelNames(t *testing.T) { func TestSelectedModelNamesRequiresSharedConductorOwner(t *testing.T) { t.Log("Read two agent-only components without a shared conductor owner") dgd := newSelectedTestDGD(t, "missing-conductor", - testLPXComponent("small", "build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("agent")}), - testLPXComponent("large", "build", dynamov1beta1.ComponentRoleSpec{Name: "worker", PodTemplate: testLPXPodTemplate("agent")}), + testLPXComponent("small", "build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}), + testLPXComponent("large", "build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}), ) before := dgd.DeepCopy() diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index 42bc46fa89d5..6bdb9dea7d2c 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -75,7 +75,7 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC continue } componentPath := componentsPath.Index(index) - conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLeader) + conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) if conductor != nil { conductorCount++ } @@ -96,7 +96,7 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC } } if componentCount == 2 && conductorCount != 1 { - allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one leader role")) + allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one conductor role")) } return allErrs } @@ -131,10 +131,10 @@ func validateSelectedLPXComponent(component *dynamov1beta1.DynamoComponentDeploy rolePath := componentPath.Child("roles").Index(index) main, mainPath, errs := validateSelectedRuntimeContainer(role.PodTemplate, "LPX "+role.Name, rolePath) allErrs = append(allErrs, errs...) - if main != nil && role.Name == dynamov1beta1.ComponentRoleWorker { + if main != nil && role.Name == dynamov1beta1.ComponentRoleLPXAgent { allErrs = append(allErrs, validateAllocationInjectionTargetFields(main, mainPath)...) } - allErrs = append(allErrs, validateLPXRolePlacement(&role.PodTemplate.Spec, rolePath.Child("podTemplate", "spec"), role.Name == dynamov1beta1.ComponentRoleWorker)...) + allErrs = append(allErrs, validateLPXRolePlacement(&role.PodTemplate.Spec, rolePath.Child("podTemplate", "spec"), role.Name == dynamov1beta1.ComponentRoleLPXAgent)...) } return allErrs } diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index 5833abbf8a45..1a64ce9c8084 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -41,7 +41,7 @@ spec: lpx: buildId: llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c roles: - - name: leader + - name: conductor replicas: 1 podTemplate: spec: @@ -87,7 +87,7 @@ spec: - configMap: name: dgd-lpx-v2-vllm-cyborg-model-card name: cyborg-model-card - - name: worker + - name: agent podTemplate: spec: affinity: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 9adce524b437..f3fdb566ef61 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -40,7 +40,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 name: dgd-lpx-v2-vllm-decode @@ -199,7 +199,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 name: dgd-lpx-v2-vllm-lpu @@ -209,7 +209,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 @@ -225,7 +225,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b nvidia.com/extra-resources-hash: a9c40c6452f37713460176f888205de0185c5478ab3a4aa2a06936caa86ff0c4 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -419,7 +419,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b nvidia.com/extra-resources-hash: d782ea49a4ae530a65fb915ae77c6d4b603d6bee3f5b336a52d026439b8f2d41 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -651,7 +651,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:c17686b52fee750e3fc2887ce0a47c0968ae96093c59c6886b3766a5eb434ae9 + lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index 044507b99025..bb3c89febf59 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v2-connected-lpx roles: - - name: leader + - name: conductor replicas: 1 podTemplate: spec: @@ -37,7 +37,7 @@ spec: resources: claims: - name: gpu - - name: worker + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 3d18af7b9e91..563c94cd9d1a 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -5,7 +5,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 name: node-local-v2-hybrid-decode @@ -45,7 +45,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 name: node-local-v2-hybrid-lpu @@ -55,7 +55,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 @@ -71,7 +71,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d nvidia.com/extra-resources-hash: d410b2a3efc122a1a232c8d3d209b74eda592811c79dad628c5d46d8fac52264 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 @@ -237,7 +237,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d nvidia.com/extra-resources-hash: 24aefcdb8d31a8d6357564441af523737eca563784d5fbe2d770e94f5c2b9c05 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 @@ -420,7 +420,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4858eae64d30bd89a636bddb1f5c38f58773d6e7cc56701db02bfec019891153 + lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 2182ba77c75d..678289555aea 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v2-connected roles: - - name: worker + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 89a7bc892d34..2dc65044719f 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -48,7 +48,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 name: node-local-v2-lpu-only-lpu @@ -58,7 +58,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 @@ -74,7 +74,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -261,7 +261,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -416,7 +416,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:670f86aff47120dca5f0d0a52ce914563c7a9f75a38362c4d93b96c1447602ad + lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index b6d20b1ebaf2..29a4b9bce171 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v2-connected roles: - - name: worker + - name: agent podTemplate: spec: containers: @@ -37,8 +37,8 @@ spec: lpx: buildId: node-local-v2-connected roles: - - name: leader - - name: worker + - name: conductor + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 3b88d7e41336..0c3c970e85f2 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -117,7 +117,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 name: node-local-v2-specdecode-lpu @@ -127,7 +127,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 @@ -143,7 +143,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -338,7 +338,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -572,7 +572,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -806,7 +806,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -1043,7 +1043,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:54f76dd0a4234c27d037af746664a0369369d724c2b336d65a227ce5a071a087 + lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index 9b9d57e4b397..b694a9cbbb8d 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v3-hx-connected-lpx roles: - - name: leader + - name: conductor replicas: 1 podTemplate: spec: @@ -32,7 +32,7 @@ spec: resources: claims: - name: gpu - - name: worker + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 0e2b973df923..74ce34b454c9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -41,7 +41,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 name: node-local-v3-hx-hybrid-lpu @@ -51,7 +51,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 @@ -67,7 +67,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 @@ -210,7 +210,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e nvidia.com/extra-resources-hash: 8803dd921b1a97775b917767487276baa0462067b92142d4456e4409df78ee42 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 @@ -587,7 +587,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:a234fc949b9761ea6f6bf523930025ea27b373bb8b2dfda3dd3b4512b6533c6c + lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index d403e3f6dc37..e28d188b83aa 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v3-hx-connected roles: - - name: worker + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 893b9295dd89..aed36d902729 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -40,7 +40,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 name: node-local-v3-hx-lpu-only-lpu @@ -50,7 +50,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 @@ -66,7 +66,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -299,7 +299,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 @@ -535,7 +535,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:ee0851b8b6dcd45aa3d79b6c5b948d63b03db8fac23ab31cd092dd3122fbeb55 + lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 5424c62339a3..2e1dd2f3a4a4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -15,7 +15,7 @@ spec: lpx: buildId: node-local-v3-hx-sd-draft roles: - - name: worker + - name: agent podTemplate: spec: containers: @@ -40,8 +40,8 @@ spec: lpx: buildId: node-local-v3-hx-sd-target roles: - - name: leader - - name: worker + - name: conductor + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index bf9e5c092df0..b12e227ca246 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -96,7 +96,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 name: node-local-v3-hx-specdecode-lpu @@ -106,7 +106,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 @@ -122,7 +122,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 @@ -357,7 +357,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 @@ -592,7 +592,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 @@ -827,7 +827,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 @@ -1065,7 +1065,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:1319f73247c627faff97dd727becfe09d56c33f6729faa08ba454407c642ebf6 + lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml index 48b8070c5a8f..1574076d33a7 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml @@ -27,8 +27,8 @@ spec: tokenizer_path: llama3.1 vocab_size: 128256 roles: - - name: leader - - name: worker + - name: conductor + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index e91708c3d9e1..7ade260df85c 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -124,7 +124,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 name: dgd-single-v2-lpu @@ -134,7 +134,7 @@ metadata: annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 @@ -150,7 +150,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -349,7 +349,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 @@ -588,7 +588,7 @@ spec: - annotations: lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:4e222502b627cc59e427f91734b8b8b1e869b36c37942ae30bcae2dab56c25aa + lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml index 819f9c7e2dc5..ec1ee0c8ae71 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml @@ -16,8 +16,8 @@ spec: lpx: buildId: single-v2-manifest-defaults/build_manifest_defaults roles: - - name: leader - - name: worker + - name: conductor + - name: agent podTemplate: spec: containers: diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 22c9da15eeff..8a0cdf279249 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -145,7 +145,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles[1].Name = "unknown" }), - wantWebhookErrs: []string{`spec.components[0].roles[1].name: Unsupported value: "unknown": supported values: "leader", "worker"`}, + wantWebhookErrs: []string{`spec.components[0].roles[1].name: Unsupported value: "unknown": supported values: "conductor", "agent"`}, }, { name: "v1alpha1 LPX image errors aggregate at the authored role indices", @@ -167,7 +167,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { component := &dgd.Spec.Components[0] component.Roles[1] = *component.Roles[0].DeepCopy() }), - wantSchemaErr: `spec.components[0].roles[1]: Duplicate value: map[string]interface {}{"name":"worker"}`, + wantSchemaErr: `spec.components[0].roles[1]: Duplicate value: map[string]interface {}{"name":"agent"}`, }, { name: "physical LPU rejects Grove opt out", @@ -196,7 +196,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles[0].PodTemplate = nil }), - wantWebhookErrs: []string{"spec.components[0].roles[0].podTemplate: Required value: the LPX worker role requires a podTemplate"}, + wantWebhookErrs: []string{"spec.components[0].roles[0].podTemplate: Required value: the LPX agent role requires a podTemplate"}, }, { name: "LPX requires a build for an LPU engine", @@ -231,7 +231,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{"spec.components[0].roles[0].providerOverride: Forbidden: LPX roles do not support provider overrides"}, }, { - name: "LPX can remove an explicit leader and retain its implicit fallback", + name: "LPX can remove an explicit conductor and retain its implicit fallback", oldDeployment: betaLPXDGDForAdmission(nil), deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles = dgd.Spec.Components[0].Roles[:1] @@ -314,18 +314,18 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "lpx requires a worker role", + name: "lpx requires a agent role", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles = nil }), - wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "worker" role`}, + wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "agent" role`}, }, { - name: "v1alpha1 lpx requires a worker role", + name: "v1alpha1 lpx requires an agent role", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Roles = nil }), - wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "worker" role`}, + wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "agent" role`}, }, { name: "v1alpha1 lpx requires lpx", @@ -413,10 +413,10 @@ func betaLPXDGDForAdmission( Replicas: k8sptr.To(int32(1)), LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"}, Roles: []nvidiacomv1beta1.ComponentRoleSpec{ - {Name: nvidiacomv1beta1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + {Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, }}}, - {Name: nvidiacomv1beta1.ComponentRoleLeader}, + {Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, }, }, } @@ -437,10 +437,10 @@ func alphaLPXDGDForAdmission( Replicas: k8sptr.To(int32(1)), LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"}, Roles: []nvidiacomv1alpha1.ComponentRoleSpec{ - {Name: nvidiacomv1alpha1.ComponentRoleWorker, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + {Name: nvidiacomv1alpha1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, }}}, - {Name: nvidiacomv1alpha1.ComponentRoleLeader}, + {Name: nvidiacomv1alpha1.ComponentRoleLPXConductor}, }, }, } diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index 5dd1c1fbdbab..ac4f7843cc59 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -20,6 +20,7 @@ package validation import ( "context" "fmt" + "slices" "strings" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -380,7 +381,8 @@ func (v *sharedValidation) validateComponentRoles( supportedRoles := []string{nvidiacomv1beta1.ComponentRoleLeader, nvidiacomv1beta1.ComponentRoleWorker} requiredRoles := supportedRoles if component.IsLPX() { - requiredRoles = []string{nvidiacomv1beta1.ComponentRoleWorker} + supportedRoles = []string{nvidiacomv1beta1.ComponentRoleLPXConductor, nvidiacomv1beta1.ComponentRoleLPXAgent} + requiredRoles = []string{nvidiacomv1beta1.ComponentRoleLPXAgent} } // Validate each authored role once, preserving its index in every error path. @@ -389,6 +391,9 @@ func (v *sharedValidation) validateComponentRoles( role := &component.Roles[i] rolePath := fldPath.Index(i) scope, knownRole := provideroverride.ScopeForComponentRole(role.Name) + if component.IsLPX() { + knownRole = slices.Contains(supportedRoles, role.Name) + } if !knownRole { allErrs = append(allErrs, field.NotSupported( rolePath.Child("name"), @@ -468,8 +473,8 @@ func (v *sharedValidation) validateComponentRoleSpec( if options.component != nil && options.component.IsLPX() { if role.PodTemplate != nil { allErrs = append(allErrs, v.validateLPXPodTemplateSpec(role.PodTemplate, fldPath.Child("podTemplate"))...) - } else if role.Name == nvidiacomv1beta1.ComponentRoleWorker { - allErrs = append(allErrs, field.Required(fldPath.Child("podTemplate"), "the LPX worker role requires a podTemplate")) + } else if role.Name == nvidiacomv1beta1.ComponentRoleLPXAgent { + allErrs = append(allErrs, field.Required(fldPath.Child("podTemplate"), "the LPX agent role requires a podTemplate")) } if role.ProviderOverride != nil { allErrs = append(allErrs, field.Forbidden(fldPath.Child("providerOverride"), "LPX roles do not support provider overrides")) diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index fa91e5c3006f..fbe070a209d2 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -267,7 +267,7 @@ spec: lpx: buildId: my-model/build roles: - - name: worker + - name: agent podTemplate: spec: containers: @@ -282,9 +282,9 @@ spec: claimName: model-storage ``` -Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with a build available through the configured LPX model registry and `` with its compatible runtime image. The `worker` role configures Agent Pods; the optional `leader` role configures the conductor. With one LPX component, the conductor reuses the worker template unless the leader supplies a separate `podTemplate`. For a hybrid engine, configure the hybrid runtime template on the leader role. +Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with a build available through the configured LPX model registry and `` with its compatible runtime image. The `agent` role configures Agent Pods; the optional `conductor` role configures the conductor. With one LPX component, the conductor reuses the agent template unless the conductor supplies a separate `podTemplate`. For a hybrid engine, configure the hybrid runtime template on the conductor role. -For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only a `worker` role; the target has `leader` and `worker` roles. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. +For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only an `agent` role; the target has `conductor` and `agent` roles. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. For per-backend setup and tuning, see [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/overview.md), [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/overview.md), and [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/overview.md). diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 956dd78fdf57..71cbb0fdc622 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -339,7 +339,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -387,7 +387,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -1966,7 +1966,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| @@ -2001,7 +2001,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 53bc540303be..9d67a1bedd87 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -61,7 +61,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c DGD-only configuration for a `type: lpx` component; omitted from the standalone DCD schema. Set the required `buildId` and optional build `settings`. See [LPXConfig](full-api-reference.mdx#lpxconfig) for the full field set. - Supported modes are LPU, hybrid, and LPU draft with LPU target. Declare one complete component, or separate speculative draft and target components. The draft has only a `worker` role; the target has `leader` and `worker` roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. + Supported modes are LPU, hybrid, and LPU draft with LPU target. Declare one complete component, or separate speculative draft and target components. The draft has only an `agent` role; the target has `conductor` and `agent` roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. @@ -73,7 +73,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Pod template for the component's pods. The operator injects its defaults (image, command, env, ports, probes, resources, volume mounts) into the container named `main`, merging your overrides by name. If no `main` container is present, the operator generates one. Every other container is treated as a user-managed sidecar and receives no injected defaults — sidecars must specify their own required fields such as `image`. Replaces the ten separate per-component fields (`resources`, `envs`, `livenessProbe`, and so on) that existed in `v1alpha1`. - For `type: lpx`, omit this field and use `roles[].podTemplate`. Each supplied LPX template requires an explicit `main` container image; see `roles` below for leader-template fallback. + For `type: lpx`, omit this field and use `roles[].podTemplate`. Each supplied LPX template requires an explicit `main` container image; see `roles` below for conductor-template fallback. core/v1.PodTemplateSpec @@ -91,24 +91,24 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Optional explicit structure of a compound component. Multinode components must contain exactly one `leader` and one `worker`. Omit the field to retain the implicit leader/worker layout. - LPX components use `worker` for Agent Pods and `leader` for the conductor. With one LPX component, the leader role is optional. In a speculative pair, the draft has only a worker role and the target declares both roles. Hybrid execution may also create Nova Pods from Agent configuration. + LPX components use `agent` for Agent Pods and `conductor` for the conductor. With one LPX component, the conductor role is optional. In a speculative pair, the draft has only an agent role and the target declares both roles. Hybrid execution may also create Nova Pods from Agent configuration. Standalone DCD roles accept `name` and `replicas`. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). - Semantic role name, either `leader` or `worker`. Names must be unique within the component. + Semantic role name: `leader` or `worker` for ordinary multinode components, `conductor` or `agent` for LPX components. Names must be unique within the component. - Native Pod template for this role. LPX-only; required for the worker. Each supplied template requires a `main` container with an explicit image. An omitted leader template reuses the same component's worker template; a single LPX component also uses this fallback when its leader role is omitted. A speculative pair uses the target's leader. An explicit leader template merges with shared defaults independently, without inheriting the worker template. + Native Pod template for this role. LPX-only; required for the agent. Each supplied template requires a `main` container with an explicit image. An omitted conductor template reuses the same component's agent template; a single LPX component also uses this fallback when its conductor role is omitted. A speculative pair uses the target's conductor. An explicit conductor template merges with shared defaults independently, without inheriting the agent template. core/v1.PodTemplateSpec Logical cardinality of the role in one complete component instance. For multinode, admission defaults and persists an omitted value as `leader: 1` or `worker: multinode.nodeCount - 1`; an explicit value must match that fixed shape. - For LPX, omitted leader replicas use `1`, subject to runtime-specific limits. Worker replicas are derived from the model build; an explicit count must match the build. + For LPX, omitted conductor replicas use `1`, subject to runtime-specific limits. Agent replicas are derived from the model build; an explicit count must match the build. Provider configuration for the workload unit generated for this role. Available only on ordinary multinode components embedded in a DGD. LPX roles do not support it; standalone DCD OpenAPI omits this field. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index ed83a723d65e..41e925ab7e30 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -477,7 +477,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. See [ComponentRoleSpec](#componentrolespec). **Validation:** Optional: \{\} @@ -656,7 +656,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} @@ -2777,7 +2777,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. See [ComponentRoleSpec](#v1beta1-componentrolespec). **Validation:** Optional: \{\} @@ -2898,7 +2898,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require a worker role and may declare a leader role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} From 7d9d8532bd3955dc37235c7599bcb4228c8f829b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:26:41 +0300 Subject: [PATCH 013/196] fix(operator): reserve LPX role container names (#432) Signed-off-by: Andrew Paprotsky (cherry picked from commit ee4c42ac02da237e9f991743a1da9f2b33bc96ae) Signed-off-by: Andrew Paprotsky --- .../dynamo/lpx/resolve_selected_workload.go | 36 ++++- .../lpx/resolve_selected_workload_test.go | 147 ++++++++++++++++++ .../dynamo/lpx/selection_validation.go | 50 ++++++ .../validation/dynamographdeployment.go | 4 + ...hdeployment_lpx_validation_envtest_test.go | 59 +++++++ 5 files changed, 289 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index d250d0829c26..8a13171b605c 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -165,6 +165,33 @@ func validateSelectedConductor( } return nil } + + // Only the LPU-only serving template materializes a renamed conductor container. + if component != ServingComponent(dgd) { + return nil + } + + // An omitted conductor role or template inherits the serving agent's template. + role := conductor + if role == nil || role.PodTemplate == nil { + role = component.ComponentRole(dynamov1beta1.ComponentRoleLPXAgent) + } + + // Preserve the authored component and role path when reporting inherited collisions. + componentIndex := slices.IndexFunc(dgd.Spec.Components, func(candidate dynamov1beta1.DynamoComponentDeploymentSharedSpec) bool { + return candidate.ComponentName == component.ComponentName + }) + roleIndex := slices.IndexFunc(component.Roles, func(candidate dynamov1beta1.ComponentRoleSpec) bool { + return candidate.Name == role.Name + }) + podSpecPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec") + + // Check both container lists before a selected workload can be published. + if err := validateRolePodSpecContainerNames(&role.PodTemplate.Spec, podSpecPath, "conductor").ToAggregate(); err != nil { + return err + } + + // Preserve explicit conductor replica and launch validation after checking inherited names. if conductor == nil { return nil } @@ -174,19 +201,14 @@ func validateSelectedConductor( if conductor.PodTemplate == nil { return nil } + // Explicit LPU conductors receive the same appended launch arguments as inherited templates. - componentIndex := slices.IndexFunc(dgd.Spec.Components, func(candidate dynamov1beta1.DynamoComponentDeploymentSharedSpec) bool { - return candidate.ComponentName == component.ComponentName - }) - roleIndex := slices.IndexFunc(component.Roles, func(role dynamov1beta1.ComponentRoleSpec) bool { - return role.Name == dynamov1beta1.ComponentRoleLPXConductor - }) for containerIndex := range conductor.PodTemplate.Spec.Containers { container := &conductor.PodTemplate.Spec.Containers[containerIndex] if container.Name != commonconsts.MainContainerName { continue } - containerPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec", "containers").Index(containerIndex) + containerPath := podSpecPath.Child("containers").Index(containerIndex) if err := validateAllocationInjectionTargetFields(container, containerPath).ToAggregate(); err != nil { return err } diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index 6eeeb7fa521a..d1449a467676 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ) @@ -291,6 +292,8 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes agent.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} agent.Spec.Containers[0].Args = []string{"--allocation=forged", "--"} agent.Spec.Hostname = "custom-host" + agent.Spec.Containers = append(agent.Spec.Containers, corev1.Container{Name: "agent", Image: "sidecar"}) + agent.Spec.InitContainers = append(agent.Spec.InitContainers, corev1.Container{Name: "agent", Image: "setup"}) conductor.Spec.NodeName = "chosen-node" conductor.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{ MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule, @@ -307,6 +310,8 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes "spec.components[0].roles[0].podTemplate.spec.containers[0].args: Forbidden: selected LPX main container must not terminate arguments before Dynamo appends --allocation", "spec.components[0].roles[0].podTemplate.spec.containers[0].command: Forbidden: selected LPX main container cannot use a shell because Dynamo appends --allocation", "spec.components[0].roles[0].podTemplate.spec.hostname: Forbidden: LPX owns role addressing and placement", + `spec.components[0].roles[0].podTemplate.spec.containers[1].name: Forbidden: LPX reserves "agent" for the materialized role container`, + `spec.components[0].roles[0].podTemplate.spec.initContainers[0].name: Forbidden: LPX reserves "agent" for the materialized role container`, "spec.components[0].roles[1].podTemplate.spec.nodeName: Forbidden: LPX owns role addressing and placement", "spec.components[0].roles[1].podTemplate.spec.topologySpreadConstraints: Forbidden: LPX owns role placement", } { @@ -319,3 +324,145 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes require.ErrorContains(t, err, `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX agent component requires a "main" runtime container`) require.ErrorContains(t, err, `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX conductor component requires a "main" runtime container`) } + +func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t *testing.T) { + t.Log("Acquire LPU-only and hybrid builds that select different runtime container identities") + lpuSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) + hybridFixture := newV2CompilerFixture() + hybridFixture.compilationMode = manifestcapnp.CompilationMode_lpx + hybridFixture.selectedPropSyncChains = nil + hybridFixture.partitions = append(hybridFixture.partitions, testV3CapnpPartition{id: 11, deviceType: manifestcapnp.DeviceType_cuda}) + hybridSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, hybridFixture)) + + t.Log("Cover explicit, inherited, draft and hybrid template ownership") + tests := []struct { + name string + conductor *v1beta1.ComponentRoleSpec + addDraft bool + containerInDraft bool + containerInConductor bool + hybrid bool + wantForbidden bool + }{ + {name: "omitted conductor role", wantForbidden: true}, + { + name: "conductor with omitted template", + conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + wantForbidden: true, + }, + { + name: "independent conductor template", + conductor: &v1beta1.ComponentRoleSpec{ + Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor"), + }, + }, + { + name: "explicit LPU-only conductor collision", + conductor: &v1beta1.ComponentRoleSpec{ + Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor"), + }, + containerInConductor: true, + wantForbidden: true, + }, + { + name: "target with omitted conductor template", + conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + addDraft: true, + wantForbidden: true, + }, + { + name: "draft does not supply the conductor template", + conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + addDraft: true, + containerInDraft: true, + }, + {name: "hybrid with omitted conductor role", hybrid: true}, + { + name: "hybrid with omitted conductor template", + conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + hybrid: true, + }, + { + name: "hybrid with explicit conductor template", + conductor: &v1beta1.ComponentRoleSpec{ + Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("cyborg"), + }, + containerInConductor: true, + hybrid: true, + }, + } + + for _, test := range tests { + for _, containerList := range []string{"containers", "initContainers"} { + t.Run(test.name+"/"+containerList, func(t *testing.T) { + t.Log("Author the serving component and its optional independent draft") + target := testLPXComponent("target", "build", + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}, + ) + if test.conductor != nil { + target.Roles = append(target.Roles, *test.conductor.DeepCopy()) + } + dgd := newSelectedTestDGD(t, "selected", target) + if test.addDraft { + dgd.Spec.Components = append(dgd.Spec.Components, testLPXComponent("draft", "build", + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}, + )) + } + + t.Log("Provide GPU resources only when selecting a hybrid runtime") + snapshot := lpuSnapshot + if test.hybrid { + snapshot = hybridSnapshot + for _, role := range dgd.Spec.Components[0].Roles { + if role.PodTemplate != nil { + role.PodTemplate.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceName(commonconsts.KubeResourceGPUNvidia): resource.MustParse("1"), + } + } + } + } + + t.Log("Add one conductor-named container to the selected authored role template") + componentIndex := 0 + if test.containerInDraft { + componentIndex = 1 + } + roleIndex := 0 + if test.containerInConductor { + roleIndex = 1 + } + podSpec := &dgd.Spec.Components[componentIndex].Roles[roleIndex].PodTemplate.Spec + container := corev1.Container{Name: "conductor", Image: "sidecar"} + containerIndex := 0 + if containerList == "initContainers" { + podSpec.InitContainers = append(podSpec.InitContainers, container) + } else { + containerIndex = len(podSpec.Containers) + podSpec.Containers = append(podSpec.Containers, container) + } + before := dgd.DeepCopy() + + t.Log("Defer conductor-name checks until the immutable build has been acquired") + require.Empty(t, ValidateAgentContainerNames(dgd)) + require.Empty(t, ValidateSelectedIntent(dgd)) + _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) + require.ErrorIs(t, err, ErrBuildSnapshotAcquisition) + + t.Log("Reject only actual conductor collisions for the selected runtime") + selected, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": snapshot}) + if test.wantForbidden { + namePath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec", containerList).Index(containerIndex).Child("name") + want := field.Forbidden(namePath, `LPX reserves "conductor" for the materialized role container`) + require.ErrorContains(t, err, want.Error()) + require.NotErrorIs(t, err, ErrBuildSnapshotAcquisition) + } else { + require.NoError(t, err) + if test.hybrid { + require.Equal(t, PipelineLPX, selected.Pipeline()) + } + } + require.Equal(t, before, dgd) + }) + } + } +} diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index 6bdb9dea7d2c..790715bc2c90 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -49,6 +49,8 @@ func validateSelectedIntent(dgd *dynamov1beta1.DynamoGraphDeployment) ([]*dynamo } components := Components(dgd) allErrs = append(allErrs, validateLPXComposition(dgd, len(components))...) + allErrs = append(allErrs, ValidateAgentContainerNames(dgd)...) + // Keep errors tied to authored component and role indices rather than runtime order. for index := range dgd.Spec.Components { component := &dgd.Spec.Components[index] @@ -139,6 +141,54 @@ func validateSelectedLPXComponent(component *dynamov1beta1.DynamoComponentDeploy return allErrs } +// ValidateAgentContainerNames reserves the Agent identity in authored LPX agent +// templates. Conductor identities depend on the immutable build's execution mode. +// dgd must be non-nil and is not mutated. +func ValidateAgentContainerNames(dgd *dynamov1beta1.DynamoGraphDeployment) field.ErrorList { + // Every LPX agent template renames main independently of the selected build. + allErrs := field.ErrorList{} + for componentIndex := range dgd.Spec.Components { + component := &dgd.Spec.Components[componentIndex] + if !component.IsLPX() { + continue + } + + // Keep collision errors on each authored agent template. + for roleIndex, role := range component.Roles { + if role.Name != dynamov1beta1.ComponentRoleLPXAgent || role.PodTemplate == nil { + continue + } + podSpecPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec") + allErrs = append(allErrs, validateRolePodSpecContainerNames(&role.PodTemplate.Spec, podSpecPath, lpuAgentContainerName)...) + } + } + return allErrs +} + +// validateRolePodSpecContainerNames checks both lists that share the Pod's name space. +// spec and fldPath must be non-nil. +func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path, reservedName string) field.ErrorList { + // Regular and init containers must not collide with the renamed main container. + allErrs := field.ErrorList{} + for _, group := range []struct { + name string + containers []corev1.Container + }{ + {"containers", spec.Containers}, + {"initContainers", spec.InitContainers}, + } { + for containerIndex, container := range group.containers { + if container.Name == reservedName { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child(group.name).Index(containerIndex).Child("name"), + fmt.Sprintf("LPX reserves %q for the materialized role container", container.Name), + )) + } + } + } + return allErrs +} + // validateLPXRolePlacement protects controller-owned addressing and scheduling. // Required node affinity remains a supported input to LPX Agent placement. func validateLPXRolePlacement(spec *corev1.PodSpec, fldPath *field.Path, agent bool) field.ErrorList { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 2bb5a81cfae2..26fb3229e8d7 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -27,6 +27,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" @@ -217,6 +218,9 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( } allErrs = append(allErrs, v.validateDynamoGraphDeploymentSpec(&dgd.Spec, field.NewPath("spec"), specOpts)...) + // Reserve the Agent name before build-dependent conductor validation runs in the LPX controller. + allErrs = append(allErrs, dynamolpx.ValidateAgentContainerNames(dgd)...) + // LPX children require the durable Grove route before runtime validation can be deferred. if dgd.HasLPXComponent() && !grovePathway { allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "components"), grovePathwayRequirement)) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 8a0cdf279249..c85981d60c28 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -169,6 +169,65 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), wantSchemaErr: `spec.components[0].roles[1]: Duplicate value: map[string]interface {}{"name":"agent"}`, }, + { + name: "LPX reserves materialized agent container names", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + role := &dgd.Spec.Components[0].Roles[0] + role.PodTemplate.Spec.Containers = append(role.PodTemplate.Spec.Containers, corev1.Container{Name: "agent", Image: "sidecar"}) + role.PodTemplate.Spec.InitContainers = []corev1.Container{{Name: "agent", Image: "setup"}} + }), + wantWebhookErrs: []string{ + `spec.components[0].roles[0].podTemplate.spec.containers[1].name: Forbidden: LPX reserves "agent" for the materialized role container`, + `spec.components[0].roles[0].podTemplate.spec.initContainers[0].name: Forbidden: LPX reserves "agent" for the materialized role container`, + }, + }, + { + name: "LPX defers implicit conductor name checks until build mode is known", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles = component.Roles[:1] + component.Roles[0].PodTemplate.Spec.Containers = append( + component.Roles[0].PodTemplate.Spec.Containers, + corev1.Container{Name: "conductor", Image: "sidecar"}, + ) + }), + }, + { + name: "v1alpha1 LPX defers inherited conductor init names until build mode is known", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + component := dgd.Spec.Services["lpx"] + component.Roles[0].PodTemplate.Spec.InitContainers = []corev1.Container{{Name: "conductor", Image: "setup"}} + }), + }, + { + name: "LPX defers explicit conductor name checks until build mode is known", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles[1].PodTemplate = component.Roles[0].PodTemplate.DeepCopy() + component.Roles[1].PodTemplate.Spec.Containers = append( + component.Roles[1].PodTemplate.Spec.Containers, + corev1.Container{Name: "conductor", Image: "sidecar"}, + ) + }), + }, + { + name: "LPX allows an agent sidecar named conductor with a separate conductor template", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := &dgd.Spec.Components[0] + component.Roles[1].PodTemplate = component.Roles[0].PodTemplate.DeepCopy() + component.Roles[0].PodTemplate.Spec.Containers = append( + component.Roles[0].PodTemplate.Spec.Containers, + corev1.Container{Name: "conductor", Image: "sidecar"}, + ) + }), + }, + { + name: "LPX allows a draft init container named conductor", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To[int32](1)) + dgd.Spec.Components[0].Roles[0].PodTemplate.Spec.InitContainers = []corev1.Container{{Name: "conductor", Image: "setup"}} + }), + }, { name: "physical LPU rejects Grove opt out", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { From 4786dc5641e89947124ed9b2ec14317cfb05324e Mon Sep 17 00:00:00 2001 From: Graden Rea Date: Fri, 11 Sep 2026 00:04:12 +0000 Subject: [PATCH 014/196] fix(operator): use canonical conductor control-plane port flag Replace --start-port with --controlplane-start-port in the generated LPX conductor command and update the rendered YAML fixtures. --start-port was an alias for --controlplane-start-port, so using the canonical name should not change runtime behavior. The port remains 12345. Recent lpu-conductor builds removed the alias, causing startup to fail with an unexpected-argument error. Validation: go test ./internal/dynamo/lpx ./internal/dynamo; go test ./internal/controller/lpx -run ^TestGenerateGrovePodCliqueSet_FromDGDYaml$ (from deploy/operator, with KUBECONFIG=/dev/null). (cherry picked from commit e80a65494d3194bf0e43191db18fd375256810a6) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/conductor_runtime.go | 2 +- .../dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml | 2 +- .../dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml | 2 +- .../testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 2 +- .../testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml | 2 +- .../internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go index 632b3de98d6b..b503a2750e0e 100644 --- a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go @@ -85,7 +85,7 @@ func updateLPUConductorContainer( "60", "--agent-server-retry-sleep-dur", "2s", - "--start-port", + "--controlplane-start-port", "12345", "--graceful-shutdown-timeout", "0s", diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 2dc65044719f..00e36983c684 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -126,7 +126,7 @@ spec: - "60" - --agent-server-retry-sleep-dur - 2s - - --start-port + - --controlplane-start-port - "12345" - --graceful-shutdown-timeout - 0s diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 0c3c970e85f2..c0d722070fbf 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -201,7 +201,7 @@ spec: - "60" - --agent-server-retry-sleep-dur - 2s - - --start-port + - --controlplane-start-port - "12345" - --graceful-shutdown-timeout - 0s diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index aed36d902729..106b83bc547d 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -126,7 +126,7 @@ spec: - "60" - --agent-server-retry-sleep-dur - 2s - - --start-port + - --controlplane-start-port - "12345" - --graceful-shutdown-timeout - 0s diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index b12e227ca246..3af36ccb2330 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -182,7 +182,7 @@ spec: - "60" - --agent-server-retry-sleep-dur - 2s - - --start-port + - --controlplane-start-port - "12345" - --graceful-shutdown-timeout - 0s diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 7ade260df85c..0c22c1e09aa9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -210,7 +210,7 @@ spec: - "60" - --agent-server-retry-sleep-dur - 2s - - --start-port + - --controlplane-start-port - "12345" - --graceful-shutdown-timeout - 0s From 2476aa98c6b0b2bd212d3e0d9e4d3a0e18412726 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:30:59 +0300 Subject: [PATCH 015/196] fix(operator): root LPX children in LGD identity (#433) Signed-off-by: Andrew Paprotsky (cherry picked from commit fdc2a19c2d79f08dfbe74a1ee45c968fc5bab59f) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_watch_setup_test.go | 11 ++- .../dgd_lpx_handoff_envtest_test.go | 6 +- .../controller/lpx/attempt_deadline.go | 2 +- .../controller/lpx/attempt_deadline_test.go | 2 +- .../internal/controller/lpx/controller.go | 58 +++++++++--- .../controller/lpx/controller_test.go | 64 +++++++++++-- .../internal/controller/lpx/dependencies.go | 4 +- .../internal/controller/lpx/eviction.go | 6 +- .../internal/controller/lpx/eviction_test.go | 17 ++-- .../operator/internal/controller/lpx/grove.go | 1 + .../internal/controller/lpx/lifecycle.go | 7 +- .../internal/controller/lpx/lifecycle_test.go | 45 +++++++++- .../internal/controller/lpx/preflight.go | 4 +- .../internal/controller/lpx/render.go | 11 +-- .../internal/controller/lpx/render_test.go | 37 ++++---- .../operator/internal/controller/lpx/watch.go | 41 +++++++-- .../internal/controller/lpx/watch_test.go | 89 +++++++++++++++---- deploy/operator/internal/dynamo/graph_lpx.go | 24 ++--- .../internal/dynamo/graph_lpx_test.go | 47 +++++----- .../internal/dynamo/lpx/cyborg_render_test.go | 8 +- .../internal/dynamo/lpx/grove_ownership.go | 11 ++- .../internal/dynamo/lpx/lpu_configmap.go | 4 +- deploy/operator/internal/dynamo/lpx/render.go | 10 +-- .../internal/dynamo/lpx/render_test.go | 14 +-- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 22 +++-- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 22 +++-- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 13 ++- .../node-local-v2-specdecode.yaml | 19 ++-- .../node-local-v3-hx-hybrid.yaml | 15 ++-- .../node-local-v3-hx-lpu-only.yaml | 13 ++- .../node-local-v3-hx-specdecode.yaml | 19 ++-- .../testdata/from_dgd_yaml/single_v2.yaml | 13 ++- 32 files changed, 466 insertions(+), 193 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go index c7948d876459..ad3f29f2f37c 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -24,9 +24,10 @@ import ( func TestGroveWatchSkipsOnlyVerifiedLPXOwners(t *testing.T) { t.Log("Distinguish LPX-owned Grove roles from ordinary DGD readiness events") const wrongAPIVersion = "unrelated/v1" + const sourceName = "source-graph" scheme := runtime.NewScheme() require.NoError(t, grovev1alpha1.AddToScheme(scheme)) - child := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "graph", UID: "child-uid"}} + child := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "independent-materialization", UID: "child-uid"}} pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ Name: "engine-pcs", Namespace: "workloads", UID: "pcs-uid", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(child, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, @@ -37,8 +38,9 @@ func TestGroveWatchSkipsOnlyVerifiedLPXOwners(t *testing.T) { }} clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ Name: "role", Namespace: "workloads", - Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: child.Name}, + Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: sourceName}, Annotations: map[string]string{ + dynamolpx.DeploymentNameAnnotation: child.Name, lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleConductor, dynamolpx.WorkloadDigestAnnotation: "sha256:workload", dynamo.LPXDeploymentUIDAnnotation: string(child.UID), @@ -100,13 +102,16 @@ func TestGroveWatchSkipsOnlyVerifiedLPXOwners(t *testing.T) { {"stale child stamp", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { c.Annotations[dynamo.LPXDeploymentUIDAnnotation] = "old-child-uid" }}, + {"missing child name stamp", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { + delete(c.Annotations, dynamolpx.DeploymentNameAnnotation) + }}, } { t.Run(test.name, func(t *testing.T) { currentPCS, currentGroup := pcs.DeepCopy(), group.DeepCopy() observed := ready.DeepCopy() test.mutate(observed, currentGroup, currentPCS) reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(currentPCS, currentGroup).Build() - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: observed.Namespace, Name: child.Name}}}, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), observed)) + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: observed.Namespace, Name: sourceName}}}, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), observed)) }) } } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 4a697bea563e..5f5a2ecd6745 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -250,9 +250,9 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") require.NoError(t, env.Client().Delete(t.Context(), quota)) - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(source), Namespace: source.Namespace}} - configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: source.Name + "-lpu", Namespace: source.Namespace}} - service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: dynamo.GetDCDResourceName(source, "lpx", ""), Namespace: source.Namespace}} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(child, source), Namespace: source.Namespace}} + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-lpu", Namespace: source.Namespace}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-lpx", Namespace: source.Namespace}} require.EventuallyWithT(t, func(c *assert.CollectT) { for _, resource := range []client.Object{pcs, configMap, service} { if assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(resource), resource)) { diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index c247adfdac59..ca4eb8eeb904 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -104,7 +104,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( if attempt == nil { return nil, time.Time{}, nil, nil } - pcsName := dynamo.PCSNameForLPX(source) + pcsName := dynamo.PCSNameForLPX(deployment, source) deadlineAt := time.Time{} if attempt.DeadlineAt != nil { deadlineAt = attempt.DeadlineAt.Time diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 382ec4b876c0..3dd9d408aa0c 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -284,7 +284,7 @@ func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { return &grovev1alpha1.PodCliqueSet{ ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.PCSNameForLPX(source), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", + Name: dynamo.PCSNameForLPX(dgd, source), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index a51b6775ef22..09d2ddbed239 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -18,6 +18,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -33,6 +34,32 @@ import ( const lpxGraphDeploymentFinalizer = "nvidia.com/lpx-graph-deployment" +func lpxSourceKey(deployment *v1alpha1.LPXGraphDeployment) (client.ObjectKey, error) { + owner := metav1.GetControllerOf(deployment) + if owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" || owner.Name == "" { + return client.ObjectKey{}, fmt.Errorf("LPXGraphDeployment requires a DynamoGraphDeployment controller owner") + } + return client.ObjectKey{Namespace: deployment.Namespace, Name: owner.Name}, nil +} + +func (r *graphReconciler) resolveLPXSource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) (*v1beta1.DynamoGraphDeployment, *lpxRejected, error) { + sourceKey, err := lpxSourceKey(deployment) + if err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } + source := &v1beta1.DynamoGraphDeployment{} + if err := r.Get(ctx, sourceKey, source); err != nil { + if apierrors.IsNotFound(err) { + return nil, &lpxRejected{reason: "The exact source DGD no longer exists"}, nil + } + return nil, nil, err + } + if err := dynamo.ValidateLPXSource(deployment, source); err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } + return source, nil, nil +} + // graphReconciler owns the complete compiled engine lifecycle. // The source DGD supplies intent, never an alternate status or owner identity. type graphReconciler struct { @@ -155,17 +182,13 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, nil } - source := &v1beta1.DynamoGraphDeployment{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Name}, source); err != nil { - if !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - state = lpxResult(&lpxRejected{reason: "The exact source DGD no longer exists"}) - return r.retireInvalidWorkload(ctx, deployment, state.Message) + source, rejected, sourceErr := r.resolveLPXSource(ctx, deployment) + if sourceErr != nil { + return ctrl.Result{}, sourceErr } - if sourceErr := dynamo.ValidateLPXSource(deployment, source); sourceErr != nil { - state = lpxResult(&lpxRejected{reason: sourceErr.Error()}) - return r.retireInvalidWorkload(ctx, deployment, sourceErr.Error()) + if rejected != nil { + state = lpxResult(rejected) + return r.retireInvalidWorkload(ctx, deployment, rejected.reason) } if !controllerutil.ContainsFinalizer(deployment, lpxGraphDeploymentFinalizer) { controllerutil.AddFinalizer(deployment, lpxGraphDeploymentFinalizer) @@ -335,7 +358,11 @@ func (r *graphReconciler) validateLPXPublicationSource(ctx context.Context, depl return err } source := &v1beta1.DynamoGraphDeployment{} - if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Name}, source); err != nil { + sourceKey, err := lpxSourceKey(deployment) + if err != nil { + return err + } + if err := r.apiReader.Get(ctx, sourceKey, source); err != nil { return err } return dynamo.ValidateLPXSource(deployment, source) @@ -397,10 +424,13 @@ func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alp } func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { + // Use the same normalized identity for endpoint creation and retirement. component := lpx.ServingComponent(source) + serviceName := dynamo.NormalizeKubeResourceName(dynamo.PCSNameForLPX(deployment, source) + "-" + component.ComponentName) + if !commoncontroller.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, source.Annotations) { service := &corev1.Service{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.NormalizeKubeResourceName(dynamo.GetDCDResourceName(source, component.ComponentName, ""))} + key := client.ObjectKey{Namespace: deployment.Namespace, Name: serviceName} if err := r.Get(ctx, key, service); err != nil { return client.IgnoreNotFound(err) } @@ -410,7 +440,7 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a return client.IgnoreNotFound(r.Delete(ctx, service, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &service.UID, ResourceVersion: &service.ResourceVersion}})) } service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ - ServiceName: dynamo.GetDCDResourceName(source, component.ComponentName, ""), Namespace: source.Namespace, + ServiceName: serviceName, Namespace: deployment.Namespace, ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, DynamoNamespace: source.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, Labels: dynamo.GetDGDComponentResourceLabels(source, component.ComponentName, component), @@ -419,6 +449,8 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a if err != nil { return err } + // Keep the endpoint on this materialization even when another child shares the source. service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue + service.Spec.Selector[grovecommon.LabelPartOfKey] = dynamo.PCSNameForLPX(deployment, source) return r.syncLPXResource(ctx, deployment, service) } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 6440f3b0ae74..80e7d8199b61 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -17,6 +17,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -167,12 +168,12 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { _, err := r.Reconcile(t.Context(), request) require.NoError(t, err) pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(source)} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child, source)} require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) pcs.UID = "staged-pcs" require.NoError(t, r.Update(t.Context(), pcs)) endpoint := &corev1.Service{} - endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.GetDCDResourceName(source, "lpx", "")} + endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: pcsKey.Name + "-lpx"} require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) endpoint.UID = "staged-endpoint" require.NoError(t, r.Update(t.Context(), endpoint)) @@ -270,8 +271,8 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T }} foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" - foreignPCS.Name = dynamo.PCSNameForLPX(source) - foreignEndpoint.Name = dynamo.GetDCDResourceName(source, "lpx", "") + foreignPCS.Name = dynamo.PCSNameForLPX(child, source) + foreignEndpoint.Name = foreignPCS.Name + "-lpx" foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences if scenario == "pending finalizer" { pcs.Finalizers = []string{"example.com/staged-cleanup"} @@ -359,13 +360,16 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T } func TestLPXEndpointLifecycle(t *testing.T) { - t.Log("Configure an LPX serving component using Kubernetes discovery") + const uppercaseComponentName = "LPX" + t.Log("Configure an uppercase LPX component and independently named materialization using Kubernetes discovery") source := newLPXTestSource(lpx.PipelineSingle, "test-build") + source.Spec.Components[0].ComponentName = uppercaseComponentName source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) source.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} source.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} source.Spec.Labels = map[string]string{"example.com/policy": "enabled"} child := newLPXTestDeployment(t, source) + child.Name = "independent-materialization" r := newLPXTestReconciler(t, nil, child, source) t.Log("Keep the ordinary model Service under the source DGD's ownership") @@ -386,10 +390,11 @@ func TestLPXEndpointLifecycle(t *testing.T) { t.Log("Publish the LPX-owned endpoint with its serving-role selector") require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) service := &corev1.Service{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.GetDCDResourceName(source, "lpx", "")} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child, source) + "-lpx"} require.NoError(t, r.Get(t.Context(), key, service)) require.True(t, metav1.IsControlledBy(service, child)) require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) + require.Equal(t, dynamo.PCSNameForLPX(child, source), service.Spec.Selector[grovecommon.LabelPartOfKey]) t.Log("Converge propagated endpoint metadata without losing identity, content or sync bookkeeping") for _, value := range []string{"changed", ""} { @@ -429,6 +434,53 @@ func TestLPXEndpointLifecycle(t *testing.T) { require.Equal(t, beforeModelService, modelService, "LPX cleanup must leave the DGD-owned model Service unchanged") } +func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { + t.Log("Render a materialization with a different name from its source DGD") + _, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + child := newLPXTestDeployment(t, source) + child.Name = "independent-materialization" + child.UID = "independent-materialization-uid" + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + + t.Log("Route Grove objects to the materialization while retaining the source provenance") + pcs := objects[0].(*grovev1alpha1.PodCliqueSet) + require.Equal(t, source.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + for _, object := range objects[1:] { + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(child)}}, mapLPXChildToRequests(t.Context(), object)) + } + gangs, err := r.listLPXPublicationPodGangs(t.Context(), child, pcs) + require.NoError(t, err) + require.NotEmpty(t, gangs) + for _, gang := range gangs { + require.Equal(t, source.Name, gang.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + } + + t.Log("Publish the runtime configuration and endpoint under the exact LGD owner") + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + + root := dynamo.PCSNameForLPX(child, source) + for _, object := range []client.Object{ + &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: root, Namespace: child.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: root + "-lpu", Namespace: child.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-" + lpx.ServingComponent(source).ComponentName, Namespace: child.Namespace}}, + } { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) + require.True(t, metav1.IsControlledBy(object, child)) + } + + t.Log("Keep scheduler-request provenance rooted in the source DGD, not the independently named child") + requests, err := r.listOwnedLPXRequests(t.Context(), child) + require.NoError(t, err) + require.NotEmpty(t, requests) + for _, request := range requests { + require.Equal(t, source.Name, request.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + } +} + func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) { t.Log("Publish a runtime ConfigMap under the exact LPX child") source := newLPXTestSource(lpx.PipelineSingle, "test-build") diff --git a/deploy/operator/internal/controller/lpx/dependencies.go b/deploy/operator/internal/controller/lpx/dependencies.go index 83ce91f020ea..a0fdee1c2ab3 100644 --- a/deploy/operator/internal/controller/lpx/dependencies.go +++ b/deploy/operator/internal/controller/lpx/dependencies.go @@ -114,12 +114,12 @@ func (r *graphReconciler) indexedLPXDependencyRequests(ctx context.Context, depe return nil } - // Cluster-scoped events still respect each source's configured namespace filter. + // Respect each source's namespace filter and enqueue its exact owned materializations. requests := make([]ctrl.Request, 0, len(sources.Items)) for i := range sources.Items { source := &sources.Items[i] if commoncontroller.NamespaceAllowed(r.Config, r.runtimeConfig, source, source.Namespace) { - requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)}) + requests = append(requests, r.mapLPXSourceToRequests(ctx, source)...) } } return requests diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index 814dcd981518..eb990a9d6a57 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -129,7 +129,11 @@ func (r *lpuEvictionReconciler) runtimePartitionByPodIndex(ctx context.Context, } // Read the generated runtime partition table independently of authored volume overrides. - configName := dynamolpx.LPUConfigMapName(trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName]) + pcsName := trigger.Labels[grovecommon.LabelPartOfKey] + if pcsName == "" { + return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no PodCliqueSet identity", trigger.Namespace, trigger.Name) + } + configName := dynamolpx.LPUConfigMapName(pcsName) var config corev1.ConfigMap if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: configName}, &config); err != nil { return nil, fmt.Errorf("get LPU runtime ConfigMap %s/%s: %w", trigger.Namespace, configName, err) diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index ca0c739d7803..229c16b7dee4 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -34,6 +34,7 @@ import ( const ( lpuEvictionTestNamespace = "test-ns" lpuEvictionTestDGD = "test-dgd" + lpuEvictionTestPCS = "materialization-12345678" lpuEvictionTestComponent = "lpx-worker" lpuEvictionTestPCSG = "lpx-worker" lpuEvictionTestModel = "test-model" @@ -57,7 +58,7 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl func newLPUEvictionConfigMap() *corev1.ConfigMap { return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: lpuEvictionTestDGD + "-lpu", Namespace: lpuEvictionTestNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS), Namespace: lpuEvictionTestNamespace}, Data: map[string]string{ "nodes_per_partition": "2\n1", "partition_node_offsets": "0\n2", @@ -72,6 +73,7 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting Namespace: lpuEvictionTestNamespace, UID: types.UID(name + "-uid"), Labels: map[string]string{ + grovecommon.LabelPartOfKey: lpuEvictionTestPCS, commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, commonconsts.KubeLabelDynamoComponent: lpuEvictionTestComponent, commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, @@ -203,14 +205,14 @@ func TestLPUEviction_PodsForTrigger(t *testing.T) { } func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) { - t.Log("Create the generated runtime table for a maximum-length DGD name") - dgdName := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) + t.Log("Create the generated runtime table for a bounded materialization name unrelated to the source DGD") + pcsName := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) config := newLPUEvictionConfigMap() - config.Name = dynamolpx.LPUConfigMapName(dgdName) + config.Name = dynamolpx.LPUConfigMapName(pcsName) t.Log("Override the Agent config volume without replacing its generated partition table") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dgdName + trigger.Labels[grovecommon.LabelPartOfKey] = pcsName trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = dynamolpx.LPUConfigMapHash(config) trigger.Spec.Volumes = []corev1.Volume{{ @@ -222,6 +224,11 @@ func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) partitions, err := r.runtimePartitionByPodIndex(t.Context(), trigger) require.NoError(t, err) require.Equal(t, map[string]int{"0": 0, "1": 0, "2": 1}, partitions) + + t.Log("Reject a missing materialization identity instead of falling back to the source DGD name") + delete(trigger.Labels, grovecommon.LabelPartOfKey) + _, err = r.runtimePartitionByPodIndex(t.Context(), trigger) + require.ErrorContains(t, err, "has no PodCliqueSet identity") } func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index c35fe483bb5a..4dd0887b3d5d 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -71,6 +71,7 @@ func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( } for _, key := range [...]string{ dynamolpx.WorkloadDigestAnnotation, + dynamolpx.DeploymentNameAnnotation, lpxDeploymentUIDAnnotation, lpxDeploymentGenerationAnnotation, dynamo.LPXInputRevisionAnnotation, diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index edfe5054557e..7ac928cdc4a4 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -337,7 +337,7 @@ func (r *graphReconciler) prepareLPXMaterializing( return nil, &lpxRejected{reason: err.Error()}, nil } projections := workload.ModelProjections() - plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment, source)) if err != nil { return nil, &lpxRejected{reason: err.Error()}, nil } @@ -1092,7 +1092,7 @@ func (r *graphReconciler) createPublishedLPXRequest( Name: materializing.requestName, Namespace: deployment.Namespace, Labels: map[string]string{ - consts.KubeLabelDynamoGraphDeploymentName: deployment.Name, + consts.KubeLabelDynamoGraphDeploymentName: metav1.GetControllerOf(deployment).Name, }, Annotations: lpxRequestAnnotations(deployment, materializing, grove), }, @@ -1283,8 +1283,7 @@ func (r *graphReconciler) listLPXPublicationPodGangs( }, client.InNamespace(deployment.Namespace), client.MatchingLabels{ - grovecommon.LabelPartOfKey: pcs.Name, - consts.KubeLabelDynamoGraphDeploymentName: deployment.Name, + grovecommon.LabelPartOfKey: pcs.Name, }, ); err != nil { return nil, err diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index cac677a21f7d..2de088c50827 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -665,9 +665,10 @@ func TestLPXAuthoritativeLifecycleListConsumesContinuePages(t *testing.T) { } func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { - t.Log("Prepare the current LPX attempt") + t.Log("Prepare the current LPX attempt with an independently named materialization") ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Name = "metadata-materialization" prepare, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) desired := renderLPXTestPodCliqueSet(t, ctx, prepare, dgd, source, selected) @@ -741,6 +742,41 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.Zero(t, fenceLists) require.Equal(t, cached.UID, synced.UID) + for _, name := range []string{"", "other-materialization"} { + t.Log("Drift only the materialization name while preserving the current workload and source metadata", name) + cached = synced.DeepCopy() + if name == "" { + delete(cached.Annotations, lpx.DeploymentNameAnnotation) + } else { + cached.Annotations[lpx.DeploymentNameAnnotation] = name + } + require.NoError(t, base.Update(ctx, cached)) + before := cached.DeepCopy() + updates = 0 + + t.Log("Restore the name with one metadata-only update and leave the observation untouched") + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) + require.NoError(t, err) + require.True(t, modified) + require.Equal(t, 1, updates) + require.Zero(t, gets) + require.Zero(t, fenceLists) + require.Equal(t, before, cached) + want := before.DeepCopy() + want.Annotations[lpx.DeploymentNameAnnotation] = dgd.Name + want.ResourceVersion = synced.ResourceVersion + require.Equal(t, want, synced) + + t.Log("A repaired materialization name is stable on the next reconciliation") + updates = 0 + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired, selected) + require.NoError(t, err) + require.False(t, modified) + require.Zero(t, updates) + require.Zero(t, fenceLists) + require.Equal(t, want, synced) + } + t.Log("A spec edit after observation conflicts instead of being overwritten by metadata repair") synced.Annotations[lpx.DGDGenerationAnnotation] = "0" require.NoError(t, base.Update(ctx, synced)) @@ -1547,7 +1583,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { request := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) require.False(t, request.DeletionTimestamp.IsZero()) pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(source)}, pcs)) + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child, source)}, pcs)) require.Zero(t, pcs.Spec.Replicas) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.ErrorContains(t, err, "waiting for LPX request") @@ -1817,7 +1853,7 @@ func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { gang := findLPXTestPodGang(t, objects, lpx.SchedulerName) switch test.name { case "LPX gang unavailable": - gang.Labels[consts.KubeLabelDynamoGraphDeploymentName] = lpxTestOtherName + gang.Labels[grovecommon.LabelPartOfKey] = lpxTestOtherName case "LPX gang is stale": gang.Annotations[lpxDeploymentGenerationAnnotation] = "0" case "LPX gang has incomplete references": @@ -2292,6 +2328,7 @@ func newLPXTestReconciler( seed := append([]client.Object{dgd.DeepCopy(), source.DeepCopy()}, objects...) base := fake.NewClientBuilder(). WithScheme(scheme). + WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences). WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimRefIndex, lpxDRAClaimReferences(false)). WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimTemplateRefIndex, lpxDRAClaimReferences(true)). @@ -2358,7 +2395,7 @@ func lpxMaterializedObjects( TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, ObjectMeta: metav1.ObjectMeta{ Name: desired.plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: "lpu-group-uid", - Generation: 1, Labels: maps.Clone(pcs.Labels), Annotations: groupTemplate.Annotations, + Generation: 1, Labels: grovecommon.GetDefaultLabelsForPodCliqueSetManagedResources(pcs.Name), Annotations: groupTemplate.Annotations, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index 8aa9a8bfb9c7..e1e81cfc4ad5 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -55,7 +55,7 @@ func (r *graphReconciler) reconcileLPXSafetyPreflight( if component != serving { continue } - combinedLength := len(dynamo.PCSNameForLPX(source)) + dynamo.LPXComponentNameBudget(component.ComponentName) + combinedLength := len(dynamo.PCSNameForLPX(deployment, source)) + dynamo.LPXComponentNameBudget(component.ComponentName) if combinedLength > consts.MaxCombinedGroveResourceNameLength { err := field.Invalid(field.NewPath("spec", "components").Index(index).Child("name"), component.ComponentName, fmt.Sprintf("combined Grove resource name length %d exceeds the %d-character limit; shorten the deployment or component name", @@ -103,7 +103,7 @@ func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( return nil, nil, fmt.Errorf("failed to list LPX requests after workload resolution: %w", listErr) } if len(requests) > 0 { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, dynamo.PCSNameForLPX(source), &requests[0], rejected.reason) + retiring, retireErr := r.retireLPXRequest(ctx, deployment, dynamo.PCSNameForLPX(deployment, source), &requests[0], rejected.reason) if retireErr != nil { return nil, nil, fmt.Errorf("failed to retire LPX attempt after workload resolution: %w", retireErr) } diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index dafd60856042..3b2d7d6812d4 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -90,11 +90,12 @@ func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha // Preflight validates the exact source controller owner before rendering resources. sourceOwner := metav1.GetControllerOf(deployment) identity := map[string]string{ - lpxDeploymentUIDAnnotation: string(deployment.UID), - lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), - dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, - dynamolpx.DGDUIDAnnotation: string(sourceOwner.UID), - dynamolpx.DGDGenerationAnnotation: deployment.Annotations[dynamolpx.DGDGenerationAnnotation], + dynamolpx.DeploymentNameAnnotation: deployment.Name, + lpxDeploymentUIDAnnotation: string(deployment.UID), + lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), + dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, + dynamolpx.DGDUIDAnnotation: string(sourceOwner.UID), + dynamolpx.DGDGenerationAnnotation: deployment.Annotations[dynamolpx.DGDGenerationAnnotation], } stamp := func(annotations *map[string]string) { if *annotations == nil { diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index b87b6b773d38..9658a8e25207 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -61,7 +61,7 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { source.Annotations = map[string]string{"kai.scheduler/padding": ""} selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} kube := newTestLPXClient(t) @@ -119,7 +119,7 @@ func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { child := newLPXRenderDeployment(t, source) selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} pcs, _, err := renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) @@ -240,7 +240,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { before, childBefore := source.DeepCopy(), child.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{ MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, @@ -327,7 +327,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, child.Spec.InputRevision, revision) equivalentSelected, err := lpx.ResolveSelectedWorkload(t.Context(), equivalent, registry) require.NoError(t, err) - equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(equivalent)) + equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(child, equivalent)) require.NoError(t, err) require.Equal(t, selected.Digest(), equivalentSelected.Digest()) require.Equal(t, plan, equivalentPlan) @@ -449,7 +449,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { before := source.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) require.NoError(t, err) t.Log("Render one shared conductor without changing either component's Agent template") @@ -708,7 +708,7 @@ func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphM func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { t.Helper() - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(dgd)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd), dgd)) require.NoError(t, err) return plan } @@ -798,6 +798,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) require.NoError(t, err) + child := newLPXRenderDeployment(t, &deployment) podCliqueSet, resources, err := renderPodCliqueSet( t.Context(), &deployment, @@ -806,11 +807,11 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { nil, nil, selected, mustPlanSelectedLPX(t, &deployment, selected), - newLPXRenderDeployment(t, &deployment), + child, ) require.NoError(t, err) - configMap := getResource[*corev1.ConfigMap](t, resources, "dgd-single-v2-manifest-defaults-lpu") + configMap := getResource[*corev1.ConfigMap](t, resources, dynamo.PCSNameForLPX(child, &deployment)+"-lpu") var hash string for _, clique := range podCliqueSet.Spec.Template.Cliques { if value := clique.Annotations[commonconsts.AnnotationExtraResourcesHash]; value != "" { @@ -979,7 +980,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { } selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(&dynamoDeployment)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment), &dynamoDeployment)) require.NoError(t, err) t.Log("Render all LPX roles with the separate LPX PCS identity") got, extraResources, err := renderPodCliqueSet( @@ -1112,7 +1113,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test customConfigVolume := corev1.Volume{ Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: dgd.Name + "-lpu"}, + LocalObjectReference: corev1.LocalObjectReference{Name: "authored-runtime-config"}, DefaultMode: ptr.To[int32](0o440), }}, } @@ -1125,7 +1126,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test projections := selected.ModelProjections() require.Len(t, projections, 1) projection := projections[0] - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(dgd)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd), dgd)) require.NoError(t, err) require.Empty(t, plan.ConductorTemplate) require.Len(t, plan.Agents, 1) @@ -1201,7 +1202,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test gasDirSource := envValueSource(main.Env, "GAS_DIR") require.NotNil(t, gasDirSource) require.NotNil(t, gasDirSource.ConfigMapKeyRef) - require.Equal(t, dgd.Name+"-lpu", gasDirSource.ConfigMapKeyRef.Name) + require.Equal(t, plan.PodCliqueSetName+"-lpu", gasDirSource.ConfigMapKeyRef.Name) require.Equal(t, "gas_dir", gasDirSource.ConfigMapKeyRef.Key) require.Equal(t, "topologies", envValueSource(main.Env, "TOPOLOGIES").ConfigMapKeyRef.Key) podIPSource := envValueSource(main.Env, "POD_IP") @@ -1249,7 +1250,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test } configVolume := volumes["config"] require.NotNil(t, configVolume.ConfigMap) - require.Equal(t, dgd.Name+"-decode", configVolume.ConfigMap.Name) + require.Equal(t, plan.PodCliqueSetName+"-decode", configVolume.ConfigMap.Name) require.Nil(t, configVolume.PersistentVolumeClaim) infinibandVolume := volumes["infiniband"] require.NotNil(t, infinibandVolume.HostPath) @@ -1285,19 +1286,19 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NotNil(t, tokenizerEnv) require.NotNil(t, tokenizerEnv.ValueFrom) require.NotNil(t, tokenizerEnv.ValueFrom.ConfigMapKeyRef) - require.Equal(t, dgd.Name+"-decode", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) + require.Equal(t, plan.PodCliqueSetName+"-decode", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) require.Equal(t, "tokenizer_dir", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Key) t.Log("Verify the preserved runtime and LPX projection use the same filtered physical partition view") - lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, dgd.Name+"-lpu") + lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, plan.PodCliqueSetName+"-lpu") require.Equal(t, "1", lpuConfig.Data["partition_ids"]) require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", lpuConfig.Data["gas_dir"]) t.Log("Verify generated Cyborg and LPU config resource order and contents") - decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, dgd.Name+"-decode") + decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, plan.PodCliqueSetName+"-decode") require.Len(t, extraResources, 2) - require.Equal(t, dgd.Name+"-decode", extraResources[0].GetName()) - require.Equal(t, dgd.Name+"-lpu", extraResources[1].GetName()) + require.Equal(t, plan.PodCliqueSetName+"-decode", extraResources[0].GetName()) + require.Equal(t, plan.PodCliqueSetName+"-lpu", extraResources[1].GetName()) require.Equal(t, dgd.Namespace, decodeConfig.Namespace) require.Len(t, decodeConfig.Data, 2) require.Equal(t, []string{"lpu-0-lpu-wkr-m-0-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index f3d7d2a6a282..e96ce799001a 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -19,6 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -28,20 +29,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" ) +const lpxSourceOwnerIndex = "lpx.sourceController" + func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { + // Source events address exact controller owners, not source-named materializations. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences); err != nil { + return fmt.Errorf("register LPX source owner index: %w", err) + } ctrlBuilder := ctrl.NewControllerManagedBy(mgr). For(&nvidiacomv1alpha1.LPXGraphDeployment{}, builder.WithPredicates(lpxDeploymentPrimaryPredicate())). Named("lpxgraphdeployment"). - Watches(&nvidiacomv1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []ctrl.Request { - return []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(obj)}} - }), builder.WithPredicates(lpxSourcePredicate())). + Watches(&nvidiacomv1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(r.mapLPXSourceToRequests), builder.WithPredicates(lpxSourcePredicate())). WithEventFilter(r.lpxControllerEventFilter()) // Primary/source events and finalizer retries are sufficient while disabled. if r.unavailableReason() != "" { @@ -81,6 +85,27 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrlBuilder.Complete(r) } +func lpxSourceOwnerReferences(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.APIVersion != nvidiacomv1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" { + return nil + } + return []string{owner.Name + "/" + string(owner.UID)} +} + +func (r *graphReconciler) mapLPXSourceToRequests(ctx context.Context, obj client.Object) []ctrl.Request { + deployments := &nvidiacomv1alpha1.LPXGraphDeploymentList{} + ownerKey := obj.GetName() + "/" + string(obj.GetUID()) + if err := r.List(ctx, deployments, client.InNamespace(obj.GetNamespace()), client.MatchingFields{lpxSourceOwnerIndex: ownerKey}); err != nil { + return nil + } + requests := make([]ctrl.Request, 0, len(deployments.Items)) + for index := range deployments.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&deployments.Items[index])}) + } + return requests +} + // lpxDeploymentPrimaryPredicate observes every field that grants publication authority. func lpxDeploymentPrimaryPredicate() predicate.Predicate { ownerChanged := predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { @@ -139,17 +164,17 @@ func (r *graphReconciler) mapLPXDRADependencyToRequests(ctx context.Context, obj // mapLPXChildToRequests receives non-nil, namespaced Grove informer objects. func mapLPXChildToRequests(_ context.Context, obj client.Object) []ctrl.Request { - dgdName := obj.GetLabels()[consts.KubeLabelDynamoGraphDeploymentName] - if dgdName == "" { + deploymentName := obj.GetAnnotations()[dynamolpx.DeploymentNameAnnotation] + if deploymentName == "" { return nil } - return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: dgdName}}} + return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: deploymentName}}} } // lpxPodGangPredicate receives non-nil PodGangs from the registered informer. func lpxPodGangPredicate() predicate.Predicate { isSchedulerWitness := func(obj client.Object) bool { - if obj.GetLabels()[consts.KubeLabelDynamoGraphDeploymentName] == "" { + if obj.GetAnnotations()[dynamolpx.DeploymentNameAnnotation] == "" { return false } schedulerName := obj.GetLabels()[grovecommon.LabelSchedulerName] diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 73ee02a4abaf..0ca1a84beb0b 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -56,9 +56,29 @@ func TestLPXDeploymentPrimaryPredicateTracksOwnerIdentity(t *testing.T) { require.True(t, filter.Update(event.UpdateEvent{ObjectOld: deployment, ObjectNew: ownerChanged})) } +func TestLPXSourceWatchMapsControllerOwnedMaterializations(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1alpha1.AddToScheme(scheme)) + require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) + source := &nvidiacomv1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "workloads", UID: "source-uid"}} + owned := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: "materialization", Namespace: source.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + }} + unrelated := owned.DeepCopy() + unrelated.Name = "unrelated" + unrelated.OwnerReferences[0].UID = "other-source-uid" + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(owned, unrelated). + WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences).Build() + reconciler := &graphReconciler{Client: kube} + + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(owned)}}, reconciler.mapLPXSourceToRequests(t.Context(), source)) +} + func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { t.Log("Build matching, unrelated, and out-of-scope graph policies") scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1alpha1.AddToScheme(scheme)) require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) matching := topologyPolicyDGD("matching", "workloads", "fabric") unrelated := topologyPolicyDGD("unrelated", "workloads", "other-fabric") @@ -67,8 +87,27 @@ func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { for _, source := range []*nvidiacomv1beta1.DynamoGraphDeployment{matching, unrelated, foreign} { source.Spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX}} } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(matching, unrelated, foreign, ordinary). + + t.Log("Give each source a differently named materialization and retain a stale same-name child") + objects := []client.Object{matching, unrelated, foreign, ordinary} + for _, source := range []*nvidiacomv1beta1.DynamoGraphDeployment{matching, unrelated, foreign, ordinary} { + source.UID = types.UID(source.Name + "-uid") + objects = append(objects, &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: "materialization-" + source.Name, Namespace: source.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + }}) + } + stale := matching.DeepCopy() + stale.UID = "previous-source-uid" + objects = append(objects, &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: matching.Name, Namespace: matching.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(stale, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + }}) + + t.Log("Index dependency references and exact controller ownership without scanning") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). + WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences). WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { options := (&client.ListOptions{}).ApplyOptions(opts) require.NotNil(t, options.FieldSelector, "topology events must use an index, not scan DGDs") @@ -97,9 +136,9 @@ func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "foreign-cluster-object"}, }})) - t.Log("Requeue only the allowed graph that references the changed binding") + t.Log("Requeue only the allowed materialization owned by the current matching source UID") requests := reconciler.indexedLPXDependencyRequests(context.Background(), binding, lpxTopologyBindingRefIndex) - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "matching"}}}, requests) + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization-matching"}}}, requests) t.Log("Ignore bindings no graph consumes") require.Empty(t, reconciler.indexedLPXDependencyRequests(context.Background(), &grovev1alpha1.ClusterTopologyBinding{ @@ -131,6 +170,7 @@ func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { gang := &groveschedulerv1alpha1.PodGang{ ObjectMeta: metav1.ObjectMeta{ Name: "agents", Namespace: "workloads", UID: "uid", Generation: 1, + Annotations: map[string]string{dynamolpx.DeploymentNameAnnotation: "materialization"}, Labels: map[string]string{ grovecommon.LabelSchedulerName: dynamolpx.SchedulerName, consts.KubeLabelDynamoGraphDeploymentName: "graph", @@ -139,7 +179,7 @@ func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { } require.True(t, predicate.Create(event.CreateEvent{Object: gang})) require.True(t, predicate.Delete(event.DeleteEvent{Object: gang})) - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "graph"}}}, + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization"}}}, mapLPXChildToRequests(t.Context(), gang)) t.Log("Ignore status-only scheduler updates") @@ -160,7 +200,7 @@ func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { ordinary.Labels[grovecommon.LabelSchedulerName] = corev1.DefaultSchedulerName require.True(t, predicate.Create(event.CreateEvent{Object: ordinary})) unrelated := ordinary.DeepCopy() - delete(unrelated.Labels, consts.KubeLabelDynamoGraphDeploymentName) + delete(unrelated.Annotations, dynamolpx.DeploymentNameAnnotation) require.False(t, predicate.Create(event.CreateEvent{Object: unrelated})) require.Empty(t, mapLPXChildToRequests(t.Context(), unrelated)) } @@ -180,13 +220,18 @@ func TestLPXWorkloadEventPredicates(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Observe creation, deletion and all metadata used to fence materialization") old := test.object + old.SetName("grove-child") + old.SetNamespace("workloads") old.SetAnnotations(map[string]string{ + dynamolpx.DeploymentNameAnnotation: "materialization", dynamolpx.WorkloadDigestAnnotation: "sha256:workload", lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, }) require.True(t, test.predicate.Create(event.CreateEvent{Object: old})) require.True(t, test.predicate.Delete(event.DeleteEvent{Object: old})) require.False(t, test.predicate.Generic(event.GenericEvent{Object: old})) + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization"}}}, + mapLPXChildToRequests(t.Context(), old), "Grove children need no source DGD label for routing") t.Log("Ordinary readiness traffic does not wake the LPX controller") unrelated := old.DeepCopyObject().(client.Object) @@ -309,6 +354,7 @@ func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { t.Log("Index consumed regular and init-container references, deduplicating shared aliases") source := newLPXTestSource(dynamolpx.PipelineLPX, "build-v2") child := newLPXTestDeployment(t, source) + child.Name = "materialization" pod := &dynamolpx.ServingComponent(source).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "direct"}, {Name: "template"}} pod.Containers = append(pod.Containers, corev1.Container{Name: "sidecar", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "direct"}}}}) @@ -329,8 +375,10 @@ func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { t.Log("Reference same-named claims and templates from two namespaces, excluding unused aliases") foreign := source.DeepCopy() foreign.Name, foreign.Namespace = "foreign", "other-namespace" + foreign.UID = "foreign-source-uid" unused := source.DeepCopy() unused.Name = "unused" + unused.UID = "unused-source-uid" unusedPod := &dynamolpx.ServingComponent(unused).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec unusedPod.Containers = unusedPod.Containers[:1] unusedPod.Containers[0].Resources.Claims, unusedPod.InitContainers = nil, nil @@ -348,51 +396,60 @@ func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { } foreignTemplate := claimTemplate.DeepCopy() foreignTemplate.Namespace = foreign.Namespace - r := newLPXTestReconciler(t, nil, child, source, foreign, unused, claim, claimTemplate, foreignTemplate) + + t.Log("Route dependencies to current materializations, not same-name children from old source UIDs") + foreignChild := newLPXTestDeployment(t, foreign) + foreignChild.Name = "foreign-materialization" + unusedChild := newLPXTestDeployment(t, unused) + unusedChild.Name = "unused-materialization" + staleChild := child.DeepCopy() + staleChild.Name, staleChild.UID = source.Name, "stale-materialization-uid" + staleChild.OwnerReferences[0].UID = "previous-source-uid" + r := newLPXTestReconciler(t, nil, child, source, foreign, unused, foreignChild, unusedChild, staleChild, claim, claimTemplate, foreignTemplate) r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { options := (&client.ListOptions{}).ApplyOptions(opts) require.NotNil(t, options.FieldSelector, "DRA events must use indexes at both hops") require.False(t, options.FieldSelector.Empty()) return delegated.List(ctx, list, opts...) }}) - sourceRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)} - foreignRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(foreign)} + childRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + foreignRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(foreignChild)} t.Log("Claim events remain namespace-local and unused aliases never enqueue a graph") - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claim)) - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claim)) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) require.Equal(t, []ctrl.Request{foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), foreignTemplate)) require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.ResourceClaimTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: source.Namespace, Name: "unrelated"}})) require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "unrelated"}})) t.Log("Class events join exact and alternative references and enqueue each graph once") for _, class := range []string{"gpu", "alternative-gpu"} { - require.ElementsMatch(t, []ctrl.Request{sourceRequest, foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) + require.ElementsMatch(t, []ctrl.Request{childRequest, foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) } r.Config.Namespace.Restricted = "unwatched-namespace" require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) r.Config.Namespace.Restricted = source.Namespace - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) t.Log("Editing only a direct claim updates its DeviceClass dependency") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claim), claim)) claim.Spec.Devices.Requests[0].Exactly.DeviceClassName = "replacement-claim-gpu" require.NoError(t, r.Update(t.Context(), claim)) - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-claim-gpu"}})) - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-claim-gpu"}})) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) t.Log("A template-only edit removes its old class references without a DGD edit") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claimTemplate), claimTemplate)) claimTemplate.Spec.Spec.Devices.Requests[0].FirstAvailable = []resourcev1.DeviceSubRequest{{Name: "replacement", DeviceClassName: "replacement-template-gpu"}} require.NoError(t, r.Update(t.Context(), claimTemplate)) - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) for _, class := range []string{"gpu", "alternative-gpu"} { require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) } t.Log("Missing templates remain indexed, but no longer contribute a DeviceClass dependency") require.NoError(t, r.Delete(t.Context(), claimTemplate)) - require.Equal(t, []ctrl.Request{sourceRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) + require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) t.Log("Dependency events never mutate the source or its revision; ordinary components are not indexed") diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 775d18757e79..8d7b36492253 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -70,7 +70,7 @@ func RenderLPXBasePodCliqueSet( discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), validatedQueueName: queue, groveClusterTopologyDomains: topologyDomains, - }, selectedWorkload) + }, selectedWorkload, pcsName) if err != nil { return nil, nil, err } @@ -116,7 +116,7 @@ func LPXRestartToken(source *v1beta1.DynamoGraphDeployment, previous string) str // Both objects must be non-nil. func ValidateLPXSource(deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { owner := metav1.GetControllerOf(deployment) - if deployment.Namespace != source.Namespace || deployment.Name != source.Name || + if deployment.Namespace != source.Namespace || owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" || owner.Name != source.Name || owner.UID != source.UID { return fmt.Errorf("LPXGraphDeployment requires its exact source DGD and controller owner") @@ -253,9 +253,9 @@ func lpxSchedulingMetadata(metadata map[string]string) map[string]string { return selected } -// PCSNameForLPX hashes the source identity within the Grove name budget, +// PCSNameForLPX hashes the materialization identity within the Grove name budget, // independently of ordinary components. The LPX planner checks materialized names. -func PCSNameForLPX(dgd *v1beta1.DynamoGraphDeployment) string { +func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment) string { // The serving component owns all generated LPX names, including draft cliques. groupBudget, cliqueBudget := 8, 0 if component := dynamolpx.ServingComponent(dgd); component != nil { @@ -264,8 +264,8 @@ func PCSNameForLPX(dgd *v1beta1.DynamoGraphDeployment) string { } budget := commonconsts.MaxCombinedGroveResourceNameLength - groupBudget - cliqueBudget budget = max(budget, 8) - name := strings.ReplaceAll(dgd.Name, ".", "-") - digest := sha256.Sum256([]byte(dgd.Namespace + "/" + dgd.Name + "/" + string(dgd.UID))) + name := strings.ReplaceAll(deployment.Name, ".", "-") + digest := sha256.Sum256([]byte(deployment.Namespace + "/" + deployment.Name + "/" + string(deployment.UID))) if budget <= 13 { return fmt.Sprintf("%x", digest[:4]) } @@ -287,13 +287,13 @@ func LPXComponentNameBudget(componentName string) int { // authored role. The full source DGD supplies discovery and shared defaults; // LPX component's roles are returned to the same PCS renderer. // Preflight has validated the runtime shape and every Agent template. -func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { +func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, materializationName string) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { // Pass naming and runtime inputs; deployment identity is stamped only on final resources. input := &dynamolpx.RenderInput{ - DGDName: p.dynamoDeployment.Name, - MinAvailable: p.component.MinAvailable, - Stages: make(map[string]corev1.PodTemplateSpec), - SSHSecretName: p.operatorConfig.MPI.SSHSecretName, + MaterializationName: materializationName, + MinAvailable: p.component.MinAvailable, + Stages: make(map[string]corev1.PodTemplateSpec), + SSHSecretName: p.operatorConfig.MPI.SSHSecretName, } // Resolve preserved alpha metadata once for all independently rendered roles. @@ -351,7 +351,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload) ( defaults := ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)) if workload.BuildFamily() == dynamolpx.BuildFamilyXT { defaults = &selectedCyborgComponentDefaults{ - ComponentDefaults: defaults, workload: workload, dgdName: p.dynamoDeployment.Name, + ComponentDefaults: defaults, workload: workload, dgdName: materializationName, replicas: *role.Replicas, lpxPodSpec: lpuTemplate.Spec, } } else { diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index fa908631d65b..de7000056c3f 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -106,7 +106,7 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { } } -func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { +func TestLPXPCSNameUsesStableMaterializationIdentity(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "chat-uid"}, Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ @@ -114,7 +114,8 @@ func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXConductor}, {Name: v1beta1.ComponentRoleLPXAgent}}}, }}, } - name := PCSNameForLPX(source) + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "materialization-uid"}} + name := PCSNameForLPX(deployment, source) t.Log("An ordinary DGD named chat-lpx must not collide with the LPX PCS of chat") ordinary := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat-lpx"}} require.NotEqual(t, PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), name) @@ -122,43 +123,44 @@ func TestLPXPCSNameUsesStableSourceIdentity(t *testing.T) { for _, test := range []struct { name string - mutate func(*v1beta1.DynamoGraphDeployment) + mutate func(*v1alpha1.LPXGraphDeployment) }{ - {"replacement source", func(d *v1beta1.DynamoGraphDeployment) { d.UID = "replacement-chat-uid" }}, - {"different namespace", func(d *v1beta1.DynamoGraphDeployment) { d.Namespace = "another-namespace" }}, + {"replacement materialization", func(d *v1alpha1.LPXGraphDeployment) { d.UID = "replacement-materialization-uid" }}, + {"different namespace", func(d *v1alpha1.LPXGraphDeployment) { d.Namespace = "another-namespace" }}, } { t.Run(test.name, func(t *testing.T) { - changed := source.DeepCopy() + changed := deployment.DeepCopy() test.mutate(changed) - require.NotEqual(t, name, PCSNameForLPX(changed)) + require.NotEqual(t, name, PCSNameForLPX(changed, source)) }) } - t.Log("Replica, generation, and revision bookkeeping must not rename the workload") - source.Generation++ - source.ResourceVersion = "2" + t.Log("Source replacement, graph edits, and materialization bookkeeping must not rename the workload") + source.UID = "replacement-source" source.Spec.Components[0].Replicas = ptr.To(int32(3)) - require.Equal(t, name, PCSNameForLPX(source)) + deployment.Generation++ + deployment.Spec.InputRevision = "sha256:changed" + require.Equal(t, name, PCSNameForLPX(deployment, source)) t.Log("A draft name cannot change the target-owned resource name budget") source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: strings.Repeat("d", 30), ComponentType: v1beta1.ComponentTypeLPX, Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent}}, }) - require.Equal(t, name, PCSNameForLPX(source)) + require.Equal(t, name, PCSNameForLPX(deployment, source)) - source.Name = "chat.example" - dotted := PCSNameForLPX(source) - source.Name = "chat-example" - require.NotEqual(t, dotted, PCSNameForLPX(source), "normalizing dots must not erase source identity") + deployment.Name = "chat.example" + dotted := PCSNameForLPX(deployment, source) + deployment.Name = "chat-example" + require.NotEqual(t, dotted, PCSNameForLPX(deployment, source), "normalizing dots must not erase materialization identity") t.Log("Both readable and compact names fit the complete Grove name budget") for componentLength := 1; componentLength <= 13; componentLength++ { source.Spec.Components[0].ComponentName = strings.Repeat("e", componentLength) - for _, sourceName := range []string{"c", "chat.example", strings.Repeat("long.", 40) + "chat"} { - source.Name = sourceName - got := PCSNameForLPX(source) + for _, materializationName := range []string{"c", "chat.example", strings.Repeat("long.", 40) + "chat"} { + deployment.Name = materializationName + got := PCSNameForLPX(deployment, source) require.Empty(t, validation.IsDNS1123Label(got), got) require.LessOrEqual(t, len(got)+max(componentLength, 8)+componentLength+len("-engine-gpu"), commonconsts.MaxCombinedGroveResourceNameLength) - require.Equal(t, got, PCSNameForLPX(source)) + require.Equal(t, got, PCSNameForLPX(deployment, source)) } } } @@ -177,10 +179,11 @@ func TestLPXInputRevision(t *testing.T) { }}, }}, } + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: "materialization"}} want, err := LPXInputRevision(source, "") require.NoError(t, err) require.Regexp(t, `^sha256:[a-f0-9]{64}$`, want) - pcsName := PCSNameForLPX(source) + pcsName := PCSNameForLPX(deployment, source) require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), pcsName) t.Log("Prefill-only edits and DGD bookkeeping do not alter the LPX revision or PCS name") @@ -194,7 +197,7 @@ func TestLPXInputRevision(t *testing.T) { got, err := LPXInputRevision(source, "") require.NoError(t, err) require.Equal(t, want, got) - require.Equal(t, pcsName, PCSNameForLPX(source)) + require.Equal(t, pcsName, PCSNameForLPX(deployment, source)) t.Log("Each LPX component or shared-input change invalidates the revision; ordinary edits do not") source.Spec.Scheduling = &v1beta1.SchedulingSpec{} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 92e97fa57f6c..f264ce2d1fa0 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -56,8 +56,8 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) input := RenderInput{ - DGDName: "dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + MaterializationName: "dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", } rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, input) require.NoError(t, err) @@ -122,8 +122,8 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { cyborg.Spec.MinAvailable = ptr.To(test.replicas) cyborg.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} _, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - DGDName: "dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + MaterializationName: "dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", }) require.ErrorContains(t, err, test.wantError) }) diff --git a/deploy/operator/internal/dynamo/lpx/grove_ownership.go b/deploy/operator/internal/dynamo/lpx/grove_ownership.go index 6c3b8efe2b3f..7addaf31d996 100644 --- a/deploy/operator/internal/dynamo/lpx/grove_ownership.go +++ b/deploy/operator/internal/dynamo/lpx/grove_ownership.go @@ -7,7 +7,6 @@ import ( "context" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -15,8 +14,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. -const DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" +const ( + // DeploymentNameAnnotation routes rendered-object events to their LPXGraphDeployment. + DeploymentNameAnnotation = "lpx.nvidia.com/deployment-name" + // DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. + DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" +) // OwnsPodClique verifies the LPX owner chain before ordinary DGD watches // suppress a role-readiness event. Authorable annotations alone are not proof. @@ -43,7 +46,7 @@ func OwnsPodClique(ctx context.Context, reader client.Reader, clique *grovev1alp owner = metav1.GetControllerOf(pcs) return owner != nil && owner.Kind == nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind && owner.APIVersion == nvidiacomv1alpha1.GroupVersion.String() && owner.UID != "" && - owner.Name == clique.Labels[consts.KubeLabelDynamoGraphDeploymentName] && + owner.Name == clique.Annotations[DeploymentNameAnnotation] && string(owner.UID) == clique.Annotations[DeploymentUIDAnnotation] } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 59014726d043..b57e60edf4db 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -54,7 +54,7 @@ func LPUConfigMapName(root string) string { func renderLPUConfigMap( namespace string, - dgdName string, + materializationName string, modelStoragePath string, projections []*ModelProjection, agents []ExpectedAgent, @@ -98,7 +98,7 @@ func renderLPUConfigMap( data["datacenter.toml"] = datacenterTOML.String() return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: LPUConfigMapName(dgdName), + Name: LPUConfigMapName(materializationName), Namespace: namespace, }, Data: data, diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 5dd08b05970f..ae8e8021afe3 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -30,10 +30,10 @@ const ( ExecutionRoleLabel = "lpx.nvidia.com/execution-role" ) -// RenderInput contains the source name and fresh stage templates consumed by rendering. +// RenderInput contains the materialization name and fresh stage templates consumed by rendering. type RenderInput struct { - // DGDName is the source DynamoGraphDeployment name. - DGDName string + // MaterializationName is the owning LPXGraphDeployment's bounded identity root. + MaterializationName string // MinAvailable is the minimum number of complete engine replicas in the gang. MinAvailable *int32 // Stages contains an independently merged LPU template for every projected stage. @@ -86,7 +86,7 @@ func RenderSelectedNodeLocal( } modelStorage.volume = *modelStorage.volume.DeepCopy() modelStorage.mount = *modelStorage.mount.DeepCopy() - configMap, err := renderLPUConfigMap(namespace, input.DGDName, modelStorage.mount.MountPath, projections, plan.Agents) + configMap, err := renderLPUConfigMap(namespace, input.MaterializationName, modelStorage.mount.MountPath, projections, plan.Agents) if err != nil { return nil, err } @@ -99,7 +99,7 @@ func RenderSelectedNodeLocal( extraResources []client.Object ) if v2HybridRuntime { - cyborgConfigMap, err = renderSelectedCyborgConfigMap(namespace, input.DGDName, plan, modelStorage.mount.MountPath, &projections[0].configuredBuild) + cyborgConfigMap, err = renderSelectedCyborgConfigMap(namespace, input.MaterializationName, plan, modelStorage.mount.MountPath, &projections[0].configuredBuild) if err != nil { return nil, err } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 9944f1adfc7e..7de0e888ea56 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -101,9 +101,9 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Log("Render into the fresh PCS without retaining stale runtime identity") rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - DGDName: "test-dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, - SSHSecretName: "ssh-secret", + MaterializationName: "test-dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, + SSHSecretName: "ssh-secret", }) require.NoError(t, err) require.Same(t, pcs, rendered) @@ -180,9 +180,9 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { t.Log("Render conductor and Agent roles from the base PodSpec") rendered, err := renderSelectedForTest(renderTestPCS(false), []*ModelProjection{projection}, RenderInput{ - DGDName: "test-dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, - SSHSecretName: "ssh-secret", + MaterializationName: "test-dgd", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, + SSHSecretName: "ssh-secret", }) require.NoError(t, err) @@ -251,7 +251,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { plan, err := selected.PlanNodeLocalMaterialization(pcs.Name) require.NoError(t, err) extraResources, err := RenderSelectedNodeLocal(pcs, selected, plan, RenderInput{ - DGDName: source.Name, Stages: stages, SSHSecretName: "ssh-secret", + MaterializationName: source.Name, Stages: stages, SSHSecretName: "ssh-secret", }) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index f3fdb566ef61..3aceedc3bbc2 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -39,11 +39,12 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-lpx-v2-vllm-decode + name: dgd-lpx-v2-94cdbc7c-lpx-decode namespace: inference-engine --- data: @@ -198,16 +199,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-lpx-v2-vllm-lpu + name: dgd-lpx-v2-94cdbc7c-lpx-lpu namespace: inference-engine --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -215,7 +218,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 labels: nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm - name: dgd-lpx-v2-2c5c68ef-lpx + name: dgd-lpx-v2-94cdbc7c-lpx namespace: inference-engine spec: replicas: 1 @@ -224,6 +227,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b nvidia.com/extra-resources-hash: a9c40c6452f37713460176f888205de0185c5478ab3a4aa2a06936caa86ff0c4 @@ -336,7 +340,7 @@ spec: valueFrom: configMapKeyRef: key: tokenizer_dir - name: dgd-lpx-v2-vllm-decode + name: dgd-lpx-v2-94cdbc7c-lpx-decode - name: TOTAL_REPLICAS value: "1" image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 @@ -406,7 +410,7 @@ spec: type: Directory name: infiniband - configMap: - name: dgd-lpx-v2-vllm-decode + name: dgd-lpx-v2-94cdbc7c-lpx-decode name: config - emptyDir: medium: Memory @@ -418,6 +422,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b nvidia.com/extra-resources-hash: d782ea49a4ae530a65fb915ae77c6d4b603d6bee3f5b336a52d026439b8f2d41 @@ -511,7 +516,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: dgd-lpx-v2-vllm-lpu + name: dgd-lpx-v2-94cdbc7c-lpx-lpu - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -545,7 +550,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: dgd-lpx-v2-vllm-lpu + name: dgd-lpx-v2-94cdbc7c-lpx-lpu - name: UCX_TLS value: tcp image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f @@ -624,7 +629,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-lpu + name: dgd-lpx-v2-94cdbc7c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -650,6 +655,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 563c94cd9d1a..75cb7d5bbfc3 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -4,11 +4,12 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: node-local-v2-hybrid-decode + name: node-local-9d06c48a-lpx-decode namespace: test --- data: @@ -44,16 +45,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: node-local-v2-hybrid-lpu + name: node-local-9d06c48a-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -61,7 +64,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid - name: node-local-9a49489a-lpx + name: node-local-9d06c48a-lpx namespace: test spec: replicas: 1 @@ -70,6 +73,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d nvidia.com/extra-resources-hash: d410b2a3efc122a1a232c8d3d209b74eda592811c79dad628c5d46d8fac52264 @@ -161,7 +165,7 @@ spec: valueFrom: configMapKeyRef: key: tokenizer_dir - name: node-local-v2-hybrid-decode + name: node-local-9d06c48a-lpx-decode - name: TOTAL_REPLICAS value: "1" image: example.invalid/gpu-runtime:test @@ -217,7 +221,7 @@ spec: terminationGracePeriodSeconds: 60 volumes: - configMap: - name: node-local-v2-hybrid-decode + name: node-local-9d06c48a-lpx-decode name: config - hostPath: path: /dev/infiniband @@ -236,6 +240,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d nvidia.com/extra-resources-hash: 24aefcdb8d31a8d6357564441af523737eca563784d5fbe2d770e94f5c2b9c05 @@ -280,7 +285,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-v2-hybrid-lpu + name: node-local-9d06c48a-lpx-lpu - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -314,7 +319,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: node-local-v2-hybrid-lpu + name: node-local-9d06c48a-lpx-lpu - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -393,7 +398,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-lpu + name: node-local-9d06c48a-lpx-lpu name: config - emptyDir: {} name: tmp @@ -419,6 +424,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 00e36983c684..9dc7309b9f01 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -47,16 +47,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: node-local-v2-lpu-only-lpu + name: node-local-33b11b4d-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -64,7 +66,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only - name: node-local-1f9267e3-lpx + name: node-local-33b11b4d-lpx namespace: test spec: replicas: 1 @@ -73,6 +75,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 @@ -244,7 +247,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-lpu + name: node-local-33b11b4d-lpx-lpu name: config - emptyDir: {} name: tmp @@ -260,6 +263,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 @@ -389,7 +393,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-lpu + name: node-local-33b11b4d-lpx-lpu name: config - emptyDir: {} name: tmp @@ -415,6 +419,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index c0d722070fbf..b63aa924614c 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -116,16 +116,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: node-local-v2-specdecode-lpu + name: node-local-63df422c-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -133,7 +135,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode - name: node-local-3bd5df51-lpx + name: node-local-63df422c-lpx namespace: test spec: replicas: 1 @@ -142,6 +144,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a @@ -319,7 +322,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-lpu + name: node-local-63df422c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -337,6 +340,7 @@ spec: - lpu-wkr-m-2 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a @@ -548,7 +552,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-lpu + name: node-local-63df422c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -571,6 +575,7 @@ spec: roleName: lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a @@ -782,7 +787,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-lpu + name: node-local-63df422c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -805,6 +810,7 @@ spec: roleName: lpu-wkr-m-1 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a @@ -1016,7 +1022,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-lpu + name: node-local-63df422c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -1042,6 +1048,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 74ce34b454c9..2c9ac369f0f5 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -40,16 +40,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: node-local-v3-hx-hybrid-lpu + name: node-local-be18433d-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -57,7 +59,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid - name: node-local-b112da53-lpx + name: node-local-be18433d-lpx namespace: test spec: replicas: 1 @@ -66,6 +68,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -209,6 +212,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e nvidia.com/extra-resources-hash: 8803dd921b1a97775b917767487276baa0462067b92142d4456e4409df78ee42 @@ -373,7 +377,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-v3-hx-hybrid-lpu + name: node-local-be18433d-lpx-lpu - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -407,7 +411,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: node-local-v3-hx-hybrid-lpu + name: node-local-be18433d-lpx-lpu - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -560,7 +564,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-hybrid-lpu + name: node-local-be18433d-lpx-lpu name: config - emptyDir: {} name: tmp @@ -586,6 +590,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 106b83bc547d..d2861339e335 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -39,16 +39,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: node-local-v3-hx-lpu-only-lpu + name: node-local-d43cad47-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -56,7 +58,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only - name: node-local-071e1c6d-lpx + name: node-local-d43cad47-lpx namespace: test spec: replicas: 1 @@ -65,6 +67,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 @@ -282,7 +285,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-lpu + name: node-local-d43cad47-lpx-lpu name: config - emptyDir: {} name: tmp @@ -298,6 +301,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 @@ -511,7 +515,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-lpu + name: node-local-d43cad47-lpx-lpu name: config - emptyDir: {} name: tmp @@ -534,6 +538,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 3af36ccb2330..47e27fde7bfc 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -95,16 +95,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: node-local-v3-hx-specdecode-lpu + name: node-local-1f7af409-lpx-lpu namespace: test --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -112,7 +114,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode - name: node-local-75563e58-lpx + name: node-local-1f7af409-lpx namespace: test spec: replicas: 1 @@ -121,6 +123,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 @@ -338,7 +341,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-lpu + name: node-local-1f7af409-lpx-lpu name: config - emptyDir: {} name: tmp @@ -356,6 +359,7 @@ spec: - lpu-wkr-m-2 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 @@ -571,7 +575,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-lpu + name: node-local-1f7af409-lpx-lpu name: config - emptyDir: {} name: tmp @@ -591,6 +595,7 @@ spec: roleName: lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 @@ -806,7 +811,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-lpu + name: node-local-1f7af409-lpx-lpu name: config - emptyDir: {} name: tmp @@ -826,6 +831,7 @@ spec: roleName: lpu-wkr-m-1 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 @@ -1041,7 +1047,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-lpu + name: node-local-1f7af409-lpx-lpu name: config - emptyDir: {} name: tmp @@ -1064,6 +1070,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 scheduling.lpu.nvidia.com/dgd-generation: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 0c22c1e09aa9..1dac01d95906 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -123,16 +123,18 @@ data: metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-single-v2-lpu + name: dgd-single-3a98fe5c-lpx-lpu namespace: inference-engine --- metadata: annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" @@ -140,7 +142,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 labels: nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 - name: dgd-single-c76fa0f7-lpx + name: dgd-single-3a98fe5c-lpx namespace: inference-engine spec: replicas: 1 @@ -149,6 +151,7 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a @@ -332,7 +335,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-lpu + name: dgd-single-3a98fe5c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -348,6 +351,7 @@ spec: - lpu-wkr-m-0 - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a @@ -561,7 +565,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-lpu + name: dgd-single-3a98fe5c-lpx-lpu name: config - emptyDir: {} name: tmp @@ -587,6 +591,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-generation: "1" + lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 scheduling.lpu.nvidia.com/dgd-generation: "1" From 825c7fcd6cf41662c8370cd0495ec0ceb0460a46 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:31:00 +0300 Subject: [PATCH 016/196] fix(operator): preserve LPX engines and immutable runtime config (#434) Signed-off-by: Andrew Paprotsky (cherry picked from commit 84d151f25d46ce4b96488e00b61c4217e0c562e7) Signed-off-by: Andrew Paprotsky --- .../dynamocomponentdeployment_types.go | 2 +- .../v1alpha1/dynamographdeployment_types.go | 2 - .../dynamocomponentdeployment_types.go | 2 +- deploy/operator/api/v1beta1/lpx_types.go | 2 - ...nvidia.com_dynamocomponentdeployments.yaml | 8 +- .../nvidia.com_dynamographdeployments.yaml | 16 +- .../bases/nvidia.com_lpxgraphdeployments.yaml | 2 - .../controller/dgd_lpx_envtest_test.go | 8 + .../dgd_lpx_handoff_envtest_test.go | 10 +- .../controller/lpx/attempt_deadline.go | 274 ++++++---- .../controller/lpx/attempt_deadline_test.go | 391 +++++++++++++- .../controller/lpx/component_roles_test.go | 2 +- .../internal/controller/lpx/controller.go | 80 ++- .../controller/lpx/controller_test.go | 54 +- .../internal/controller/lpx/eviction.go | 8 +- .../internal/controller/lpx/eviction_test.go | 68 ++- .../operator/internal/controller/lpx/grove.go | 67 +-- .../internal/controller/lpx/lifecycle.go | 413 +++++++-------- .../internal/controller/lpx/lifecycle_test.go | 493 +++++++++--------- .../internal/controller/lpx/preflight.go | 52 +- .../internal/controller/lpx/render.go | 10 +- .../internal/controller/lpx/render_test.go | 120 +++-- .../internal/controller/lpx/watch_test.go | 6 +- deploy/operator/internal/dynamo/graph_lpx.go | 51 +- .../internal/dynamo/graph_lpx_test.go | 48 +- deploy/operator/internal/dynamo/grove_lpx.go | 11 +- .../internal/dynamo/lpx/conductor_runtime.go | 14 +- .../lpx/cyborg_launcher_fixture_test.go | 90 ++++ .../internal/dynamo/lpx/cyborg_render.go | 12 +- .../internal/dynamo/lpx/cyborg_render_test.go | 7 +- .../internal/dynamo/lpx/cyborg_runtime.go | 30 +- .../dynamo/lpx/cyborg_runtime_test.go | 13 +- .../dynamo/lpx/hybrid_cyborg_bindings.go | 85 ++- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 29 +- .../internal/dynamo/lpx/lpu_configmap.go | 17 +- .../internal/dynamo/lpx/lpu_configmap_test.go | 8 +- .../internal/dynamo/lpx/materialization.go | 29 +- .../dynamo/lpx/materialization_test.go | 51 ++ .../internal/dynamo/lpx/pod_roles_test.go | 8 +- deploy/operator/internal/dynamo/lpx/render.go | 7 +- .../internal/dynamo/lpx/render_test.go | 14 +- .../dynamo/lpx/resolve_selected_workload.go | 4 - .../lpx/resolve_selected_workload_test.go | 10 + .../internal/dynamo/lpx/runtime_agent.go | 4 +- .../internal/dynamo/lpx/runtime_startup.go | 66 +++ .../dynamo/lpx/runtime_startup_test.go | 68 +++ .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 116 ++--- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 60 ++- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 42 +- .../node-local-v2-specdecode.yaml | 59 +-- .../node-local-v3-hx-hybrid.yaml | 28 +- .../node-local-v3-hx-lpu-only.yaml | 45 +- .../node-local-v3-hx-specdecode.yaml | 59 +-- .../testdata/from_dgd_yaml/single_v2.yaml | 45 +- ...hdeployment_lpx_validation_envtest_test.go | 5 +- 55 files changed, 1879 insertions(+), 1346 deletions(-) create mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_startup.go create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_startup_test.go diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 7b9a53923904..4404ac6dcdcd 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -87,7 +87,7 @@ type DynamoComponentDeploymentSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.componentType) || self.componentType != 'lpu'",message="componentType lpu is not supported; use lpx" // +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.componentType) && self.componentType == 'lpx')",message="lpx may only be set when componentType is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx') || has(self.lpx)",message="lpx is required when componentType is lpx" -// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))",message="replicas must be between 1 and 9 when componentType is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.replicas) && self.replicas < 1)",message="replicas must be positive when componentType is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)",message="scalingAdapter is not supported when componentType is lpx" type DynamoComponentDeploymentSharedSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index fd8050b75259..669d8dbe1f90 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -96,8 +96,6 @@ type LPXAttemptStatus struct { DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` // +listType=map // +listMapKey=name - // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=9 Requests []LPXAttemptRequestStatus `json:"requests"` } diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 3baf88d89b97..6ea40f210459 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -90,7 +90,7 @@ type DynamoComponentDeploymentSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.type) && self.type == 'lpx')",message="lpx may only be set when type is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx') || has(self.lpx)",message="lpx is required when type is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx') || !has(self.podTemplate)",message="LPX Pod templates belong to roles" -// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))",message="replicas must be between 1 and 9 when type is lpx" +// +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx' && has(self.replicas) && self.replicas < 1)",message="replicas must be positive when type is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx' && has(self.scalingAdapter))",message="scalingAdapter is not supported when type is lpx" type DynamoComponentDeploymentSharedSpec struct { // providerOverride configures the primary Grove unit representing this DGD diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index 927f51097bb7..c45ee4349fa7 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -40,8 +40,6 @@ type LPXAttemptStatus struct { DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` // +listType=map // +listMapKey=name - // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=9 Requests []LPXAttemptRequestStatus `json:"requests"` } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 252e671a8f37..51d94fbfd721 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -19728,8 +19728,8 @@ spec: rule: '!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)' - message: componentType lpu is not supported; use lpx rule: '!has(self.componentType) || self.componentType != ''lpu''' - - message: replicas must be between 1 and 9 when componentType is lpx - rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: replicas must be positive when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && self.replicas < 1)' - message: scalingAdapter is not supported when componentType is lpx rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)' status: @@ -36902,8 +36902,8 @@ spec: rule: '!has(oldSelf.type) || (has(self.type) && self.type == oldSelf.type)' - message: LPX Pod templates belong to roles rule: '!(has(self.type) && self.type == ''lpx'') || !has(self.podTemplate)' - - message: replicas must be between 1 and 9 when type is lpx - rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: replicas must be positive when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && self.replicas < 1)' - message: scalingAdapter is not supported when type is lpx rule: '!(has(self.type) && self.type == ''lpx'' && has(self.scalingAdapter))' status: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 37a8d18120ed..cc5dd6298568 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -15850,8 +15850,8 @@ spec: rule: '!has(self.lpx) || (has(self.componentType) && self.componentType == ''lpx'')' - message: lpx is required when componentType is lpx rule: '!(has(self.componentType) && self.componentType == ''lpx'') || has(self.lpx)' - - message: replicas must be between 1 and 9 when componentType is lpx - rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: replicas must be positive when componentType is lpx + rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && self.replicas < 1)' - message: scalingAdapter is not supported when componentType is lpx rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.scalingAdapter) && has(self.scalingAdapter.enabled) && self.scalingAdapter.enabled == true)' description: Services are the services to deploy as part of this deployment. @@ -16030,8 +16030,6 @@ spec: - attemptDigest - name type: object - maxItems: 9 - minItems: 1 type: array x-kubernetes-list-map-keys: - name @@ -16111,8 +16109,6 @@ spec: - attemptDigest - name type: object - maxItems: 9 - minItems: 1 type: array x-kubernetes-list-map-keys: - name @@ -25289,8 +25285,8 @@ spec: rule: '!(has(self.type) && self.type == ''lpx'') || has(self.lpx)' - message: LPX Pod templates belong to roles rule: '!(has(self.type) && self.type == ''lpx'') || !has(self.podTemplate)' - - message: replicas must be between 1 and 9 when type is lpx - rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && (self.replicas < 1 || self.replicas > 9))' + - message: replicas must be positive when type is lpx + rule: '!(has(self.type) && self.type == ''lpx'' && has(self.replicas) && self.replicas < 1)' - message: scalingAdapter is not supported when type is lpx rule: '!(has(self.type) && self.type == ''lpx'' && has(self.scalingAdapter))' maxItems: 25 @@ -25888,8 +25884,6 @@ spec: - attemptDigest - name type: object - maxItems: 9 - minItems: 1 type: array x-kubernetes-list-map-keys: - name @@ -25969,8 +25963,6 @@ spec: - attemptDigest - name type: object - maxItems: 9 - minItems: 1 type: array x-kubernetes-list-map-keys: - name diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index 3e8681d4e41b..976e87cf15e5 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -299,8 +299,6 @@ spec: - attemptDigest - name type: object - maxItems: 9 - minItems: 1 type: array x-kubernetes-list-map-keys: - name diff --git a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go index 4742aee565d4..87d7bece97a2 100644 --- a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go @@ -107,4 +107,12 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { storedSource := &v1beta1.DynamoGraphDeployment{} require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) require.Equal(t, source.Status, storedSource.Status) + + t.Log("An empty scheduling batch must persist as an array, not null") + stored.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: stored.Generation}} + require.True(t, apierrors.IsInvalid(env.Client().Status().Update(t.Context(), stored))) + stored.Status.Placement.LPXAttempt.Requests = []v1beta1.LPXAttemptRequestStatus{} + require.NoError(t, env.Client().Status().Update(t.Context(), stored)) + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(stored), stored)) + require.Equal(t, []v1beta1.LPXAttemptRequestStatus{}, stored.Status.Placement.LPXAttempt.Requests) } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 5f5a2ecd6745..df931c557d7d 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -20,6 +20,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" @@ -250,14 +251,19 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") require.NoError(t, env.Client().Delete(t.Context(), quota)) - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(child, source), Namespace: source.Namespace}} - configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-lpu", Namespace: source.Namespace}} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(child), Namespace: source.Namespace}} + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: source.Namespace}} service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-lpx", Namespace: source.Namespace}} require.EventuallyWithT(t, func(c *assert.CollectT) { + if !assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) { + return + } + configMap.Name = dynamolpx.LPUConfigMapName(pcs.Name, pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash]) for _, resource := range []client.Object{pcs, configMap, service} { if assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(resource), resource)) { assert.Equal(c, metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK), metav1.GetControllerOf(resource)) } } + assert.Equal(c, ptr.To(true), configMap.Immutable) }, 20*time.Second, 50*time.Millisecond) } diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index ca4eb8eeb904..b5cfc04e982c 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -9,11 +9,12 @@ import ( "context" "fmt" "sort" - "strconv" "time" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -24,7 +25,6 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" ) const ( @@ -89,22 +89,12 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, ) (lpxClassification, time.Time, []lpxv1alpha1.LPUPipelineRequest, error) { - // Failed reconciliations can retain a current durable attempt before the - // complete result is observed. Invalidate only the attempt's own generation, - // including when the new source no longer selects a scheduling deadline. + // Source edits do not reset a scheduling batch's absolute deadline. attempt := currentLPXAttemptStatus(deployment) - if attempt != nil && attempt.ObservedGeneration != deployment.Generation { - deployment.Status.Placement = nil - attempt = nil - } deadlineSeconds := lpxAttemptDeadlineSeconds(source) - if !source.HasLPXComponent() || deadlineSeconds == nil { - return nil, time.Time{}, nil, nil - } if attempt == nil { return nil, time.Time{}, nil, nil } - pcsName := dynamo.PCSNameForLPX(deployment, source) deadlineAt := time.Time{} if attempt.DeadlineAt != nil { deadlineAt = attempt.DeadlineAt.Time @@ -114,15 +104,18 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( return nil, deadlineAt, nil, err } observed := lpxRequestsByName(requests) - if updateLPXAttemptFromLive(deployment, deadlineSeconds, attempt, observed) { + if updateLPXAttemptFromLive(deadlineSeconds, attempt, observed) { + if attempt.DeadlineAt != nil { + deadlineAt = attempt.DeadlineAt.Time + } return newLPXDeadlineTransition( attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, "Persisted the exact LPX request identity and immutable scheduling deadline", time.Nanosecond, - ), attempt.DeadlineAt.Time, nil, nil + ), deadlineAt, nil, nil } if attempt.ExceededAt != nil { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, pcsName, attempt, observed, lpxSchedulingDeadlineExceededReason, + ctx, deployment, attempt, observed, lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; no retry is authorized", ) return state, time.Time{}, nil, err @@ -132,10 +125,18 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( return nil, time.Time{}, requests, nil } now := time.Now() + // Completion is authoritative even when the status write lags a Bound observation. + if disposition { + observedAt := metav1.NewTime(now) + attempt.DisarmedAt = &observedAt + return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStatePending, + lpxSchedulingDispositionObservedReason, "Recorded the scheduling batch disposition", time.Nanosecond), time.Time{}, requests, nil + } if attempt.DeadlineAt != nil && (attempt.DisarmedAt == nil || !missing) && !now.Before(attempt.DeadlineAt.Time) { decision := metav1.NewTime(now) attempt.ExceededAt = &decision attempt.DisarmedAt = nil + attempt.ObservedGeneration = deployment.Generation return newLPXDeadlineTransition( attempt, nvidiacomv1beta1.DGDStateFailed, lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; persisting expiry before exact cleanup", time.Nanosecond, @@ -143,22 +144,14 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( } if missing { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, pcsName, attempt, observed, lpxAttemptAuthorityLostReason, - "An exact LPX request disappeared; retiring the aggregate without replacement", + ctx, deployment, attempt, observed, lpxAttemptAuthorityLostReason, + "An exact LPX request disappeared; retiring its scheduling batch without replacement", ) return state, time.Time{}, nil, err } if attempt.DeadlineAt == nil { return nil, time.Time{}, requests, nil } - if disposition { - observedAt := metav1.NewTime(now) - attempt.DisarmedAt = &observedAt - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingDispositionObservedReason, - "Recorded the exact aggregate scheduling disposition before deadline expiry", time.Nanosecond, - ), attempt.DeadlineAt.Time, nil, nil - } if attempt.DisarmedAt != nil { attempt.DisarmedAt = nil return newLPXDeadlineTransition( @@ -169,45 +162,144 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( return nil, deadlineAt, requests, nil } +// retireLPXEngineRequest records intentional removal before deleting the exact request. +func (r *graphReconciler) retireLPXEngineRequest(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest, reason string, intentional bool) (lpxClassification, error) { + // Persist batch membership before deletion can be mistaken for lost authority. + if current := currentLPXAttemptStatus(deployment); current != nil && intentional && current.ExceededAt == nil { + for i, row := range current.Requests { + if row.Name == request.Name && row.AttemptDigest != "" && (row.UID == request.UID || row.UID == "" && row.AttemptDigest == request.Annotations[lpxAttemptDigestAnnotation]) { + next := current.DeepCopy() + // Retain exact cleanup authority until a replacement digest can be prepared. + next.Requests[i].UID = request.UID + next.Requests[i].AttemptDigest = "" + return newLPXDeadlineTransition(next, nvidiacomv1beta1.DGDStatePending, + lpxRetiringReason, reason, time.Nanosecond), nil + } + } + } + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return nil, err + } + if request.DeletionTimestamp.IsZero() { + uid, rv := request.UID, request.ResourceVersion + if err := r.Delete(ctx, request, client.Preconditions{UID: &uid, ResourceVersion: &rv}); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + } + // Invalid live intent needs exact Pod cleanup; native replacement already owns its Pods. + if !intentional { + if err := r.retireLPXAttemptPods(ctx, request); err != nil { + return nil, err + } + } + if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { + return nil, err + } + return &lpxRetiring{retirementReason: reason}, nil +} + +// retireLPXAttemptPods releases only Pods committed to this exact scheduling request. +func (r *graphReconciler) retireLPXAttemptPods(ctx context.Context, request *lpxv1alpha1.LPUPipelineRequest) error { + if request.Status == nil || request.Status.Committed == nil || request.Status.Committed.Execution.NodeLocal == nil { + return nil + } + for _, partition := range request.Status.Committed.Execution.NodeLocal.PartitionSelections { + for _, row := range partition.SelectedRows { + refs := make([]lpxv1alpha1.ObjectReference, 0, 2) + if row.Current != nil { + refs = append(refs, row.Current.PodRef) + } + if row.Target != nil { + refs = append(refs, row.Target.PodRef) + } + for _, ref := range refs { + if ref.Namespace != request.Namespace || ref.Name == "" || ref.UID == "" { + return fmt.Errorf("LPX request %q contains an invalid committed Pod identity", request.Name) + } + uid := types.UID(ref.UID) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, Name: ref.Name}} + if err := r.Delete(ctx, pod, client.Preconditions{UID: &uid}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return err + } + } + } + } + return nil +} + func (r *graphReconciler) reconcileLPXAttemptPreparation( deployment *nvidiacomv1alpha1.LPXGraphDeployment, deadlineSeconds *int64, want *nvidiacomv1beta1.LPXAttemptStatus, currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) (*lpxDeadlineTransition, error) { +) *lpxDeadlineTransition { if want == nil { - return nil, nil + return nil } + // New engines join an active batch, but never a completed batch awaiting disarm. current := currentLPXAttemptStatus(deployment) - if current == nil || current.ObservedGeneration != deployment.Generation { - updateLPXAttemptFromLive(deployment, deadlineSeconds, want, currents) - reason, message := lpxAttemptPreparedReason, "Prepared the complete LPX request set before publication" - if want.DeadlineAt != nil { - reason, message = lpxSchedulingAttemptActiveReason, "Recovered the exact first publication and its deadline" + if current != nil && current.ExceededAt != nil { + return newLPXDeadlineTransition(current, nvidiacomv1beta1.DGDStateFailed, + lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; waiting for cleanup and a later edit", lpxRetirementRequeueAfter) + } + next := want.DeepCopy() + previous := make(map[string]nvidiacomv1beta1.LPXAttemptRequestStatus) + completed := false + if current != nil { + var missing bool + _, completed, missing = exactLPXAttemptState(current, currents) + active := !completed && (current.DisarmedAt == nil || missing) + if active { + next = current.DeepCopy() + } + for _, row := range current.Requests { + if active || row.UID != "" && row.AttemptDigest == "" { + previous[row.Name] = row + } } - return newLPXDeadlineTransition( - want, nvidiacomv1beta1.DGDStatePending, reason, message, time.Nanosecond, - ), nil - } - if !sameLPXAttemptProjection(current, want) { - return nil, fmt.Errorf("current LPX attempt status does not match the desired request projection") } - if current.PodCliqueSetUID != want.PodCliqueSetUID { - return nil, fmt.Errorf("current LPX attempt status does not match the exact Grove identity") + next.Requests = make([]nvidiacomv1beta1.LPXAttemptRequestStatus, 0, len(want.Requests)) + // Desired rows are already sorted; only newly joining engines need disposition checks. + for _, row := range want.Requests { + if old, tracked := previous[row.Name]; tracked { + if old.UID != "" || row.AttemptDigest == "" { + row = old + } + } else if request := currents[row.Name]; request != nil && request.DeletionTimestamp.IsZero() { + if row.AttemptDigest == "" { + row.AttemptDigest = request.Annotations[lpxAttemptDigestAnnotation] + } + switch state := classifyPublishedLPX(request).(type) { + case *lpxBound: + continue + case *lpxSchedulerObserved: + if state.Phase == lpxv1alpha1.RequestPhaseNoFit || state.Phase == lpxv1alpha1.RequestPhaseUnsupported { + continue + } + } + } + next.Requests = append(next.Requests, row) } - if _, _, missing := exactLPXAttemptState(current, currents); missing { - return newLPXDeadlineTransition( - current, nvidiacomv1beta1.DGDStateFailed, lpxAttemptAuthorityLostReason, - "An exact LPX request disappeared; publication remains fenced", time.Nanosecond, - ), nil + // Cancel the last pending engine durably; replacement placeholders keep their clock. + if len(next.Requests) == 0 { + if current == nil || completed { + return nil + } + next = current.DeepCopy() + next.Requests = []nvidiacomv1beta1.LPXAttemptRequestStatus{} + if next.DisarmedAt == nil { + disarmedAt := metav1.Now() + next.DisarmedAt = &disarmedAt + } } - if !updateLPXAttemptFromLive(deployment, deadlineSeconds, current, currents) { - return nil, nil + updateLPXAttemptFromLive(deadlineSeconds, next, currents) + if apiequality.Semantic.DeepEqual(current, next) { + return nil } return newLPXDeadlineTransition( - current, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, - "Persisted the exact LPX request identity and immutable scheduling deadline", time.Nanosecond, - ), nil + next, nvidiacomv1beta1.DGDStatePending, lpxAttemptPreparedReason, + "Prepared the scheduling batch before publication", time.Nanosecond, + ) } // revalidateLPXAttemptPublication fences live authority after preparation has @@ -238,7 +330,7 @@ func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, d } func lpxAttemptDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) *int64 { - if source.Spec.Scheduling == nil { + if source == nil || source.Spec.Scheduling == nil { return nil } return source.Spec.Scheduling.AttemptDeadlineSeconds @@ -265,23 +357,9 @@ func currentLPXAttemptStatus(deployment *nvidiacomv1alpha1.LPXGraphDeployment) * return deployment.Status.Placement.LPXAttempt } -func sameLPXAttemptProjection(a, b *nvidiacomv1beta1.LPXAttemptStatus) bool { - if a == nil || b == nil || a.ObservedGeneration != b.ObservedGeneration || len(a.Requests) != len(b.Requests) { - return false - } - for i := range a.Requests { - if a.Requests[i].Name != b.Requests[i].Name || - a.Requests[i].AttemptDigest != b.Requests[i].AttemptDigest { - return false - } - } - return true -} - // updateLPXAttemptFromLive incorporates server-persisted requests into the non-nil, // reconcile-owned attempt and reports whether a status transition is needed. func updateLPXAttemptFromLive( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, deadlineSeconds *int64, attempt *nvidiacomv1beta1.LPXAttemptStatus, observed map[string]*lpxv1alpha1.LPUPipelineRequest, @@ -290,13 +368,15 @@ func updateLPXAttemptFromLive( var firstPublishedAt time.Time for i := range attempt.Requests { row := &attempt.Requests[i] + if row.AttemptDigest == "" { + continue + } request := observed[row.Name] if request == nil { continue } if row.UID == "" { - if request.Annotations[lpxAttemptDigestAnnotation] != row.AttemptDigest || - request.Annotations[lpxDeploymentGenerationAnnotation] != strconv.FormatInt(deployment.Generation, 10) { + if request.Annotations[lpxAttemptDigestAnnotation] != row.AttemptDigest { continue } changed = true @@ -313,7 +393,7 @@ func updateLPXAttemptFromLive( firstPublishedAt = request.CreationTimestamp.Time } } - if attempt.DeadlineAt == nil && !firstPublishedAt.IsZero() { + if attempt.DeadlineAt == nil && deadlineSeconds != nil && !firstPublishedAt.IsZero() { deadline := metav1.NewTime(firstPublishedAt.Add( time.Duration(*deadlineSeconds) * time.Second, )) @@ -347,12 +427,12 @@ func exactLPXAttemptState( attempt *nvidiacomv1beta1.LPXAttemptStatus, observed map[string]*lpxv1alpha1.LPUPipelineRequest, ) (active bool, disposition bool, missing bool) { - disposition = true + disposition = len(attempt.Requests) > 0 for i := range attempt.Requests { // Distinguish unpublished rows from recorded request authority loss. row := &attempt.Requests[i] request := observed[row.Name] - if row.UID == "" { + if row.AttemptDigest == "" || row.UID == "" { disposition = false continue } @@ -383,12 +463,14 @@ func exactLPXAttemptState( func (r *graphReconciler) continueTerminalLPXAttempt( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, attempt *nvidiacomv1beta1.LPXAttemptStatus, observed map[string]*lpxv1alpha1.LPUPipelineRequest, reason string, message string, ) (lpxClassification, error) { + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return nil, err + } // Retire recorded identities in their canonical request order. pending := false for _, row := range attempt.Requests { @@ -398,6 +480,9 @@ func (r *graphReconciler) continueTerminalLPXAttempt( } pending = true if !request.DeletionTimestamp.IsZero() { + if err := r.retireLPXAttemptPods(ctx, request); err != nil { + return nil, err + } if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { return nil, err } @@ -409,18 +494,14 @@ func (r *graphReconciler) continueTerminalLPXAttempt( }}); err != nil && !apierrors.IsNotFound(err) { return nil, fmt.Errorf("delete terminal LPX request %q: %w", request.Name, err) } - } - if pending { - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID); err != nil { + if err := r.retireLPXAttemptPods(ctx, request); err != nil { return nil, err } - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, lpxRetirementRequeueAfter, - ), nil } - pending, err := r.retireLPXAttemptPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID) - if err != nil { - return nil, err + // A failed scheduling batch never scales down or deletes the shared PCS. + if !pending && attempt.ExceededAt != nil && deployment.Generation > attempt.ObservedGeneration { + return newLPXDeadlineTransition(nil, nvidiacomv1beta1.DGDStatePending, + lpxAttemptPreparedReason, "The failed batch is gone; a later edit permits scheduling", time.Nanosecond), nil } requeueAfter := time.Duration(0) if pending { @@ -430,34 +511,3 @@ func (r *graphReconciler) continueTerminalLPXAttempt( attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, requeueAfter, ), nil } - -func (r *graphReconciler) retireLPXAttemptPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, - uid types.UID, -) (bool, error) { - pcs := &grovev1alpha1.PodCliqueSet{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName} - if err := r.apiReader.Get(ctx, key, pcs); err != nil { - return false, client.IgnoreNotFound(err) - } - if uid == "" || pcs.UID != uid { - return false, nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return false, fmt.Errorf("refusing to retire PodCliqueSet %q without the exact LPXGraphDeployment owner", pcs.Name) - } - if pcs.Spec.Replicas != 0 { - pcs.Spec.Replicas = 0 - return true, r.Update(ctx, pcs) - } - if !pcs.DeletionTimestamp.IsZero() { - return true, nil - } - pcsUID, resourceVersion := pcs.UID, pcs.ResourceVersion - err := r.Delete(ctx, pcs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ - UID: &pcsUID, ResourceVersion: &resourceVersion, - }}) - return true, client.IgnoreNotFound(err) -} diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 3dd9d408aa0c..b73ebb72741e 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -7,6 +7,7 @@ package lpx import ( "context" + "errors" "testing" "time" @@ -14,9 +15,11 @@ import ( grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -48,9 +51,10 @@ func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) require.Len(t, requests, 1) require.Contains(t, requests[0].Finalizers, lpxAttemptRecordingFinalizer) dgd.Status.Placement = nil - _, classification, err = reconciler.reconcileSelectedLPXSafetyPreflight(ctx, dgd, source) + classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) - require.NotNil(t, classification, "an existing complete LPR set must recover through preflight") + recovered := applyLPXAttemptTransition(t, dgd, classification) + require.Equal(t, requests[0].UID, recovered.Requests[0].UID, "publication reconciliation recovers the existing request") dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: prepared} require.NoError(t, reconciler.Delete(ctx, &requests[0])) @@ -62,6 +66,64 @@ func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) require.NoError(t, err) require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) + + for _, test := range []struct { + name string + deadline, terminating bool + }{ + {name: "absent with deadline", deadline: true}, + {name: "terminating with deadline", deadline: true, terminating: true}, + {name: "absent without deadline"}, + {name: "terminating without deadline", terminating: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Keep ready Grove identities after losing an already-recorded request") + deployment, source := dgd.DeepCopy(), source.DeepCopy() + attempt := currentLPXAttemptStatus(deployment) + attempt.DeadlineAt = nil + if test.deadline { + attempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Hour))) + } else { + source.Spec.Scheduling = nil + attempt.DisarmedAt = ptr.To(metav1.Now()) + } + objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) + objects = append(objects, deadlineTestRequest(deployment, "removed-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound)) + if test.terminating { + request := requests[0].DeepCopy() + request.DeletionTimestamp = ptr.To(metav1.Now()) + request.Finalizers = []string{"scheduler.example/cleanup"} + objects = append(objects, request) + } + r := newLPXTestReconciler(t, registry, deployment, source, objects...) + key := client.ObjectKeyFromObject(deployment) + + t.Log("Full reconciliation repeatedly reports terminal authority loss without republishing or deleting the PCS") + for range 3 { + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key, deployment)) + failed := meta.FindStatusCondition(deployment.Status.Conditions, "Failed") + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionTrue, failed.Status) + require.Equal(t, lpxAttemptAuthorityLostReason, failed.Reason) + live := ownedLPXRequests(t, ctx, r, deployment) + if test.terminating { + require.Len(t, live, 1) + require.Equal(t, requests[0].UID, live[0].UID) + require.False(t, live[0].DeletionTimestamp.IsZero()) + } else { + require.Empty(t, live) + require.Zero(t, result.RequeueAfter) + } + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) + require.Equal(t, armed.PodCliqueSetUID, pcs.UID) + require.Equal(t, int32(1), pcs.Spec.Replicas) + require.True(t, pcs.DeletionTimestamp.IsZero()) + } + }) + } } func TestLPXPreflightReusesActiveDeadlineRequestObservation(t *testing.T) { @@ -148,12 +210,13 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { dgd := newLPXTestDeployment(t, source) source.Spec.Scheduling = deadlineTestScheduling() request := deadlineTestRequest(dgd, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) + running := deadlineTestRequest(dgd, "older-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} - pcs := deadlineTestPCS(dgd, source, "pcs-uid") + pcs := deadlineTestPCS(dgd, "pcs-uid") attempt := deadlineTestAttempt(dgd, request, time.Now().Add(time.Minute)) attempt.DeadlineAt, attempt.PodCliqueSetUID = nil, "" dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - reconciler := newLPXTestReconciler(t, nil, dgd, source, request, pcs) + reconciler := newLPXTestReconciler(t, nil, dgd, source, request, running, pcs) t.Log("Recover the native deadline and record the Bound scheduling disposition") classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) @@ -174,7 +237,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { t.Log("Resumed scheduling expires under the original elapsed deadline") request.Status.Phase = lpxv1alpha1.RequestPhasePending - reconciler = newLPXTestReconciler(t, nil, dgd, source, request, pcs) + reconciler = newLPXTestReconciler(t, nil, dgd, source, request, running, pcs) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) result := lpxResult(classification) @@ -183,11 +246,11 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { require.NotNil(t, transition.attempt.ExceededAt) require.Nil(t, transition.attempt.DisarmedAt) - t.Log("Persist expiry before cleaning up the exact request and PCS") + t.Log("Persist expiry before cleaning up the exact scheduling request") applyLPXAttemptTransition(t, dgd, classification) require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) - t.Log("Start exact request deletion before holding the shared PCS at zero") + t.Log("Delete only the expired request, preserving the PCS and previously running engine") _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) deleting := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name) @@ -195,7 +258,9 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { storedPCS := &grovev1alpha1.PodCliqueSet{} pcsKey := client.ObjectKeyFromObject(pcs) require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) - require.Zero(t, storedPCS.Spec.Replicas) + require.EqualValues(t, 1, storedPCS.Spec.Replicas) + require.Equal(t, pcs.UID, storedPCS.UID) + require.Equal(t, running.UID, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).UID) t.Log("Release the recording finalizer only after scheduler cleanup finishes") deleting.Finalizers = []string{lpxAttemptRecordingFinalizer} @@ -206,7 +271,9 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { require.NoError(t, err) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) - require.True(t, apierrors.IsNotFound(reconciler.Get(t.Context(), pcsKey, storedPCS))) + require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) + require.EqualValues(t, 1, storedPCS.Spec.Replicas) + require.True(t, apierrors.IsNotFound(reconciler.Get(t.Context(), client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) require.Zero(t, classification.(*lpxDeadlineTransition).requeueAfter) t.Log("A completed old attempt cannot touch same-name replacement identities") @@ -214,7 +281,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { replacement.UID = "replacement-uid" replacement.ResourceVersion = "2" replacement.Annotations[lpxPCSUIDAnnotation] = "replacement-pcs-uid" - pcs = deadlineTestPCS(dgd, source, "replacement-pcs-uid") + pcs = deadlineTestPCS(dgd, "replacement-pcs-uid") reconciler = newLPXTestReconciler(t, nil, dgd, source, replacement, pcs) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) @@ -224,14 +291,308 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(pcs), storedPCS)) require.Equal(t, int32(1), storedPCS.Spec.Replicas) - t.Log("A late Bound observation without a durable disposition still expires") + t.Log("A stale cached attempt cannot clean up siblings after a newer status records replacement") + key := client.ObjectKeyFromObject(dgd) + require.NoError(t, reconciler.Get(t.Context(), key, dgd)) + cached := dgd.DeepCopy() + current := dgd.DeepCopy() + current.Status.Placement.LPXAttempt = deadlineTestAttempt(current, replacement, time.Now().Add(time.Minute)) + require.NoError(t, reconciler.Status().Update(t.Context(), current)) + require.NotEqual(t, cached.ResourceVersion, current.ResourceVersion) + running.ResourceVersion = "" + require.NoError(t, reconciler.Create(t.Context(), running)) + cached.Status.Placement.LPXAttempt.Requests = append(cached.Status.Placement.LPXAttempt.Requests, + deadlineTestAttempt(cached, running, time.Now()).Requests[0]) + _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), cached, source) + require.ErrorContains(t, err, "authority changed") + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).DeletionTimestamp.IsZero()) + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, replacement.Name).DeletionTimestamp.IsZero()) + + t.Log("An exact Bound observation is authoritative even before its disposition is persisted") late := deadlineTestAttempt(dgd, request, time.Now().Add(-time.Minute)) dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: late} request.Status.Phase = lpxv1alpha1.RequestPhaseBound reconciler = newLPXTestReconciler(t, nil, dgd, source, request) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) - require.Equal(t, lpxSchedulingDeadlineExceededReason, lpxResult(classification).Reason) + require.Equal(t, lpxSchedulingDispositionObservedReason, lpxResult(classification).Reason) + require.NotNil(t, classification.(*lpxDeadlineTransition).attempt.DisarmedAt) +} + +func TestLPXAttemptBatchEditsPreserveUnchangedEngines(t *testing.T) { + for _, tc := range []struct { + name string + phase lpxv1alpha1.RequestPhase + add, remove, untimed bool + unrecorded, deleting bool + }{ + {name: "bound-before-disarm", phase: lpxv1alpha1.RequestPhaseBound, add: true}, + {name: "no-fit", phase: lpxv1alpha1.RequestPhaseNoFit, add: true}, + {name: "unsupported", phase: lpxv1alpha1.RequestPhaseUnsupported, add: true}, + {name: "active-addition", phase: lpxv1alpha1.RequestPhasePending, add: true}, + {name: "timeout-option-removed", phase: lpxv1alpha1.RequestPhasePending, add: true, untimed: true}, + {name: "metadata", phase: lpxv1alpha1.RequestPhasePending}, + {name: "intentional-removal", phase: lpxv1alpha1.RequestPhasePending, remove: true}, + {name: "remove-and-add", phase: lpxv1alpha1.RequestPhasePending, remove: true, add: true}, + {name: "remove-and-replace", phase: lpxv1alpha1.RequestPhasePending, remove: true, add: true, deleting: true}, + {name: "unrecorded-pending", phase: lpxv1alpha1.RequestPhasePending, unrecorded: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Keep the existing engine and its deadline across a new source generation") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + deployment := newLPXTestDeployment(t, source) + now := time.Now().UTC().Truncate(time.Second) + old := deadlineTestRequest(deployment, "old", now.Add(-time.Hour), tc.phase) + deadline := now.Add(time.Minute) + if tc.phase != lpxv1alpha1.RequestPhasePending { + deadline = now.Add(-time.Minute) + } + current := deadlineTestAttempt(deployment, old, deadline) + if tc.unrecorded { + current.Requests[0].UID = "" + } + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: current} + deployment.Generation++ + want := &nvidiacomv1beta1.LPXAttemptStatus{ObservedGeneration: deployment.Generation, PodCliqueSetUID: current.PodCliqueSetUID} + currents := map[string]*lpxv1alpha1.LPUPipelineRequest{old.Name: old} + if !tc.remove { + want.Requests = append(want.Requests, current.Requests[0]) + want.Requests[0].AttemptDigest = "" + } + if tc.add { + added := deadlineTestRequest(deployment, "new", now, lpxv1alpha1.RequestPhasePending) + if tc.deleting { + added.DeletionTimestamp = ptr.To(metav1.Now()) + } + currents[added.Name] = added + want.Requests = append(want.Requests, nvidiacomv1beta1.LPXAttemptRequestStatus{Name: added.Name}) + } + + t.Log("New engines get a fresh batch only after the old scheduling batch completes") + r := newLPXTestReconciler(t, nil, deployment, source) + timeout := ptr.To[int64](30) + if tc.untimed { + timeout = nil + } + if transition := r.reconcileLPXAttemptPreparation(deployment, timeout, want, currents); transition != nil { + current = applyLPXAttemptTransition(t, deployment, transition) + } + if tc.phase != lpxv1alpha1.RequestPhasePending { + require.Len(t, current.Requests, 1) + require.Equal(t, "new", current.Requests[0].Name) + require.Equal(t, now.Add(30*time.Second), current.DeadlineAt.Time) + } else { + require.Equal(t, deadline, current.DeadlineAt.Time) + require.Equal(t, len(want.Requests), len(current.Requests)) + } + if tc.add || tc.unrecorded { + require.Nil(t, current.DisarmedAt) + if !tc.deleting { + require.NotEmpty(t, current.Requests[0].UID) + } + } + require.True(t, old.DeletionTimestamp.IsZero()) + require.Equal(t, types.UID("old-uid"), old.UID) + }) + } +} + +func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { + for _, tc := range []struct { + name string + phase lpxv1alpha1.RequestPhase + replacing, keepClique bool + }{ + {name: "scale-in", phase: lpxv1alpha1.RequestPhasePending}, + {name: "completed-scale-in", phase: lpxv1alpha1.RequestPhaseBound}, + {name: "native-replacement", phase: lpxv1alpha1.RequestPhasePending, replacing: true}, + {name: "native-child-still-deleting", phase: lpxv1alpha1.RequestPhasePending, replacing: true, keepClique: true}, + {name: "completed-native-replacement", phase: lpxv1alpha1.RequestPhaseBound, replacing: true}, + } { + t.Run(tc.name, func(t *testing.T) { + replacing := tc.replacing + t.Log("Publish two engines, with only the second engine recorded in the scheduling batch") + ctx := t.Context() + deployment, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Components[0].Replicas = ptr.To[int32](2) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) + objects := lpxMaterializedObjects(t, r, deployment, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + _, err := r.reconcileSelectedLPX(ctx, deployment, desired) + require.NoError(t, err) + running := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[0].requestName) + running.Status = newLPXNodeLocalBoundStatus(running.Generation, 1, "sha256:bound") + pending := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[1].requestName) + pending.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduler.example/cleanup"} + pending.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(pending.Generation), + } + if tc.phase == lpxv1alpha1.RequestPhaseBound { + pending.Status = newLPXNodeLocalBoundStatus(pending.Generation, 1, "sha256:bound") + } + deadline := time.Now().UTC().Truncate(time.Second).Add(-time.Minute) + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: deadlineTestAttempt(deployment, pending, deadline)} + source.Spec.Scheduling = deadlineTestScheduling() + if replacing { + clique := findLPXTestClique(t, objects, desired.plan.ForReplica(1).Agents[0].CliqueName) + clique.Finalizers = []string{"grove.example/cleanup"} + clique.DeletionTimestamp = ptr.To(metav1.Now()) + } else { + source.Spec.Components[0].Replicas = ptr.To[int32](1) + } + objects = append(objects, running, pending) + r = newLPXTestReconciler(t, registry, deployment, source, objects...) + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + return delegated.Delete(ctx, object.DeepCopyObject().(client.Object), opts...) + }, + }) + key := client.ObjectKeyFromObject(deployment) + + t.Log("Persist cancellation, while pending native replacement retains its deadline") + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key, deployment)) + attempt := currentLPXAttemptStatus(deployment) + if replacing { + require.Len(t, attempt.Requests, 1) + require.Equal(t, pending.UID, attempt.Requests[0].UID) + require.Empty(t, attempt.Requests[0].AttemptDigest) + if tc.phase == lpxv1alpha1.RequestPhasePending { + require.NotNil(t, attempt.ExceededAt) + } else { + require.NotNil(t, attempt.DisarmedAt) + } + } else { + require.Equal(t, []nvidiacomv1beta1.LPXAttemptRequestStatus{}, attempt.Requests) + require.Nil(t, attempt.ExceededAt) + require.NotNil(t, attempt.DisarmedAt) + } + require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) + if replacing && !tc.keepClique { + t.Log("Grove can finish deleting the old clique before scheduler retirement resumes") + clique := &grovev1alpha1.PodClique{} + cliqueKey := client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.ForReplica(1).Agents[0].CliqueName} + require.NoError(t, r.Get(ctx, cliqueKey, clique)) + clique.Finalizers = nil + require.NoError(t, r.Update(ctx, clique)) + } + if !replacing { + t.Log("A source read failure on the next reconcile cannot expire the canceled batch") + readError := errors.New("temporary source observation failure") + original := r.Client + r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, source := object.(*nvidiacomv1beta1.DynamoGraphDeployment); source { + return readError + } + return delegated.Get(ctx, key, object, opts...) + }, + }) + _, readErr := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + r.Client = original + require.NoError(t, r.Get(ctx, key, deployment)) + require.Nil(t, currentLPXAttemptStatus(deployment).ExceededAt) + require.ErrorIs(t, readErr, readError) + } + + t.Log("Cancellation completes, while native replacement still expires under the original clock") + for range 3 { + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + if tc.keepClique && err != nil { + t.Log("The next observation retries an optimistic concurrency conflict:", err) + require.True(t, apierrors.IsConflict(err)) + } else { + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, key, deployment)) + attempt = currentLPXAttemptStatus(deployment) + if replacing && tc.phase == lpxv1alpha1.RequestPhaseBound { + require.Nil(t, attempt.DeadlineAt) + } else { + require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) + } + } + if replacing { + require.Equal(t, tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) + require.Nil(t, attempt.DisarmedAt) + retiring := getLPXRequest(t, ctx, r.Client, deployment.Namespace, pending.Name) + require.False(t, retiring.DeletionTimestamp.IsZero()) + retiring.Finalizers = []string{lpxAttemptRecordingFinalizer} + require.NoError(t, r.Update(ctx, retiring)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pending), &lpxv1alpha1.LPUPipelineRequest{}))) + } else { + require.Nil(t, attempt.ExceededAt) + require.NotNil(t, attempt.DisarmedAt) + } + stored := getLPXRequest(t, ctx, r.Client, deployment.Namespace, running.Name) + require.Equal(t, running.UID, stored.UID) + require.True(t, stored.DeletionTimestamp.IsZero()) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) + require.Equal(t, types.UID("pcs-uid"), pcs.UID) + }) + } +} + +func TestLPXAttemptRetryRequiresLaterEditAndCompletedCleanup(t *testing.T) { + for _, tc := range []struct { + name string + laterEdit, oldGone bool + }{{name: "same-generation", oldGone: true}, {name: "cleanup-pending", laterEdit: true}, {name: "later-edit", laterEdit: true, oldGone: true}} { + t.Run(tc.name, func(t *testing.T) { + t.Log("Keep the failed batch fenced until both a later edit and exact-request cleanup") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + deployment := newLPXTestDeployment(t, source) + old := deadlineTestRequest(deployment, "old", time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending) + old.Finalizers = []string{"scheduler.example/cleanup"} + attempt := deadlineTestAttempt(deployment, old, time.Now().Add(-time.Second)) + attempt.ExceededAt = ptr.To(metav1.Now()) + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} + if tc.laterEdit { + deployment.Generation++ + } + var objects []client.Object + if !tc.oldGone { + objects = append(objects, old) + } + r := newLPXTestReconciler(t, nil, deployment, source, objects...) + currents := lpxRequestsByName(ownedLPXRequests(t, t.Context(), r, deployment)) + classification, err := r.continueTerminalLPXAttempt(t.Context(), deployment, attempt, currents, lpxSchedulingDeadlineExceededReason, "expired") + require.NoError(t, err) + transition := classification.(*lpxDeadlineTransition) + if tc.laterEdit && tc.oldGone { + require.Nil(t, transition.attempt) + } else { + require.Equal(t, lpxSchedulingDeadlineExceededReason, transition.result.Reason) + require.Equal(t, attempt, transition.attempt) + } + }) + } +} + +func TestLPXAttemptRecoversUIDWithoutTimeoutOrSource(t *testing.T) { + for _, sourceAvailable := range []bool{false, true} { + t.Log("Recover an already-published UID even when its timeout option or source is unavailable", sourceAvailable) + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + deployment := newLPXTestDeployment(t, source) + request := deadlineTestRequest(deployment, "unrecorded", time.Now(), lpxv1alpha1.RequestPhasePending) + attempt := deadlineTestAttempt(deployment, request, time.Now()) + attempt.DeadlineAt, attempt.Requests[0].UID = nil, "" + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} + r := newLPXTestReconciler(t, nil, deployment, source, request) + if !sourceAvailable { + source = nil + } + classification, wake, _, err := r.reconcileLPXAttemptDeadline(t.Context(), deployment, source) + require.NoError(t, err) + recovered := applyLPXAttemptTransition(t, deployment, classification) + require.Equal(t, request.UID, recovered.Requests[0].UID) + require.Nil(t, recovered.DeadlineAt) + require.True(t, wake.IsZero()) + } } func applyLPXAttemptTransition(t *testing.T, dgd *nvidiacomv1alpha1.LPXGraphDeployment, classification lpxClassification) *nvidiacomv1beta1.LPXAttemptStatus { @@ -261,7 +622,7 @@ func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, Generation: generation, CreationTimestamp: metav1.NewTime(created), Annotations: map[string]string{ lpxAttemptDigestAnnotation: "sha256:attempt", lpxDeploymentUIDAnnotation: string(dgd.UID), - lpxDeploymentGenerationAnnotation: "1", lpxPCSUIDAnnotation: "pcs-uid", + lpxPCSUIDAnnotation: "pcs-uid", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }, @@ -281,10 +642,10 @@ func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, return request } -func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { +func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { return &grovev1alpha1.PodCliqueSet{ ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.PCSNameForLPX(dgd, source), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", + Name: dynamo.PCSNameForLPX(dgd), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go index efe37165340c..b171c5d33bab 100644 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -72,7 +72,7 @@ func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { case "foreign": firstDraft.OwnerReferences = []metav1.OwnerReference{{UID: "foreign", Controller: ptr.To(true)}} case "old revision": - firstDraft.Annotations[dynamo.LPXInputRevisionAnnotation] = "old" + firstDraft.Status.CurrentPodCliqueSetGenerationHash = ptr.To("old") case "unobserved": firstDraft.Status.ObservedGeneration = ptr.To(firstDraft.Generation - 1) } diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 09d2ddbed239..9f5fda10a42e 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -55,7 +55,7 @@ func (r *graphReconciler) resolveLPXSource(ctx context.Context, deployment *v1al return nil, nil, err } if err := dynamo.ValidateLPXSource(deployment, source); err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil + return source, nil, err } return source, nil, nil } @@ -144,7 +144,16 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} var deadlineAt time.Time + var source *v1beta1.DynamoGraphDeployment defer func() { + // Reconcile intended removals before deadline cleanup, including on failed renders. + if unavailableReason == "" { + classification, wake, _, deadlineErr := r.reconcileLPXAttemptDeadline(ctx, deployment, source) + deadlineAt, err = wake, errors.Join(err, deadlineErr) + if classification != nil { + state, result = lpxResult(classification), projectLPXLifecycleStatus(deployment, classification) + } + } if err != nil { state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXReconciliationFailed", truncateLPXMessage(err.Error()) } else if unavailableReason == "" { @@ -182,7 +191,9 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, nil } - source, rejected, sourceErr := r.resolveLPXSource(ctx, deployment) + var rejected *lpxRejected + var sourceErr error + source, rejected, sourceErr = r.resolveLPXSource(ctx, deployment) if sourceErr != nil { return ctrl.Result{}, sourceErr } @@ -195,18 +206,13 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{RequeueAfter: time.Nanosecond}, r.Update(ctx, deployment) } - classification, wake, requests, err := r.reconcileLPXAttemptDeadline(ctx, deployment, source) - deadlineAt = wake - var selected *lpxMaterializing - if classification == nil && err == nil { - selected, classification, err = r.reconcileLPXSafetyPreflight(ctx, deployment, source, requests) - } + selected, classification, err := r.reconcileLPXSafetyPreflight(ctx, deployment, source, nil) if err != nil { return ctrl.Result{}, err } if classification != nil { - state, result, err = r.finishLPXPreflight(ctx, deployment, classification) - return result, err + state = lpxResult(classification) + return projectLPXLifecycleStatus(deployment, classification), nil } ready, err := r.reconcileModelDownloads(ctx, deployment, source, selected == nil) if err != nil { @@ -222,24 +228,14 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, err } if classification != nil { - state, result, err = r.finishLPXPreflight(ctx, deployment, classification) - return result, err + state = lpxResult(classification) + return projectLPXLifecycleStatus(deployment, classification), nil } } state, result, err = r.reconcileWorkload(ctx, deployment, source, selected) return result, err } -// finishLPXPreflight projects a non-nil classification and retires rejected workloads. -func (r *graphReconciler) finishLPXPreflight(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, classification lpxClassification) (reconcileOutcome, ctrl.Result, error) { - state := lpxResult(classification) - if _, rejected := classification.(*lpxRejected); rejected { - result, err := r.retireInvalidWorkload(ctx, deployment, state.Message) - return state, result, err - } - return state, projectLPXLifecycleStatus(deployment, classification), nil -} - // reconcileWorkload publishes and observes the selected engine after source, // deadline, placement, and model-download gates have passed. deployment, source // and selected are non-nil and supplied by that preflight. @@ -255,14 +251,6 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a } desired, resources, err := renderPodCliqueSet(ctx, source, r.Config, r.runtimeConfig, r.Client, r.DockerSecretRetriever, selected.workload, selected.plan, deployment) if err != nil { - retiring, fenceErr := r.retireInvalidLPXWorkload(ctx, deployment, fmt.Sprintf("The current LPX workload can no longer be rendered safely: %v", err)) - if fenceErr != nil { - return state, ctrl.Result{}, errors.Join(err, fenceErr) - } - if retiring != nil { - state = lpxResult(retiring) - return state, projectLPXLifecycleStatus(deployment, retiring), nil - } return state, ctrl.Result{}, err } if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { @@ -273,21 +261,34 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a return state, ctrl.Result{}, err } } - synced, changed, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired, selected) + synced, changed, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired) if err != nil { var retiring *lpxRetiring if errors.As(err, &retiring) { state = lpxResult(retiring) return state, projectLPXLifecycleStatus(deployment, retiring), nil } - if errors.Is(err, errLPXGrovePodCliqueSetRecreating) { - return state, ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter}, nil - } return state, ctrl.Result{}, err } if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { return state, ctrl.Result{}, err } + // Grove seeds scaling groups once; only explicit component replicas override native scale. + if replicas := lpx.ServingComponent(source).Replicas; replicas != nil { + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.LPXScalingGroup}, group); err != nil { + if !apierrors.IsNotFound(err) { + return state, ctrl.Result{}, err + } + } else if !metav1.IsControlledBy(group, synced) || !group.DeletionTimestamp.IsZero() { + return state, ctrl.Result{}, fmt.Errorf("LPX scaling group lacks the current PCS owner") + } else if group.Spec.Replicas != *replicas { + group.Spec.Replicas = *replicas + if err := r.Update(ctx, group); err != nil { + return state, ctrl.Result{}, err + } + } + } readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, source, deployment, synced) if err != nil { return state, ctrl.Result{}, err @@ -375,14 +376,9 @@ func (r *graphReconciler) validateLPXDeploymentAuthority(ctx context.Context, de if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(deployment), current); err != nil { return err } - if current.UID != deployment.UID || current.Generation != deployment.Generation || !apiequality.Semantic.DeepEqual(current.Spec, deployment.Spec) || !apiequality.Semantic.DeepEqual(current.OwnerReferences, deployment.OwnerReferences) || !current.DeletionTimestamp.IsZero() { + if current.UID != deployment.UID || current.ResourceVersion != deployment.ResourceVersion || !current.DeletionTimestamp.IsZero() { return fmt.Errorf("LPXGraphDeployment authority changed before LPX resource mutation") } - for _, key := range []string{dynamo.LPXRestartAnnotation, lpx.DGDGenerationAnnotation} { - if current.Annotations[key] != deployment.Annotations[key] { - return fmt.Errorf("LPXGraphDeployment publication metadata changed before LPX resource mutation") - } - } return nil } @@ -426,7 +422,7 @@ func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alp func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { // Use the same normalized identity for endpoint creation and retirement. component := lpx.ServingComponent(source) - serviceName := dynamo.NormalizeKubeResourceName(dynamo.PCSNameForLPX(deployment, source) + "-" + component.ComponentName) + serviceName := dynamo.NormalizeKubeResourceName(dynamo.PCSNameForLPX(deployment) + "-" + component.ComponentName) if !commoncontroller.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, source.Annotations) { service := &corev1.Service{} @@ -451,6 +447,6 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a } // Keep the endpoint on this materialization even when another child shares the source. service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue - service.Spec.Selector[grovecommon.LabelPartOfKey] = dynamo.PCSNameForLPX(deployment, source) + service.Spec.Selector[grovecommon.LabelPartOfKey] = dynamo.PCSNameForLPX(deployment) return r.syncLPXResource(ctx, deployment, service) } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 80e7d8199b61..add8523cf782 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -70,7 +70,7 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { require.Nil(t, rejected) afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, afterSelected) require.Equal(t, beforePCS, afterPCS) - require.Equal(t, child.Annotations[lpx.DGDGenerationAnnotation], afterPCS.Annotations[lpx.DGDGenerationAnnotation]) + require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) _, err = r.reconcileSelectedLPX(t.Context(), child, afterSelected) require.NoError(t, err) afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} @@ -146,16 +146,16 @@ func TestLPXGPUCapacityReportsTheCompleteEngine(t *testing.T) { } } -func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { +func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { for _, scenario := range []struct { name, reason, message string preserve bool }{ {name: "intent", reason: "LPXRejected", message: "providerOverride"}, {name: "name budget", reason: "LPXRejected", message: "spec.components[0].name: Invalid value"}, - {name: "selected workload", reason: "LPXRejected", message: "scaling-group replicas"}, + {name: "selected workload", reason: "LPXRejected", message: "replica count"}, {name: "render", reason: "LPXReconciliationFailed", message: "model storage volume mount"}, - {name: "invalid source", reason: "LPXRejected", message: "source"}, + {name: "invalid source", reason: "LPXReconciliationFailed", message: "source"}, {name: "transient snapshot", reason: "LPXReconciliationFailed", message: "temporary snapshot timeout", preserve: true}, {name: "inconsistent snapshot", reason: "LPXReconciliationFailed", message: "immutable LPX build snapshot"}, } { @@ -168,7 +168,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { _, err := r.Reconcile(t.Context(), request) require.NoError(t, err) pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child, source)} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child)} require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) pcs.UID = "staged-pcs" require.NoError(t, r.Update(t.Context(), pcs)) @@ -200,7 +200,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { case "name budget": component.ComponentName = "serving-engines" case "selected workload": - component.Replicas = ptr.To(int32(2)) + component.Replicas = ptr.To(int32(-1)) case "render": component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].VolumeMounts = nil case "invalid source": @@ -219,7 +219,7 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { require.NoError(t, err) require.NoError(t, r.Update(t.Context(), child)) - t.Log("Converge cleanup and retain the original actionable failure on repeated reconciliation") + t.Log("Report the actionable failure without destroying existing resources") for range 3 { _, err = r.Reconcile(t.Context(), request) if scenario.reason == "LPXReconciliationFailed" && err != nil { @@ -236,15 +236,10 @@ func TestLPXTerminalFailureRetiresUnpublishedWorkload(t *testing.T) { require.Equal(t, metav1.ConditionTrue, failure.Status) require.Equal(t, scenario.reason, failure.Reason) require.Contains(t, failure.Message, scenario.message) - if scenario.preserve { - require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - require.Equal(t, beforePCS, pcs) - require.Equal(t, beforeEndpoint, endpoint) - } else { - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, pcs)), "terminal failure retained the staged PCS") - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), endpointKey, endpoint)), "terminal failure retained the obsolete endpoint") - } + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + require.Equal(t, beforePCS, pcs) + require.Equal(t, beforeEndpoint, endpoint) require.NoError(t, r.List(t.Context(), requests)) require.Empty(t, requests.Items) }) @@ -271,7 +266,7 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T }} foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" - foreignPCS.Name = dynamo.PCSNameForLPX(child, source) + foreignPCS.Name = dynamo.PCSNameForLPX(child) foreignEndpoint.Name = foreignPCS.Name + "-lpx" foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences if scenario == "pending finalizer" { @@ -390,11 +385,11 @@ func TestLPXEndpointLifecycle(t *testing.T) { t.Log("Publish the LPX-owned endpoint with its serving-role selector") require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) service := &corev1.Service{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child, source) + "-lpx"} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child) + "-lpx"} require.NoError(t, r.Get(t.Context(), key, service)) require.True(t, metav1.IsControlledBy(service, child)) require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) - require.Equal(t, dynamo.PCSNameForLPX(child, source), service.Spec.Selector[grovecommon.LabelPartOfKey]) + require.Equal(t, dynamo.PCSNameForLPX(child), service.Spec.Selector[grovecommon.LabelPartOfKey]) t.Log("Converge propagated endpoint metadata without losing identity, content or sync bookkeeping") for _, value := range []string{"changed", ""} { @@ -462,10 +457,10 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) - root := dynamo.PCSNameForLPX(child, source) + root := dynamo.PCSNameForLPX(child) for _, object := range []client.Object{ &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: root, Namespace: child.Namespace}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: root + "-lpu", Namespace: child.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: lpx.LPUConfigMapName(root, pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-" + lpx.ServingComponent(source).ComponentName, Namespace: child.Namespace}}, } { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) @@ -542,6 +537,7 @@ func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) } func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { + const componentProvider = "component provider" for _, scenario := range []struct { name, componentName string sharedDraft bool @@ -556,7 +552,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { "spec.components[0].topologyConstraint: Forbidden:", }}, {name: "disabled Grove", messages: []string{"Grove is disabled"}}, - {name: "component provider", messages: []string{"requires the Grove workload provider"}}, + {name: componentProvider, messages: []string{"requires the Grove workload provider"}}, {name: "too long serving name", componentName: "serving-engines", messages: []string{ "spec.components[0].name: Invalid value", "combined Grove resource name length 46 exceeds the 45-character limit", }}, @@ -597,7 +593,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { component.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{Enabled: true}} source.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} component.TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} - case "component provider": + case componentProvider: delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent } @@ -620,7 +616,11 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { t.Log("Reject unsupported intent before downloads and let valid names reach the download gate") _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) + if scenario.name == componentProvider { + require.ErrorContains(t, err, "requires the Grove workload provider") + } else { + require.NoError(t, err) + } stored := &v1alpha1.LPXGraphDeployment{} require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) failed := meta.FindStatusCondition(stored.Status.Conditions, "Failed") @@ -637,6 +637,8 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { require.Equal(t, metav1.ConditionTrue, failed.Status) if scenario.name == "disabled Grove" { require.Equal(t, "LPXUnavailable", failed.Reason) + } else if scenario.name == componentProvider { + require.Equal(t, "LPXReconciliationFailed", failed.Reason) } else { require.Equal(t, "LPXRejected", failed.Reason) } @@ -668,7 +670,7 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs, selected) + _, _, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs) require.ErrorContains(t, err, "input revision") allPCS := &grovev1alpha1.PodCliqueSetList{} require.NoError(t, r.List(t.Context(), allPCS)) @@ -679,5 +681,5 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), liveChild)) liveChild.Annotations[lpx.DGDGenerationAnnotation] = "999" require.NoError(t, r.Update(t.Context(), liveChild)) - require.ErrorContains(t, r.validateLPXPublicationSource(t.Context(), child), "publication metadata changed") + require.ErrorContains(t, r.validateLPXPublicationSource(t.Context(), child), "authority changed") } diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index eb990a9d6a57..1c05ca25cc61 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -16,6 +16,7 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -133,7 +134,7 @@ func (r *lpuEvictionReconciler) runtimePartitionByPodIndex(ctx context.Context, if pcsName == "" { return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no PodCliqueSet identity", trigger.Namespace, trigger.Name) } - configName := dynamolpx.LPUConfigMapName(pcsName) + configName := dynamolpx.LPUConfigMapName(pcsName, triggerConfigHash) var config corev1.ConfigMap if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: configName}, &config); err != nil { return nil, fmt.Errorf("get LPU runtime ConfigMap %s/%s: %w", trigger.Namespace, configName, err) @@ -192,9 +193,14 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev } model := trigger.Annotations[lpxv1alpha1.PodModelAnnotation] + owner := metav1.GetControllerOf(trigger) return slices.DeleteFunc(pods.Items, func(pod corev1.Pod) bool { + // Native updates can replace the clique or individual Pods within it. + candidateOwner := metav1.GetControllerOf(&pod) return !r.isLPUAgentPod(&pod) || + owner == nil || owner.UID == "" || candidateOwner == nil || owner.UID != candidateOwner.UID || + pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model }), nil } diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 229c16b7dee4..e98ef44df7bc 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -57,21 +58,25 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl } func newLPUEvictionConfigMap() *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS), Namespace: lpuEvictionTestNamespace}, + config := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: lpuEvictionTestNamespace}, + Immutable: ptr.To(true), Data: map[string]string{ "nodes_per_partition": "2\n1", "partition_node_offsets": "0\n2", }, } + config.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, dynamolpx.LPUConfigMapHash(config)) + return config } func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting bool, conditionReason string) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: lpuEvictionTestNamespace, - UID: types.UID(name + "-uid"), + Name: name, + Namespace: lpuEvictionTestNamespace, + UID: types.UID(name + "-uid"), + OwnerReferences: []metav1.OwnerReference{{APIVersion: "grove.io/v1alpha1", Kind: "PodClique", Name: "agents", UID: "agents-uid", Controller: ptr.To(true)}}, Labels: map[string]string{ grovecommon.LabelPartOfKey: lpuEvictionTestPCS, commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, @@ -154,13 +159,13 @@ func TestLPUEviction_PodsForTrigger(t *testing.T) { t.Run("lpu-gpu rejects a runtime table newer than the trigger", func(t *testing.T) { trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = "stale" + trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = "missing-config-version" sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") r, _ := newLPUEvictionReconciler(trigger, sibling) pods, err := r.podsForTrigger(context.Background(), trigger) - require.ErrorContains(t, err, "does not match trigger pod") + require.ErrorContains(t, err, "not found") assert.Nil(t, pods) }) @@ -208,7 +213,7 @@ func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) t.Log("Create the generated runtime table for a bounded materialization name unrelated to the source DGD") pcsName := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) config := newLPUEvictionConfigMap() - config.Name = dynamolpx.LPUConfigMapName(pcsName) + config.Name = dynamolpx.LPUConfigMapName(pcsName, dynamolpx.LPUConfigMapHash(config)) t.Log("Override the Agent config volume without replacing its generated partition table") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") @@ -231,6 +236,53 @@ func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) require.ErrorContains(t, err, "has no PodCliqueSet identity") } +func TestLPUEvictionPreservesNativeReplacement(t *testing.T) { + t.Log("Keep the original snapshot while a new configuration and native clique coexist") + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + replacement := newLPUEvictionPod("agent-1", "0", "1", false, "") + replacement.OwnerReferences[0].UID = "replacement-clique" + newConfig := newLPUEvictionConfigMap() + newConfig.Data["nodes_per_partition"] = "1\n2" + newConfig.Data["partition_node_offsets"] = "0\n1" + newConfig.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, dynamolpx.LPUConfigMapHash(newConfig)) + r, _ := newLPUEvictionReconciler(trigger, replacement, newConfig) + partitions, err := r.runtimePartitionByPodIndex(t.Context(), trigger) + require.NoError(t, err) + require.Equal(t, map[string]int{"0": 0, "1": 0, "2": 1}, partitions) + pods, err := r.podsForTrigger(t.Context(), trigger) + require.NoError(t, err) + require.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) +} + +func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { + for _, mode := range []lpxv1alpha1.WorkloadMode{ + lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly, + lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid, + } { + t.Run(string(mode), func(t *testing.T) { + t.Log("OnDelete can retain old Pods and replace others under the same clique and ConfigMap") + trigger := newLPUEvictionPod("agent-old-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + sibling := newLPUEvictionPod("agent-old-1", "0", "1", false, "") + replacement := newLPUEvictionPod("agent-new-1", "0", "1", false, "") + trigger.Labels[grovecommon.LabelPodTemplateHash] = "old-image" + sibling.Labels[grovecommon.LabelPodTemplateHash] = "old-image" + replacement.Labels[grovecommon.LabelPodTemplateHash] = "new-image" + trigger.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:old"}} + sibling.Spec.Containers = trigger.Spec.Containers + replacement.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:new"}} + require.Equal(t, trigger.OwnerReferences, replacement.OwnerReferences) + require.Equal(t, trigger.Annotations[commonconsts.AnnotationExtraResourcesHash], replacement.Annotations[commonconsts.AnnotationExtraResourcesHash]) + r, c := newLPUEvictionReconciler(trigger, sibling, replacement) + + t.Log("Evict the old revision's sibling without deleting the replacement") + reconcileLPUEviction(t, r, trigger.Name) + require.Equal(t, []string{replacement.Name}, remainingLPUEvictionPods(t, c)) + }) + } +} + func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { tests := []struct { name string diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index 4dd0887b3d5d..2e3aa520cd2a 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -19,88 +19,29 @@ package lpx import ( "context" - "fmt" - "strconv" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // reconcileGrovePodCliqueSetForLPX converges the already-rendered PodCliqueSet -// behind the LPX retirement fence. Only existingPodCliqueSet may be nil, when +// through native Grove rollout. Only existingPodCliqueSet may be nil, when // the controller observed no PCS; all decisions and writes use that observation. func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, existingPodCliqueSet *grovev1alpha1.PodCliqueSet, desired *grovev1alpha1.PodCliqueSet, - selectedLPX *lpxMaterializing, ) (*grovev1alpha1.PodCliqueSet, bool, error) { // A missing observation permits only creation after the publication fence. if existingPodCliqueSet == nil { if err := r.fenceLPXPublicationBeforeGroveSpecWrite(ctx, deployment, desired.Name); err != nil { return nil, false, err } - modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired) - return synced, modified, err } - // Retire and replace a changed attempt or spec; never update its spec in place. - change, err := commoncontroller.GetSpecChangeResult(existingPodCliqueSet, desired) - if err != nil { - return nil, false, fmt.Errorf("compare cached LPX PodCliqueSet with its successor: %w", err) - } - if change.SpecNeedsUpdate || !selectedLPX.hasCurrentLPXAttemptAnnotations(existingPodCliqueSet, deployment) { - return nil, false, r.recreateStaleLPXPodCliqueSet(ctx, deployment, desired, selectedLPX) - } - - // A current PCS keeps its spec and identity; only owned metadata may change. - if err := commoncontroller.CheckControllerOwnership(existingPodCliqueSet, deployment, r.Scheme()); err != nil { - return nil, false, err - } - prepared := existingPodCliqueSet - if change.NeedsUpdate { - prepared = existingPodCliqueSet.DeepCopy() - metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, *change.NewHash) - metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, commoncontroller.NvidiaAnnotationGenerationKey, strconv.FormatInt(change.NewGeneration, 10)) - } - for _, key := range [...]string{ - dynamolpx.WorkloadDigestAnnotation, - dynamolpx.DeploymentNameAnnotation, - lpxDeploymentUIDAnnotation, - lpxDeploymentGenerationAnnotation, - dynamo.LPXInputRevisionAnnotation, - dynamolpx.DGDUIDAnnotation, - dynamolpx.DGDGenerationAnnotation, - } { - value, found := desired.Annotations[key] - currentValue, currentFound := prepared.Annotations[key] - if found == currentFound && (!found || value == currentValue) { - continue - } - if prepared == existingPodCliqueSet { - prepared = existingPodCliqueSet.DeepCopy() - } - if found { - metav1.SetMetaDataAnnotation(&prepared.ObjectMeta, key, value) - } else { - delete(prepared.Annotations, key) - } - } - if prepared == existingPodCliqueSet { - return existingPodCliqueSet, false, nil - } - - // The observed resource version prevents overwriting any concurrent PCS edit. - if err := r.Update(ctx, prepared); err != nil { - r.GetRecorder().Eventf(existingPodCliqueSet, nil, corev1.EventTypeWarning, "UpdatePodCliqueSet", "Update", "Failed to update metadata for PodCliqueSet %s: %s", desired.Namespace, err) - return nil, false, err - } - r.GetRecorder().Eventf(prepared, nil, corev1.EventTypeNormal, "UpdatePodCliqueSet", "Update", "Updated metadata for PodCliqueSet %s", desired.Namespace) - return prepared, true, nil + // Match normal PCS synchronization; Grove owns rollout and immutable-field validation. + modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired) + return synced, modified, err } diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 7ac928cdc4a4..b1632b4f8e73 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -9,10 +9,10 @@ import ( "context" "crypto/sha256" "encoding/binary" + "encoding/json" "errors" "fmt" "hash" - "slices" "sort" "strconv" "strings" @@ -36,21 +36,19 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" ) const ( - lpxAttemptDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-attempt-digest" - lpxDeploymentGenerationAnnotation = dynamo.LPXDeploymentGenerationAnnotation - lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation - lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" - lpxPCSUIDAnnotation = "scheduling.lpu.nvidia.com/podcliqueset-uid" - lpxPodGangUIDAnnotation = "scheduling.lpu.nvidia.com/podgang-uid" - lpxRetirementRequeueAfter = 5 * time.Second - lpxLifecycleListPageSize = 64 - maxDGDConditionMessageSize = 32768 + lpxAttemptDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-attempt-digest" + lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation + lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" + lpxPCSUIDAnnotation = "scheduling.lpu.nvidia.com/podcliqueset-uid" + lpxPodGangUIDAnnotation = "scheduling.lpu.nvidia.com/podgang-uid" + lpxRetirementRequeueAfter = 5 * time.Second + lpxLifecycleListPageSize = 64 + maxDGDConditionMessageSize = 32768 ) const ( @@ -58,7 +56,7 @@ const ( lpxSchedulerStatusInvalidReason string = "LPXSchedulerStatusInvalid" ) -var errLPXGrovePodCliqueSetRecreating = errors.New("LPX successor is waiting while the stale Grove PodCliqueSet") +var errLPXEngineReplacing = errors.New("Grove is replacing the engine's PodCliques") // lpxClassification is deliberately private and closed: reconciliation can // report only one legal protocol state, never a freely assembled collection @@ -107,6 +105,7 @@ type lpxGroveIdentity struct { podGangName string podGangUID types.UID cyborgClique *lpxv1alpha1.CyborgPodCliqueReference + agentUIDs []types.UID complete bool } @@ -302,14 +301,10 @@ func (r *graphReconciler) reconcileLPXKnownIntentFence( } selected := source.HasLPXComponent() - modelNames, modelErr := lpx.SelectedModelNames(source) for index := range requests { request := &requests[index] - keep := selected && modelErr == nil && - slices.Contains(modelNames, request.Annotations[lpxModelAnnotation]) && + keep := selected && request.Annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && - request.Annotations[lpxDeploymentGenerationAnnotation] == strconv.FormatInt(deployment.Generation, 10) && - request.Annotations[dynamo.LPXInputRevisionAnnotation] == deployment.Spec.InputRevision && request.Annotations[lpx.ExecutionBackendAnnotation] == "" if keep { continue @@ -337,10 +332,31 @@ func (r *graphReconciler) prepareLPXMaterializing( return nil, &lpxRejected{reason: err.Error()}, nil } projections := workload.ModelProjections() - plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment, source)) + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) if err != nil { return nil, &lpxRejected{reason: err.Error()}, nil } + // Omitted replicas leave the native scaling group in charge of engine count. + if lpx.ServingComponent(source).Replicas == nil { + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: plan.LPXScalingGroup}, group); err != nil { + if !apierrors.IsNotFound(err) { + return nil, nil, err + } + } else { + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: plan.PodCliqueSetName}, pcs); err != nil { + return nil, nil, err + } + if !metav1.IsControlledBy(pcs, deployment) || !metav1.IsControlledBy(group, pcs) { + return nil, nil, fmt.Errorf("refusing to use replicas from a foreign LPX scaling group") + } + plan.Replicas = group.Spec.Replicas + } + } + if err := plan.ValidateReplicaCount(); err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } workloadDigest := workload.Digest().String() requests := make([]lpxModelMaterializing, 0, len(projections)*int(plan.Replicas)) for replicaIndex := int32(0); replicaIndex < plan.Replicas; replicaIndex++ { @@ -349,14 +365,11 @@ func (r *graphReconciler) prepareLPXMaterializing( deployment.Namespace, deployment.Name, deployment.UID, - deployment.Generation, projection.Model(), - projection.Digest().String(), replicaIndex, ) requests = append(requests, lpxModelMaterializing{ requestName: lpxRequestName(deployment.Name, requestDigest), - attemptDigest: requestDigest, modelProjection: projection, replicaIndex: replicaIndex, }) @@ -426,20 +439,14 @@ func digestAttemptKey( namespace string, name string, uid types.UID, - generation int64, model string, - projectionDigest string, replicaIndex int32, ) string { h := sha256.New() writeLPXHashField(h, "namespace", namespace) writeLPXHashField(h, "name", name) writeLPXHashField(h, "uid", string(uid)) - writeLPXHashField(h, "generation", strconv.FormatInt(generation, 10)) writeLPXHashField(h, "model", model) - // Keep the versioned field tag stable so a naming-only refactor does not - // rotate deterministic LPR names. - writeLPXHashField(h, "compiler-snapshot", projectionDigest) if replicaIndex > 0 { writeLPXHashField(h, "group-replica", strconv.FormatInt(int64(replicaIndex), 10)) } @@ -465,10 +472,7 @@ func (desired *lpxMaterializing) hasCurrentLPXAttemptAnnotations( } annotations := object.GetAnnotations() executionBackend := annotations[lpx.ExecutionBackendAnnotation] - return annotations[lpx.WorkloadDigestAnnotation] == desired.workloadDigest && - annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && - annotations[lpxDeploymentGenerationAnnotation] == strconv.FormatInt(deployment.Generation, 10) && - annotations[dynamo.LPXInputRevisionAnnotation] == deployment.Spec.InputRevision && + return annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && executionBackend == "" } @@ -486,7 +490,7 @@ func (r *graphReconciler) reconcileLPXAttemptFence( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, -) (map[string]*lpxv1alpha1.LPUPipelineRequest, *lpxRetiring, error) { +) (map[string]*lpxv1alpha1.LPUPipelineRequest, lpxClassification, error) { // Retain each desired position for digest lookup and collision precedence. desiredByName := make(map[string]int, len(desired.requests)) for index := range desired.requests { @@ -509,7 +513,7 @@ func (r *graphReconciler) reconcileLPXAttemptFence( } return nil } - if desiredName && request.Annotations[lpxAttemptDigestAnnotation] == desired.requests[desiredIndex].attemptDigest { + if desiredName { currents[request.Name] = request return nil } @@ -523,8 +527,31 @@ func (r *graphReconciler) reconcileLPXAttemptFence( return nil, nil, err } if stale != nil { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, stale, "The immutable build snapshot or complete attempt identity changed") - return nil, retiring, retireErr + // Persist cancellation or a replacement scheduling batch before retiring removed engines. + if attempt := currentLPXAttemptStatus(deployment); attempt != nil && attempt.ExceededAt == nil { + want := desiredLPXAttemptStatus(desired, deployment.Generation, attempt.PodCliqueSetUID) + if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, want, currents); transition != nil { + return currents, transition, nil + } + } + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, stale, "The engine is no longer requested", true) + return currents, retiring, retireErr + } + // Keep exact cleanup authority until the replaced request is authoritatively absent. + if attempt := currentLPXAttemptStatus(deployment); attempt != nil { + for index, row := range attempt.Requests { + if row.UID == "" || row.AttemptDigest != "" { + continue + } + if request := currents[row.Name]; request != nil && request.UID == row.UID { + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, "The engine's previous request is retiring", true) + return currents, retiring, retireErr + } + next := attempt.DeepCopy() + next.Requests[index].UID = "" + return currents, newLPXDeadlineTransition(next, nvidiacomv1beta1.DGDStatePending, + lpxAttemptPreparedReason, "The previous engine request has finished cleanup", time.Nanosecond), nil + } } // A deterministic-name collision must never be adopted or deleted. if foreignCollision < len(desired.requests) { @@ -733,35 +760,21 @@ func (r *graphReconciler) reconcileSelectedLPX( deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, ) (lpxClassification, error) { - currents := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) - - // Index the desired publication names before listing current requests. - desiredNames := make(map[string]struct{}, len(desired.requests)) - for _, request := range desired.requests { - desiredNames[request.requestName] = struct{}{} - } - - // Collect only exact LPXGraphDeployment-owned publications for the desired attempt. - if err := visitLifecycleObjectPages( - ctx, - r.apiReader, - &lpxv1alpha1.LPUPipelineRequestList{}, - func(object k8sruntime.Object) error { - live := object.(*lpxv1alpha1.LPUPipelineRequest) - if _, desired := desiredNames[live.Name]; !desired { - return nil - } - if !metav1.IsControlledBy(live, deployment) { - return fmt.Errorf("refusing to adopt foreign LPX request %q", live.Name) - } - currents[live.Name] = live - return nil - }, - client.InNamespace(deployment.Namespace), - ); err != nil { + // Retiring a removed engine must not block publication for surviving engines. + currents, retiring, err := r.reconcileLPXAttemptFence(ctx, deployment, desired) + if err != nil { return nil, err } - return r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents, true) + if _, recording := retiring.(*lpxDeadlineTransition); recording { + return retiring, nil + } + classification, err := r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents, true) + if retiring != nil && err == nil { + if _, recording := classification.(*lpxDeadlineTransition); !recording { + return retiring, nil + } + } + return classification, err } //nolint:gocyclo // Current-request retirement must remain visibly ordered before Grove observation. @@ -776,31 +789,6 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( currents = make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) } - // Retire invalid live requests before observing their Grove dependencies. - for _, request := range desired.requests { - live := currents[request.requestName] - if live == nil { - continue - } - if sizeErr := lpx.ValidateRequestSize(live); sizeErr != nil { - return r.retireLPXRequest( - ctx, - deployment, - desired.plan.PodCliqueSetName, - live, - fmt.Sprintf("LPX request exceeds the scheduler size budget: %v", sizeErr), - ) - } - if !live.DeletionTimestamp.IsZero() { - return r.retireLPXRequest( - ctx, - deployment, - desired.plan.PodCliqueSetName, - live, - "Waiting for the current LPX request finalizer to finish cleanup", - ) - } - } groveIdentities, pending, classification, err := r.reconcileSelectedLPXGroveIdentityPrefix( ctx, deployment, @@ -827,19 +815,21 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( } } - // Publish missing requests in the fully observed prefix through one immutable attempt projection. + // Prepare the batch from any ready engine, independently of replica zero's rollout. var attemptProjection *nvidiacomv1beta1.LPXAttemptStatus - if baseGroveIdentity := groveIdentities[0]; baseGroveIdentity != nil && baseGroveIdentity.complete { - if desired.deadlineSeconds != nil { - attemptProjection = desiredLPXAttemptStatus(desired, deployment.Generation, baseGroveIdentity.pcsUID) + trackAttempt := desired.deadlineSeconds != nil || currentLPXAttemptStatus(deployment) != nil + for _, baseGroveIdentity := range groveIdentities { + if baseGroveIdentity == nil || !baseGroveIdentity.complete { + continue } - transition, err := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) - if err != nil { - return nil, err + if trackAttempt { + attemptProjection = desiredLPXAttemptStatus(desired, deployment.Generation, baseGroveIdentity.pcsUID) } + transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) if transition != nil { return transition, nil } + break } for index := range desired.requests { request := &desired.requests[index] @@ -848,19 +838,22 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( continue } if attemptProjection != nil { + // Classify known authority loss before attempting a replacement publication. + attempt := currentLPXAttemptStatus(deployment) + if _, _, missing := exactLPXAttemptState(attempt, currents); missing { + return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStateFailed, + lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring its scheduling batch without replacement", 0), nil + } if err := r.revalidateLPXAttemptPublication(ctx, deployment, desired.plan.PodCliqueSetName); err != nil { return nil, err } } - live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity, desired.deadlineSeconds != nil) + live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity, trackAttempt) if createErr != nil { return nil, createErr } currents[request.requestName] = live - transition, transitionErr := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) - if transitionErr != nil { - return nil, transitionErr - } + transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) if transition != nil { return transition, nil } @@ -901,20 +894,7 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( return nil, nil, nil, err } if snapshot == nil { - if len(currents) == 0 { - return nil, nil, &lpxClosed{incomplete: incomplete}, nil - } - - // Use the first published request as the witness for attempt-wide retirement when the shared Grove snapshot disappears. - reason := fmt.Sprintf("The published Grove identity became unavailable: %s", incomplete) - firstName := "" - for name := range currents { - if firstName == "" || name < firstName { - firstName = name - } - } - retiring, err := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, currents[firstName], reason) - return nil, nil, retiring, err + return nil, nil, &lpxClosed{incomplete: incomplete}, nil } groveIdentities := make([]*lpxGroveIdentity, desired.plan.Replicas) @@ -928,10 +908,21 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( } } var pending *lpxClosed + retire := make(map[int32]bool) + intentional := make(map[int32]bool) for replicaIndex, plan := range replicaPlans { identity, incomplete, err := r.observeLPXGroveIdentity(ctx, deployment, desired, snapshot, plan, allAgentGroups) if err != nil { - return nil, nil, nil, err + var drift *lpxRetiring + if errors.Is(err, errLPXEngineReplacing) { + intentional[int32(replicaIndex)] = true + incomplete = err.Error() + } else if errors.As(err, &drift) { + incomplete = drift.retirementReason + } else { + return nil, nil, nil, err + } + retire[int32(replicaIndex)] = true } if incomplete != "" { if pending == nil { @@ -939,30 +930,61 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( } } if identity != nil { - identity.complete = pending == nil + identity.complete = incomplete == "" groveIdentities[replicaIndex] = identity } } - // Fence exact PCS, PodGang, and Cyborg replacement identities. + // Observe retirement per engine; an incomplete sibling never fences this engine. for index := range desired.requests { request := &desired.requests[index] live := currents[request.requestName] + identity := groveIdentities[request.replicaIndex] + if identity != nil && identity.complete { + spec, err := json.Marshal(struct { + Spec lpxv1alpha1.LPUPipelineRequestSpec + AgentUIDs []types.UID + }{request.modelProjection.RequestSpec(deployment.Namespace, identity.podGangName, identity.cyborgClique), identity.agentUIDs}) + if err != nil { + return nil, nil, nil, err + } + request.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) + } else if live != nil { + request.attemptDigest = live.Annotations[lpxAttemptDigestAnnotation] + } if live == nil { continue } - identity := groveIdentities[request.replicaIndex] - if identity == nil { - reason := "The published replica no longer has its exact Grove identity" - if pending != nil && pending.incomplete != "" { - reason = fmt.Sprintf("%s: %s", reason, pending.incomplete) - } - retiring, err := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, live, reason) - return nil, nil, retiring, err + if identity != nil && identity.complete && validateCurrentLPXRequest(deployment, request, identity, live) != nil { + intentional[request.replicaIndex] = true } - if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, desired.plan.PodCliqueSetName, live, err.Error()) - return nil, nil, retiring, retireErr + if intentional[request.replicaIndex] || !live.DeletionTimestamp.IsZero() || lpx.ValidateRequestSize(live) != nil { + retire[request.replicaIndex] = true + } + } + + // All model requests of a replacing engine must finish before any is republished. + for index := range desired.requests { + request := &desired.requests[index] + if !retire[request.replicaIndex] { + continue + } + request.attemptDigest = "" + groveIdentities[request.replicaIndex] = nil + pending = &lpxClosed{incomplete: "Waiting for the replaced engine's scheduler requests to finish cleanup"} + if live := currents[request.requestName]; live != nil { + classification, err := r.retireLPXEngineRequest(ctx, deployment, live, pending.incomplete, intentional[request.replicaIndex]) + if transition, recording := classification.(*lpxDeadlineTransition); recording { + if _, completed, _ := exactLPXAttemptState(currentLPXAttemptStatus(deployment), currents); completed { + disarmedAt := metav1.Now() + transition.attempt.DisarmedAt = &disarmedAt + } + return nil, nil, classification, err + } + if err != nil { + return nil, nil, classification, err + } + delete(currents, request.requestName) } } return groveIdentities, pending, nil, nil @@ -1006,72 +1028,6 @@ func (r *graphReconciler) fenceLPXPublicationBeforeGroveSpecWrite( return err } -// recreateStaleLPXPodCliqueSet gives a successor LPX publication a fresh Grove -// identity whenever the selected attempt changed or its Grove spec needs a -// write. Grove permits rollout updates to mutable template fields, but rejects -// changes to clique composition and several scheduling fields. Treating the -// whole retired spec as replace-only avoids duplicating that evolving admission -// contract in Dynamo. -// -// The old request is the scheduler's publication boundary. Deletion is allowed -// only after every exactly-owned request is authoritatively gone and the -// exactly-owned PCS is held at zero, which is also the point at which LPX has -// finished releasing the old workload Pod cohort. -// desiredPCS and desired are non-nil; the caller observed a stale PCS. -func (r *graphReconciler) recreateStaleLPXPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desiredPCS *grovev1alpha1.PodCliqueSet, - desired *lpxMaterializing, -) error { - // Retire all exactly-owned requests before changing shared Grove state. - retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, desiredPCS.Name, "LPX publication was retired before recreating its Grove PodCliqueSet") - if err != nil { - return err - } - if retiring != nil { - return retiring - } - - key := types.NamespacedName{Namespace: deployment.Namespace, Name: desiredPCS.Name} - live := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, key, live); err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("%w is observed deleted", errLPXGrovePodCliqueSetRecreating) - } - return err - } - if !metav1.IsControlledBy(live, deployment) { - return fmt.Errorf("refusing to recreate PodCliqueSet %q without the exact LPXGraphDeployment controller owner", live.Name) - } - change, err := commoncontroller.GetSpecChangeResult(live, desiredPCS) - if err != nil { - return fmt.Errorf("compare authoritative LPX PodCliqueSet with its successor: %w", err) - } - if desired.hasCurrentLPXAttemptAnnotations(live, deployment) && !change.SpecNeedsUpdate { - // The authoritative object is current and the controller cache is not. - return fmt.Errorf("%w is observed by the controller cache", errLPXGrovePodCliqueSetRecreating) - } - if !live.DeletionTimestamp.IsZero() { - return fmt.Errorf("%w is deleted", errLPXGrovePodCliqueSetRecreating) - } - if live.Spec.Replicas != 0 { - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, live.Name, ""); err != nil { - return err - } - return fmt.Errorf("%w is scaled to zero", errLPXGrovePodCliqueSetRecreating) - } - - uid := live.UID - resourceVersion := live.ResourceVersion - if err := r.Delete(ctx, live, &client.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}, - }); err != nil && !apierrors.IsNotFound(err) { - return err - } - return fmt.Errorf("%w is deleted", errLPXGrovePodCliqueSetRecreating) -} - func (r *graphReconciler) createPublishedLPXRequest( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -1126,16 +1082,13 @@ func lpxRequestAnnotations( // Callers validate the exact source controller owner before constructing requests. sourceOwner := metav1.GetControllerOf(deployment) return map[string]string{ - lpxAttemptDigestAnnotation: materializing.attemptDigest, - dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, - lpx.DGDUIDAnnotation: string(sourceOwner.UID), - lpx.DGDGenerationAnnotation: deployment.Annotations[lpx.DGDGenerationAnnotation], - lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), - lpxDeploymentUIDAnnotation: string(deployment.UID), - lpxModelAnnotation: materializing.modelProjection.Model(), - lpxPCSUIDAnnotation: string(grove.pcsUID), - lpxPodGangUIDAnnotation: string(grove.podGangUID), - lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), + lpxAttemptDigestAnnotation: materializing.attemptDigest, + lpx.DGDUIDAnnotation: string(sourceOwner.UID), + lpxDeploymentUIDAnnotation: string(deployment.UID), + lpxModelAnnotation: materializing.modelProjection.Model(), + lpxPCSUIDAnnotation: string(grove.pcsUID), + lpxPodGangUIDAnnotation: string(grove.podGangUID), + lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), } } @@ -1216,6 +1169,33 @@ func (r *graphReconciler) observeLPXGroveIdentity( plan *lpx.MaterializationPlan, allAgentGroups map[string]struct{}, ) (*lpxGroveIdentity, string, error) { + incomplete := snapshot.incomplete + // Observe native retirement before gang availability or successor revision readiness. + agentUIDs := make([]types.UID, 0, len(plan.Agents)) + projections := desired.workload.ModelProjections() + for index, agent := range plan.Agents { + clique := &grovev1alpha1.PodClique{} + if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: agent.CliqueName}, clique); err != nil { + if apierrors.IsNotFound(err) { + incomplete = fmt.Sprintf("Waiting for Agent PodClique %q", agent.CliqueName) + continue + } + return nil, "", err + } + if !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup) || clique.UID == "" { + incomplete = fmt.Sprintf("Waiting for owned Agent PodClique %q", agent.CliqueName) + continue + } + if !clique.DeletionTimestamp.IsZero() { + return nil, "", errLPXEngineReplacing + } + agentUIDs = append(agentUIDs, clique.UID) + if !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || + clique.Annotations[lpx.WorkloadDigestAnnotation] != projections[index].Digest().String() || + clique.Status.CurrentPodCliqueSetGenerationHash == nil || *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash { + incomplete = fmt.Sprintf("Waiting for current Agent PodClique %q", agent.CliqueName) + } + } lpxPodGang, ordinaryPodGang, wait := selectLPXPublicationPodGangs( deployment.Namespace, snapshot.podGangs, @@ -1230,8 +1210,8 @@ func (r *graphReconciler) observeLPXGroveIdentity( pcsUID: snapshot.podCliqueSet.UID, podGangName: lpxPodGang.Name, podGangUID: lpxPodGang.UID, + agentUIDs: agentUIDs, } - incomplete := snapshot.incomplete if !desired.hasCurrentLPXAttemptAnnotations(lpxPodGang, deployment) || !desired.hasCurrentLPXAttemptAnnotations(ordinaryPodGang, deployment) { if incomplete == "" { @@ -1328,13 +1308,6 @@ func (r *graphReconciler) observeLPXPodCliqueSet( } func observedLPXPCSGenerationHash(pcs *grovev1alpha1.PodCliqueSet) (string, string) { - if pcs.Status.ObservedGeneration == nil || *pcs.Status.ObservedGeneration < pcs.Generation { - return "", fmt.Sprintf( - "Waiting for Grove to observe PodCliqueSet %q generation %d", - pcs.Name, - pcs.Generation, - ) - } if pcs.Status.CurrentGenerationHash == nil || *pcs.Status.CurrentGenerationHash == "" { return "", fmt.Sprintf( "Waiting for Grove to publish PodCliqueSet %q generation identity", @@ -1388,7 +1361,6 @@ func (r *graphReconciler) observeLPXServingClique( clique.Spec.PodSpec.SchedulerName != corev1.DefaultSchedulerName || clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != role || !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || - (isCyborg && !hasLPXCyborgDeviceIntent(clique.Spec.PodSpec)) || (!isCyborg && !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup)) { return nil, fmt.Sprintf( "%s PodClique %q does not match the current ordinary-scheduler identity", @@ -1396,6 +1368,9 @@ func (r *graphReconciler) observeLPXServingClique( ), nil } if !isCyborg { + if clique.Status.CurrentPodCliqueSetGenerationHash == nil || *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash { + return nil, fmt.Sprintf("Waiting for current conductor PodClique %q", clique.Name), nil + } return nil, "", nil } @@ -1406,11 +1381,15 @@ func (r *graphReconciler) observeLPXServingClique( clique.Name, ), nil } - if !hasExpectedLPXCyborgClaimReferences(snapshot.podCliqueSet, clique, plan.CyborgTemplate) { - return nil, fmt.Sprintf( - "Cyborg PodClique %q DRA Claim references changed during Grove materialization", - clique.Name, - ), nil + if !hasLPXCyborgDeviceIntent(clique.Spec.PodSpec) || !hasExpectedLPXCyborgClaimReferences(snapshot.podCliqueSet, clique, plan.CyborgTemplate) { + reason := fmt.Sprintf("Cyborg PodClique %q GPU intent or DRA Claim references changed during Grove materialization", clique.Name) + // Until Grove has accepted the PCS edit, differing references may still be its old engine. + pcs := snapshot.podCliqueSet + if pcs.Status.ObservedGeneration != nil && *pcs.Status.ObservedGeneration == pcs.Generation && + clique.Status.CurrentPodCliqueSetGenerationHash != nil && *clique.Status.CurrentPodCliqueSetGenerationHash == snapshot.generationHash { + return nil, "", &lpxRetiring{retirementReason: reason} + } + return nil, reason, nil } reference := &lpxv1alpha1.CyborgPodCliqueReference{Name: clique.Name, UID: string(clique.UID)} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 2de088c50827..9346622a95fb 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -31,6 +31,7 @@ import ( manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/require" @@ -304,7 +305,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { agentClique.Spec.Replicas = 9 createLPXTestObjects(t, ctx, reconciler.Client, objects...) - t.Log("Allow only the deterministic Cyborg and parent-PCSG reads; reject Pods and Agent cliques") + t.Log("Allow only deterministic engine-clique and parent-PCSG reads; reject Pods and broad clique lists") reader, ok := reconciler.apiReader.(client.WithWatch) require.True(t, ok) cachedClient, ok := reconciler.Client.(client.WithWatch) @@ -315,6 +316,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { scalingGroupGets := 0 conductorGets := 0 cyborgGets := 0 + agentGets := 0 forbidden := interceptor.Funcs{ Get: func( ctx context.Context, @@ -336,6 +338,12 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { known := false for _, request := range desired.requests { plan := desired.plan.ForReplica(request.replicaIndex) + for _, agent := range plan.Agents { + if key.Name == agent.CliqueName { + agentGets++ + known = true + } + } if key.Name == plan.ConductorClique { conductorGets++ known = true @@ -386,6 +394,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { require.Equal(t, 1, scalingGroupGets) require.Zero(t, conductorGets) require.Equal(t, 2, cyborgGets) + require.Equal(t, 2, agentGets) requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) require.NoError(t, err) require.Len(t, requests, 2) @@ -404,6 +413,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { require.Equal(t, 2, scalingGroupGets) require.Zero(t, conductorGets) require.Equal(t, 4, cyborgGets) + require.Equal(t, 4, agentGets) t.Log("Leave unsupported generated-child replica mutations untouched") require.NoError(t, cachedClient.Get(ctx, client.ObjectKeyFromObject(scalingGroup), scalingGroup)) @@ -437,7 +447,7 @@ func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { lifecycleRequestListCalls, dependencyListCalls = 0, 0 t.Log("Re-establish scale-to-zero using only the authoritative attempt-wide request observation") - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconciler.retireLPXRequest(ctx, dgd, desired.plan.PodCliqueSetName, &requests[0], "test group retirement") require.NoError(t, err) require.IsType(t, &lpxRetiring{}, classification) require.Positive(t, lifecycleRequestListCalls) @@ -490,21 +500,18 @@ func TestSelectedLPXObservesLimitsOnlyCyborgGPUIntent(t *testing.T) { require.NoError(t, reconciler.Update(t.Context(), storedCyborg)) classification, err = reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) require.NoError(t, err) - retiring, ok := classification.(*lpxRetiring) - require.True(t, ok) - require.Contains(t, retiring.retirementReason, "ordinary-scheduler identity") + require.IsType(t, &lpxClosed{}, classification) requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) } -func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { +func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { for _, test := range []struct { - name string - snapshotError error - retirementMessage string + name string + snapshotError error }{ - {name: "inconsistent snapshot", snapshotError: fmt.Errorf("%w: compiler metadata changed during duplicate reads", lpx.ErrBuildSnapshotInconsistent), retirementMessage: "The model build became inconsistent while acquiring the immutable snapshot:"}, + {name: "inconsistent snapshot", snapshotError: fmt.Errorf("%w: compiler metadata changed during duplicate reads", lpx.ErrBuildSnapshotInconsistent)}, {name: "transient snapshot", snapshotError: errors.New("temporary object-store timeout")}, - {name: "render failure", retirementMessage: "The current LPX workload can no longer be rendered safely:"}, + {name: "render failure"}, } { t.Run(test.name, func(t *testing.T) { t.Log("Publish a request backed by the rendered LPX workload") @@ -529,34 +536,12 @@ func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { } else { reconciler.Config.MPI.SSHSecretName = "" } - result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) pcs := &grovev1alpha1.PodCliqueSet{} - if test.retirementMessage == "" { - t.Log("A transient snapshot failure must preserve the exact published request and PCS") - require.Equal(t, requestBefore, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name)) - require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) - require.Equal(t, pcsBefore, pcs) - } else { - t.Log("Retire the request while retaining its stopped PCS and actionable reason") - require.NoError(t, err) - require.Positive(t, result.RequeueAfter) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name) - require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) - require.Zero(t, pcs.Spec.Replicas) - require.NoError(t, reconciler.Get(ctx, key, dgd)) - ready := meta.FindStatusCondition(dgd.Status.Conditions, "Ready") - require.NotNil(t, ready) - require.Equal(t, lpxRetiringReason, ready.Reason) - require.Contains(t, ready.Message, test.retirementMessage) - require.Contains(t, ready.Message, message) - - t.Log("Remove the old PCS only after its request is gone") - result, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.Positive(t, result.RequeueAfter) - require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, pcs))) - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - } + t.Log("Failed desired input must preserve the exact published request and PCS") + require.Equal(t, requestBefore, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name)) + require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) + require.Equal(t, pcsBefore, pcs) t.Log("Persist the actual actionable failure and retain its wrapped cause") require.ErrorContains(t, err, message) @@ -665,14 +650,13 @@ func TestLPXAuthoritativeLifecycleListConsumesContinuePages(t *testing.T) { } func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { - t.Log("Prepare the current LPX attempt with an independently named materialization") + t.Log("Render initial root metadata and seed an exact-owned PCS with native Grove controls") ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Name = "metadata-materialization" + source.Spec.Labels = map[string]string{"test.example/version": "before", "kai.scheduler/removed": "before"} + source.Spec.Annotations = maps.Clone(source.Spec.Labels) prepare, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) desired := renderLPXTestPodCliqueSet(t, ctx, prepare, dgd, source, selected) - - t.Log("Seed a PCS already owned by the exact LPX child; metadata synchronization never adopts resources") cached := desired.DeepCopy() require.NoError(t, ctrl.SetControllerReference(dgd, cached, prepare.Scheme())) cached.UID = "cached-pcs-uid" @@ -682,119 +666,79 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.NoError(t, err) metav1.SetMetaDataAnnotation(&cached.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) delete(cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) - cached.Annotations[lpx.DGDGenerationAnnotation] = "0" - cached.Annotations["test.example/keep"] = "kept" + cached.Finalizers = []string{groveconstants.FinalizerPodCliqueSet} + cached.Annotations[groveconstants.AnnotationDisableManagedResourceProtection] = "true" + cached.Annotations[groveconstants.AnnotationReconcileTrigger] = "previous-trigger" + before := cached.DeepCopy() + for _, metadata := range []map[string]string{desired.Labels, desired.Annotations} { + metadata["test.example/version"] = "after" + metadata["kai.scheduler/added"] = "" + delete(metadata, "kai.scheduler/removed") + } - t.Log("Observe metadata writes and confirm reconciliation does not fetch another cached PCS") + t.Log("Record writes and reject extra reads of the supplied observation") reconciler := newLPXTestReconciler(t, registry, dgd, source, cached) base, ok := reconciler.Client.(client.WithWatch) require.True(t, ok) - gets := 0 updates := 0 reconciler.Client = interceptor.NewClient(base, interceptor.Funcs{ Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - gets++ - return delegated.Get(ctx, key, object, opts...) + return errors.New("PCS synchronization must use its supplied observation") }, Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { updates++ return delegated.Update(ctx, object, opts...) }, }) - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - fenceLists := 0 - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - fenceLists++ - return delegated.List(ctx, list, opts...) - }, - }) - t.Log("Repair source generation and bookkeeping together without changing the attempt or spec") - synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) + t.Log("Only bookkeeping changes; root metadata additions, edits and removals are ignored") + synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) require.NoError(t, err) require.True(t, modified) - require.Zero(t, gets) require.Equal(t, 1, updates) - require.Zero(t, fenceLists) - require.Equal(t, cached.UID, synced.UID) - bookkeeping := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(desired), bookkeeping)) - require.Equal(t, bookkeeping.ResourceVersion, synced.ResourceVersion) - require.True(t, metav1.IsControlledBy(synced, dgd)) - require.Equal(t, cached.Spec, synced.Spec) - require.Equal(t, desired.Annotations[lpx.DGDGenerationAnnotation], synced.Annotations[lpx.DGDGenerationAnnotation]) - require.Equal(t, desired.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) - require.Equal(t, desired.Annotations[lpxDeploymentUIDAnnotation], synced.Annotations[lpxDeploymentUIDAnnotation]) - require.Equal(t, "kept", synced.Annotations["test.example/keep"]) - require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) - require.Equal(t, "4", synced.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) - require.Equal(t, "0", cached.Annotations[lpx.DGDGenerationAnnotation], "cached input must remain untouched") - require.NotContains(t, cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) + want := before.DeepCopy() + want.ResourceVersion = synced.ResourceVersion + want.Annotations[commoncontroller.NvidiaAnnotationGenerationKey] = "4" + require.Equal(t, want, synced) + require.Equal(t, before, cached, "the supplied observation must not be mutated") - t.Log("An unchanged observation is a no-op") + t.Log("Root metadata differences alone are a no-op once bookkeeping is current") updates = 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired, selected) + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) require.NoError(t, err) require.False(t, modified) require.Zero(t, updates) - require.Zero(t, fenceLists) - require.Equal(t, cached.UID, synced.UID) - - for _, name := range []string{"", "other-materialization"} { - t.Log("Drift only the materialization name while preserving the current workload and source metadata", name) - cached = synced.DeepCopy() - if name == "" { - delete(cached.Annotations, lpx.DeploymentNameAnnotation) - } else { - cached.Annotations[lpx.DeploymentNameAnnotation] = name - } - require.NoError(t, base.Update(ctx, cached)) - before := cached.DeepCopy() - updates = 0 + require.Equal(t, want, synced) - t.Log("Restore the name with one metadata-only update and leave the observation untouched") - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) - require.NoError(t, err) - require.True(t, modified) - require.Equal(t, 1, updates) - require.Zero(t, gets) - require.Zero(t, fenceLists) - require.Equal(t, before, cached) - want := before.DeepCopy() - want.Annotations[lpx.DeploymentNameAnnotation] = dgd.Name - want.ResourceVersion = synced.ResourceVersion - require.Equal(t, want, synced) - - t.Log("A repaired materialization name is stable on the next reconciliation") - updates = 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired, selected) - require.NoError(t, err) - require.False(t, modified) - require.Zero(t, updates) - require.Zero(t, fenceLists) - require.Equal(t, want, synced) - } + t.Log("Template metadata still updates through Spec without changing root metadata or native controls") + desired.Spec.Template.Cliques[0].Annotations = map[string]string{"test.example/template": "next"} + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + require.NoError(t, err) + require.True(t, modified) + require.Equal(t, 1, updates) + want.Spec = desired.Spec + want.ResourceVersion = synced.ResourceVersion + want.Annotations[commoncontroller.NvidiaAnnotationHashKey], err = commoncontroller.GetSpecHash(desired) + require.NoError(t, err) + want.Annotations[commoncontroller.NvidiaAnnotationGenerationKey] = "5" + require.True(t, apiequality.Semantic.DeepEqual(want, synced)) - t.Log("A spec edit after observation conflicts instead of being overwritten by metadata repair") - synced.Annotations[lpx.DGDGenerationAnnotation] = "0" - require.NoError(t, base.Update(ctx, synced)) + t.Log("A concurrent spec edit conflicts instead of being overwritten from a stale observation") cached = synced.DeepCopy() concurrent := synced.DeepCopy() concurrent.Spec.Replicas++ concurrent.Generation++ require.NoError(t, base.Update(ctx, concurrent)) - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired, selected) + desired.Spec.Replicas += 2 + updates = 0 + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) require.Nil(t, synced) require.False(t, modified) require.Equal(t, 1, updates) - require.Zero(t, gets) - require.Zero(t, fenceLists) - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(concurrent), bookkeeping)) - require.True(t, apiequality.Semantic.DeepEqual(concurrent.Spec, bookkeeping.Spec)) - require.Equal(t, concurrent.ResourceVersion, bookkeeping.ResourceVersion) + stored := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(concurrent), stored)) + require.Equal(t, concurrent, stored) } func TestLPXPodCliqueSetPublicationFence(t *testing.T) { @@ -831,7 +775,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { }) t.Log("A failed publication observation prevents initial creation") - synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.ErrorIs(t, err, fenceErr) require.False(t, modified) require.Nil(t, synced) @@ -842,13 +786,15 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { t.Log("Recover the observation and create the exact child-owned PCS") failFence = false writes, lists = 0, 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.NoError(t, err) require.True(t, modified) require.Equal(t, 1, lists) require.Equal(t, 1, writes) require.True(t, apiequality.Semantic.DeepEqual(desired.Spec, synced.Spec)) require.True(t, metav1.IsControlledBy(synced, dgd)) + require.Equal(t, desired.Labels, synced.Labels) + require.Equal(t, desired.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) hash, hashErr := commoncontroller.GetSpecHash(desired) require.NoError(t, hashErr) require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) @@ -858,11 +804,11 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) t.Log("A stale missing observation does not adopt an already-created PCS") - _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy(), selected) + _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) require.False(t, modified) - t.Log("An observation outage also fences replacement without changing the current PCS") + t.Log("An in-place update does not require listing every scheduler request") live := synced desired = live.DeepCopy() desired.ResourceVersion, desired.UID = "", "" @@ -870,18 +816,18 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" failFence = true writes, lists = 0, 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired, selected) - require.ErrorIs(t, err, fenceErr) - require.False(t, modified) - require.Nil(t, synced) - require.Zero(t, writes) - require.Equal(t, 1, lists) + synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired) + require.NoError(t, err) + require.True(t, modified) + require.Equal(t, live.UID, synced.UID) + require.Equal(t, 1, writes) + require.Zero(t, lists) require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(live), stored)) - require.True(t, apiequality.Semantic.DeepEqual(live.Spec, stored.Spec)) - require.Equal(t, live.Annotations, stored.Annotations) + require.True(t, apiequality.Semantic.DeepEqual(desired.Spec, stored.Spec)) + require.Equal(t, live.Annotations[lpx.WorkloadDigestAnnotation], stored.Annotations[lpx.WorkloadDigestAnnotation]) } -func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) { +func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) @@ -892,6 +838,7 @@ func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) require.NoError(t, err) _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) + published := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} drifted := &grovev1alpha1.PodCliqueSet{} @@ -906,16 +853,15 @@ func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) dgd, drifted, rendered, - desired, ) - var retiring *lpxRetiring - require.ErrorAs(t, err, &retiring) - require.NotNil(t, retiring) - requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) + require.NoError(t, err) + require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) - t.Log("Hold the retired PCS at zero before any successor write") + t.Log("Correct the spec in place without stopping the existing PCS") require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - require.Zero(t, drifted.Spec.Replicas) + require.Equal(t, int32(1), drifted.Spec.Replicas) + require.Equal(t, types.UID("pcs-uid"), drifted.UID) + require.NotEqual(t, "externally-mutated", drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName) foreignPCS := drifted.DeepCopy() foreignPCS.ResourceVersion = "" @@ -925,6 +871,9 @@ func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) _, err = foreignReconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil, false) require.EqualError(t, err, fmt.Sprintf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", foreignPCS.Name)) + t.Log("An explicitly stopped PCS still cannot publish another request") + drifted.Spec.Replicas = 0 + require.NoError(t, reconciler.Update(ctx, drifted)) reader, ok := reconciler.apiReader.(client.WithWatch) require.True(t, ok) pcsGets := 0 @@ -948,37 +897,26 @@ func TestLPXPreflightDoesNotRepublishBeforeDeferredGroveRecreation(t *testing.T) require.Equal(t, 1, pcsGets) reconciler.apiReader = reader - require.Equal(t, "externally-mutated", drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName) + t.Log("Restore the explicit hold in place without rotating the existing request") _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( ctx, dgd, drifted, rendered, - desired, - ) - require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) - require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is deleted") - require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, &grovev1alpha1.PodCliqueSet{}))) - requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) - - _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( - ctx, - dgd, - nil, - rendered, - desired, ) require.NoError(t, err) recreated := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) + require.Equal(t, drifted.UID, recreated.UID) for _, clique := range recreated.Spec.Template.Cliques { require.NotEqual(t, "externally-mutated", clique.Spec.PodSpec.PriorityClassName) } - requireLPXRequestNotFound(t, ctx, reconciler.apiReader, dgd.Namespace, desired.requests[0].requestName) + require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) } -func TestGroveSpecSyncRecreatesStaleLPXAttemptBeforeSuccessor(t *testing.T) { +func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { + t.Log("Keep an existing two-partition engine while preparing a three-partition successor") ctx := t.Context() oldDGD, oldSource, oldRegistry := newLPXTestDGD(t, lpx.PipelineSingle) oldPrepareReconciler := newLPXTestReconciler(t, oldRegistry, oldDGD, oldSource) @@ -986,6 +924,7 @@ func TestGroveSpecSyncRecreatesStaleLPXAttemptBeforeSuccessor(t *testing.T) { require.Nil(t, rejected) oldPCS := findLPXTestPodCliqueSet(t, lpxMaterializedObjects(t, oldPrepareReconciler, oldDGD, oldSource, oldDesired)) + published := deadlineTestRequest(oldDGD, oldDesired.requests[0].requestName, time.Now(), lpxv1alpha1.RequestPhaseBound) const successorBuildID = "build-v2-successor" successorRegistry := newLPXTestRegistryWithPartitionsAndMode( @@ -998,75 +937,52 @@ func TestGroveSpecSyncRecreatesStaleLPXAttemptBeforeSuccessor(t *testing.T) { successorSource := oldSource.DeepCopy() successorDGD.Generation++ successorSource.Spec.Components[0].LPX.BuildID = successorBuildID - reconciler := newLPXTestReconciler(t, successorRegistry, successorDGD, successorSource) + reconciler := newLPXTestReconciler(t, successorRegistry, successorDGD, successorSource, oldPCS, published) successorDesired, rejected := requirePreparedLPX(t, reconciler, ctx, successorDGD, successorSource) require.Nil(t, rejected) - createLPXTestObjects(t, ctx, reconciler.Client, oldPCS) + t.Log("The successor changes Agent minAvailable, which pinned Grove rejects as immutable") pcs := &grovev1alpha1.PodCliqueSet{} pcsKey := types.NamespacedName{ Namespace: successorDGD.Namespace, Name: successorDesired.plan.PodCliqueSetName, } require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) - oldCliqueCount := len(pcs.Spec.Template.Cliques) - oldAgentReplicas := int32(0) - for _, clique := range pcs.Spec.Template.Cliques { + published = getLPXRequest(t, ctx, reconciler.Client, successorDGD.Namespace, published.Name) + successorRendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, successorDGD, successorSource, successorDesired) + for index, clique := range successorRendered.Spec.Template.Cliques { if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { - oldAgentReplicas = clique.Spec.Replicas + require.Equal(t, ptr.To(int32(4)), pcs.Spec.Template.Cliques[index].Spec.MinAvailable) + require.Equal(t, ptr.To(int32(6)), clique.Spec.MinAvailable) } } - require.Positive(t, oldAgentReplicas) - successorRendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, successorDGD, successorSource, successorDesired) - _, _, err := reconciler.reconcileGrovePodCliqueSetForLPX( + t.Log("Return the admission error without replacing or scaling down the existing workload") + admissionErr := apierrors.NewForbidden(grovev1alpha1.SchemeGroupVersion.WithResource("podcliquesets").GroupResource(), pcs.Name, + errors.New("spec.template.cliques.spec.minAvailable: field is immutable")) + reconciler.Client = interceptor.NewClient(reconciler.Client.(client.WithWatch), interceptor.Funcs{ + Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + return admissionErr + } + return delegated.Update(ctx, object, opts...) + }, + }) + synced, changed, err := reconciler.reconcileGrovePodCliqueSetForLPX( ctx, successorDGD, pcs, successorRendered, - successorDesired, ) - require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) - require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is scaled to zero") + require.ErrorIs(t, err, admissionErr) + require.Nil(t, synced) + require.False(t, changed) + t.Log("The complete stored PCS and published request remain unchanged") stored := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, stored)) - require.Zero(t, stored.Spec.Replicas) - require.Equal(t, oldDesired.workloadDigest, stored.Annotations[lpx.WorkloadDigestAnnotation]) - - _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( - ctx, - successorDGD, - stored, - successorRendered, - successorDesired, - ) - require.ErrorIs(t, err, errLPXGrovePodCliqueSetRecreating) - require.EqualError(t, err, "LPX successor is waiting while the stale Grove PodCliqueSet is deleted") - require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, pcsKey, &grovev1alpha1.PodCliqueSet{}))) - - _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( - ctx, - successorDGD, - nil, - successorRendered, - successorDesired, - ) - require.NoError(t, err) - - recreated := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) - require.Equal(t, successorDesired.workloadDigest, recreated.Annotations[lpx.WorkloadDigestAnnotation]) - require.Equal(t, string(successorDGD.UID), recreated.Annotations[lpxDeploymentUIDAnnotation]) - require.Equal(t, strconv.FormatInt(successorDGD.Generation, 10), recreated.Annotations[lpxDeploymentGenerationAnnotation]) - require.Len(t, recreated.Spec.Template.Cliques, oldCliqueCount) - newAgentReplicas := int32(0) - for _, clique := range recreated.Spec.Template.Cliques { - if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { - newAgentReplicas = clique.Spec.Replicas - } - } - require.Greater(t, newAgentReplicas, oldAgentReplicas) + require.Equal(t, pcs, stored) + require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, successorDGD.Namespace, published.Name)) } func TestLPXRetirementOverridesRuntimeReadiness(t *testing.T) { @@ -1118,11 +1034,12 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { } rederived, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) require.Nil(t, rejected) - require.Equal(t, desired.requests[0].attemptDigest, rederived.requests[0].attemptDigest) + wantDigest := desired.requests[0].attemptDigest desired = rederived classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) + require.Equal(t, wantDigest, desired.requests[0].attemptDigest) t.Log("Observe the exact current scheduler receipt after the controller restart") request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) @@ -1404,41 +1321,112 @@ func TestLPXSchedulerDiagnosticMessageRespectsConditionLimit(t *testing.T) { require.True(t, strings.HasSuffix(message, "…")) } +func TestLPXNativeRolloutPreservesOtherEngines(t *testing.T) { + for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { + t.Run(string(pipeline), func(t *testing.T) { + t.Log("Publish two engines and stage a third under the same PCS") + ctx := t.Context() + _, source, registry := newLPXTestDGD(t, pipeline) + source.Spec.Components[0].Replicas = ptr.To[int32](3) + deployment := newLPXTestDeployment(t, source) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) + objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + allRequests := desired.requests + desired.requests = allRequests[:2] + _, err := reconciler.reconcileSelectedLPX(ctx, deployment, desired) + require.NoError(t, err) + desired.requests = allRequests + old := &lpxv1alpha1.LPUPipelineRequest{} + kept := &lpxv1alpha1.LPUPipelineRequest{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[0].requestName}, old)) + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[1].requestName}, kept)) + old.Finalizers = []string{"test.example/scheduler"} + require.NoError(t, reconciler.Update(ctx, old)) + require.Nil(t, old.Status, "native identity changes must not require a scheduler receipt") + + t.Log("Grove replaces engine zero; engine one remains old, while engine two is ready to publish") + pcs := findLPXTestPodCliqueSet(t, objects) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + pcs.Status.CurrentGenerationHash = ptr.To("next-hash") + pcs.Status.ObservedGeneration = ptr.To[int64](0) + require.NoError(t, reconciler.Update(ctx, pcs)) + for _, object := range objects { + clique, ok := object.(*grovev1alpha1.PodClique) + if !ok || clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] == "1" { + continue + } + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(clique), clique)) + require.NoError(t, reconciler.Delete(ctx, clique)) + clique.UID += "-replacement" + clique.ResourceVersion = "" + clique.Status.CurrentPodCliqueSetGenerationHash = ptr.To("next-hash") + require.NoError(t, reconciler.Create(ctx, clique)) + } + _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) + require.NoError(t, err) + live := &lpxv1alpha1.LPUPipelineRequest{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(old), live)) + require.NotNil(t, live.DeletionTimestamp) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(kept), kept)) + require.Nil(t, kept.DeletionTimestamp) + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[2].requestName}, &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("Publish the replacement under its stable name after its own finalizer finishes") + live.Finalizers = nil + require.NoError(t, reconciler.Update(ctx, live)) + _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) + require.NoError(t, err) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(old), live)) + require.NotEqual(t, old.UID, live.UID) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.Equal(t, types.UID("pcs-uid"), pcs.UID) + + t.Log("An owned deleting Agent retires its request even while its PodGang is absent") + live.Finalizers = []string{"test.example/scheduler"} + require.NoError(t, reconciler.Update(ctx, live)) + agent := findLPXTestClique(t, objects, desired.plan.Agents[0].CliqueName) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(agent), agent)) + agent.Finalizers = []string{"test.example/grove"} + require.NoError(t, reconciler.Update(ctx, agent)) + require.NoError(t, reconciler.Delete(ctx, agent)) + require.NoError(t, reconciler.Delete(ctx, &groveschedulerv1alpha1.PodGang{ObjectMeta: metav1.ObjectMeta{ + Name: live.Spec.PodGangRef.Name, Namespace: deployment.Namespace, + }})) + _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) + require.NoError(t, err) + require.False(t, getLPXRequest(t, ctx, reconciler.Client, deployment.Namespace, live.Name).DeletionTimestamp.IsZero()) + require.Equal(t, kept.UID, getLPXRequest(t, ctx, reconciler.Client, deployment.Namespace, kept.Name).UID) + }) + } +} + func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { type digestInput struct { - namespace string - name string - uid types.UID - generation int64 - model string - workloadDigest string + namespace string + name string + uid types.UID + model string } digest := func(input digestInput, replicaIndex int32) string { return digestAttemptKey( input.namespace, input.name, input.uid, - input.generation, input.model, - input.workloadDigest, replicaIndex, ) } base := digestInput{ - namespace: "ns", name: "dgd", uid: "uid-a", generation: 4, - model: "default", workloadDigest: "sha256:workload-a", + namespace: "ns", name: "dgd", uid: "uid-a", model: "default", } baseDigest := digest(base, 0) - require.Equal(t, "sha256:ad538980e64759a4f9761c61ee637e8856513d2cec2cf5ddaf645a8c88b4f5da", baseDigest) + require.NotEmpty(t, baseDigest) for name, mutate := range map[string]func(*digestInput){ "namespace": func(key *digestInput) { key.namespace = "other-ns" }, "name": func(key *digestInput) { key.name = "other-dgd" }, "uid": func(key *digestInput) { key.uid = "uid-b" }, - "generation": func(key *digestInput) { - key.generation++ - }, - "model": func(key *digestInput) { key.model = "model-b" }, - "projection": func(key *digestInput) { key.workloadDigest = "sha256:workload-b" }, + "model": func(key *digestInput) { key.model = "model-b" }, } { t.Run(name, func(t *testing.T) { changed := base @@ -1479,8 +1467,8 @@ func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { dgd.Generation++ classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.Nil(t, classification) + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.requests[0].requestName}, &lpxv1alpha1.LPUPipelineRequest{})) classification, _, err = reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) require.NoError(t, err) @@ -1583,7 +1571,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { request := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) require.False(t, request.DeletionTimestamp.IsZero()) pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child, source)}, pcs)) + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child)}, pcs)) require.Zero(t, pcs.Spec.Replicas) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.ErrorContains(t, err, "waiting for LPX request") @@ -1595,8 +1583,8 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { } } -func TestLPXControllerRetiresPublishedWorkOutsideGrove(t *testing.T) { - t.Log("Retire a published attempt when its source leaves the Grove provider") +func TestLPXControllerPreservesPublishedWorkWhenProviderEditIsInvalid(t *testing.T) { + t.Log("Reject an incompatible provider edit without deleting the existing engine") dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) createLPXTestObjects(t, t.Context(), reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) @@ -1606,10 +1594,10 @@ func TestLPXControllerRetiresPublishedWorkOutsideGrove(t *testing.T) { require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(source), source)) source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent require.NoError(t, reconciler.Update(t.Context(), source)) - result, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) - require.NoError(t, err) - require.Equal(t, lpxRetirementRequeueAfter, result.RequeueAfter) - requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) + require.ErrorContains(t, err, "LPX requires the Grove workload provider") + request := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.Nil(t, request.DeletionTimestamp) } func TestLPUOnlyWaitsForCurrentConductorWitness(t *testing.T) { @@ -1687,14 +1675,23 @@ func TestNodeLocalHybridPublishesAndFencesCurrentCyborgMaterialization(t *testin storedCyborg := &grovev1alpha1.PodClique{} require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(cyborg), storedCyborg)) - var retirementReason string if test.observedGeneration == nil { t.Log("Drift a live Claim reference without an observed generation") require.NotEmpty(t, storedCyborg.Spec.PodSpec.ResourceClaims) storedCyborg.Generation++ storedCyborg.Spec.PodSpec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("stale-gpu") require.NoError(t, reconciler.Update(ctx, storedCyborg)) - retirementReason = "DRA Claim references changed" + + t.Log("An unobserved PCS edit permits temporary old-child references without retiring the engine") + pcs := findLPXTestPodCliqueSet(t, objects) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + pcs.Generation++ + require.NoError(t, reconciler.Update(ctx, pcs)) + _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + require.NoError(t, err) + require.Equal(t, publishedUID, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, request.Name).UID) + pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) + require.NoError(t, reconciler.Update(ctx, pcs)) } else { t.Log("Advance only the global generation without rotating the published request") storedCyborg.Status.ObservedGeneration = ptr.To(test.generation) @@ -1713,15 +1710,12 @@ func TestNodeLocalHybridPublishesAndFencesCurrentCyborgMaterialization(t *testin replacement.DeletionTimestamp = nil replacement.CreationTimestamp = metav1.Time{} require.NoError(t, reconciler.Create(ctx, replacement)) - retirementReason = "immutable intent" } t.Log("Retire the request when its exact published materialization no longer holds") classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) - retiring, ok := classification.(*lpxRetiring) - require.True(t, ok) - require.Contains(t, retiring.retirementReason, retirementReason) + require.IsType(t, &lpxClosed{}, classification) requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) }) } @@ -1855,7 +1849,7 @@ func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { case "LPX gang unavailable": gang.Labels[grovecommon.LabelPartOfKey] = lpxTestOtherName case "LPX gang is stale": - gang.Annotations[lpxDeploymentGenerationAnnotation] = "0" + gang.Annotations[lpxDeploymentUIDAnnotation] = "different-lgd" case "LPX gang has incomplete references": gang.Spec.PodGroups[0].PodReferences = nil case "LPX gang has an extra group": @@ -1877,14 +1871,21 @@ func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { } createLPXTestObjects(t, ctx, reconciler.Client, objects...) - t.Log("Keep publication closed for the actual defect, without creating an LPR") + t.Log("Keep the invalid engine unpublished while allowing an independent sibling") classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) - require.IsType(t, &lpxClosed{}, classification) - require.Equal(t, test.wantMessage, classification.(*lpxClosed).incomplete) requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) require.NoError(t, err) - require.Empty(t, requests) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + if test.name == crossReplicaAgent { + require.IsType(t, &lpxOpen{}, classification) + require.Len(t, requests, 1) + require.Equal(t, desired.requests[1].requestName, requests[0].Name) + } else { + require.IsType(t, &lpxClosed{}, classification) + require.Equal(t, test.wantMessage, classification.(*lpxClosed).incomplete) + require.Empty(t, requests) + } }) } } @@ -1937,9 +1938,7 @@ func TestLPXLiveRequestSizeGrowthRetiresPublication(t *testing.T) { t.Log("Retire the now-oversized publication") classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) - sizeRejected, ok := classification.(*lpxRetiring) - require.True(t, ok) - require.Contains(t, sizeRejected.retirementReason, "scheduler size budget") + require.IsType(t, &lpxClosed{}, classification) requireLPXRequestNotFound(t, ctx, storedClient, dgd.Namespace, desired.requests[0].requestName) } @@ -2029,7 +2028,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t t.Log("Refresh publication state and refuse the foreign requests created after the empty preflight") classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.ErrorContains(t, err, "refusing to adopt foreign LPX request") + require.ErrorContains(t, err, "occupied by a foreign owner") require.Nil(t, classification) require.Len(t, requestListCalls, 2, "final publication must refresh the authoritative request snapshot") for _, request := range []*lpxv1alpha1.LPUPipelineRequest{foreign, second} { diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index e1e81cfc4ad5..3743e7662899 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -19,7 +19,6 @@ package lpx import ( "context" - "errors" "fmt" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -55,7 +54,7 @@ func (r *graphReconciler) reconcileLPXSafetyPreflight( if component != serving { continue } - combinedLength := len(dynamo.PCSNameForLPX(deployment, source)) + dynamo.LPXComponentNameBudget(component.ComponentName) + combinedLength := len(dynamo.PCSNameForLPX(deployment)) + dynamo.LPXComponentNameBudget(component.ComponentName) if combinedLength > consts.MaxCombinedGroveResourceNameLength { err := field.Invalid(field.NewPath("spec", "components").Index(index).Child("name"), component.ComponentName, fmt.Sprintf("combined Grove resource name length %d exceeds the %d-character limit; shorten the deployment or component name", @@ -85,57 +84,12 @@ func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( // Resolve one immutable workload projection for selection and rendering. selectedLPX, rejected, prepareErr := r.prepareLPXMaterializing(ctx, deployment, source) if prepareErr != nil { - // Only inconsistent metadata invalidates publication; a transient read failure leaves it intact. - if errors.Is(prepareErr, dynamolpx.ErrBuildSnapshotInconsistent) { - retiring, fenceErr := r.retireInvalidLPXWorkload(ctx, deployment, fmt.Sprintf("The model build became inconsistent while acquiring the immutable snapshot: %v", prepareErr)) - if fenceErr != nil { - return nil, nil, fmt.Errorf("failed to retire the LPX publication after build snapshot inconsistency: %w", fenceErr) - } - if retiring != nil { - return nil, retiring, nil - } - } return nil, nil, fmt.Errorf("failed to resolve the LPX workload: %w", prepareErr) } if rejected != nil { - requests, listErr := r.listOwnedLPXRequests(ctx, deployment) - if listErr != nil { - return nil, nil, fmt.Errorf("failed to list LPX requests after workload resolution: %w", listErr) - } - if len(requests) > 0 { - retiring, retireErr := r.retireLPXRequest(ctx, deployment, dynamo.PCSNameForLPX(deployment, source), &requests[0], rejected.reason) - if retireErr != nil { - return nil, nil, fmt.Errorf("failed to retire LPX attempt after workload resolution: %w", retireErr) - } - return nil, retiring, nil - } return nil, rejected, nil } - // Fence stale attempts before any Grove resource can be rendered or written. - currentLPXRequests, retiring, fenceErr := r.reconcileLPXAttemptFence(ctx, deployment, selectedLPX) - if fenceErr != nil { - return nil, nil, fmt.Errorf("failed to fence stale LPX attempt: %w", fenceErr) - } - if retiring != nil { - return nil, retiring, nil - } - - // Validate the complete existing request set now; first publication follows Grove sync. - classification, reconcileErr := r.reconcileSelectedLPXFromCurrentRequests( - ctx, - deployment, - selectedLPX, - currentLPXRequests, - false, - ) - if reconcileErr != nil { - return nil, nil, fmt.Errorf("failed to validate selected LPX lifecycle: %w", reconcileErr) - } - switch classification.(type) { - case *lpxOpen, *lpxBound, *lpxSchedulerObserved, *lpxClosed: - return selectedLPX, nil, nil - default: - return nil, classification, nil - } + // Grove accepts the desired spec before individual engine requests are reconciled. + return selectedLPX, nil, nil } diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 3b2d7d6812d4..89023b9c0957 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "maps" - "strconv" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -48,7 +47,7 @@ func renderPodCliqueSet( return nil, nil, err } - // Stamp the selected child revision and identify the engine's serving endpoint. + // Stamp stable ownership and identify the engine's serving endpoint. stampLPXIdentity(deployment, pcs, resources) serving := plan.ConductorTemplate if serving == "" { @@ -84,23 +83,20 @@ func renderPodCliqueSet( return pcs, resources, nil } -// stampLPXIdentity separates attempt ownership from the scheduler's source-DGD -// identity. The source generation is frozen when effective LPX input changes. +// stampLPXIdentity propagates stable ownership, never source revision, into Pod templates. func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { // Preflight validates the exact source controller owner before rendering resources. sourceOwner := metav1.GetControllerOf(deployment) identity := map[string]string{ dynamolpx.DeploymentNameAnnotation: deployment.Name, lpxDeploymentUIDAnnotation: string(deployment.UID), - lpxDeploymentGenerationAnnotation: strconv.FormatInt(deployment.Generation, 10), - dynamo.LPXInputRevisionAnnotation: deployment.Spec.InputRevision, dynamolpx.DGDUIDAnnotation: string(sourceOwner.UID), - dynamolpx.DGDGenerationAnnotation: deployment.Annotations[dynamolpx.DGDGenerationAnnotation], } stamp := func(annotations *map[string]string) { if *annotations == nil { *annotations = make(map[string]string) } + delete(*annotations, dynamolpx.DGDGenerationAnnotation) maps.Copy(*annotations, identity) } stamp(&pcs.Annotations) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 9658a8e25207..f7b5e4968385 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "slices" "sort" @@ -61,7 +62,7 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { source.Annotations = map[string]string{"kai.scheduler/padding": ""} selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} kube := newTestLPXClient(t) @@ -119,7 +120,7 @@ func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { child := newLPXRenderDeployment(t, source) selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} pcs, _, err := renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) @@ -240,7 +241,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { before, childBefore := source.DeepCopy(), child.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{ MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, @@ -310,9 +311,10 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { } for _, annotations := range annotationMaps { require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) - require.Equal(t, "7", annotations[lpx.DGDGenerationAnnotation]) + require.NotContains(t, annotations, lpx.DGDGenerationAnnotation) require.Equal(t, string(child.UID), annotations[dynamo.LPXDeploymentUIDAnnotation]) - require.Equal(t, "1", annotations[dynamo.LPXDeploymentGenerationAnnotation]) + require.NotContains(t, annotations, "lpx.nvidia.com/deployment-generation") + require.NotContains(t, annotations, "lpx.nvidia.com/input-revision") } t.Log("Component and role order preserve the exact child, names and resources") @@ -327,7 +329,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, child.Spec.InputRevision, revision) equivalentSelected, err := lpx.ResolveSelectedWorkload(t.Context(), equivalent, registry) require.NoError(t, err) - equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(child, equivalent)) + equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(child)) require.NoError(t, err) require.Equal(t, selected.Digest(), equivalentSelected.Digest()) require.Equal(t, plan, equivalentPlan) @@ -338,6 +340,31 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, firstResources, equivalentResources) require.Equal(t, equivalentBefore, equivalent) + t.Log("Metadata edits and scale 1, 2, 10, 12, 10 preserve every role template and immutable config version") + replicaCounts := []int32{1, 2, 10, 12, 10} + if selected.Pipeline() == lpx.PipelineSpecDecode { + replicaCounts = []int32{1} // Shared-draft topology supports one target engine. + } + for _, replicas := range replicaCounts { + scaled := source.DeepCopy() + metav1.SetMetaDataAnnotation(&scaled.ObjectMeta, "unrelated", "changed") + lpx.ServingComponent(scaled).Replicas = ptr.To(replicas) + scaledChild := child.DeepCopy() + scaledChild.Generation++ + scaledChild.Spec.InputRevision, err = dynamo.LPXInputRevision(scaled, restartToken) + require.NoError(t, err) + workload, err := lpx.ResolveSelectedWorkload(t.Context(), scaled, registry) + require.NoError(t, err) + scaledPlan, err := workload.PlanNodeLocalMaterialization(plan.PodCliqueSetName) + require.NoError(t, err) + pcs, resources, err := renderPodCliqueSet(t.Context(), scaled, config, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, workload, scaledPlan, scaledChild) + require.NoError(t, err) + require.Equal(t, firstResources, resources) + require.Equal(t, first.Spec.Template.Cliques, pcs.Spec.Template.Cliques) + require.Equal(t, replicas, *pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas) + } + t.Log("Connect every model projection to its rendered Agent while preserving shared runtime metadata and replica units") require.Len(t, first.Spec.Template.PodCliqueScalingGroupConfigs, 1) group := first.Spec.Template.PodCliqueScalingGroupConfigs[0] @@ -449,7 +476,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { before := source.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source), source)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) t.Log("Render one shared conductor without changing either component's Agent template") @@ -466,11 +493,9 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { if clique.Name == plan.ConductorTemplate { conductors++ require.Equal(t, wantImage, container.Image) - require.Equal(t, []string{"/bin/bash"}, container.Command) - require.GreaterOrEqual(t, len(container.Args), 3) - require.Equal(t, "-c", container.Args[0]) - require.Contains(t, container.Args[1], `; exec /bin/nova "$@"`) - require.Equal(t, "--", container.Args[2]) + require.Equal(t, []string{"/bin/sh", "-ec"}, container.Command) + require.Contains(t, container.Args[0], `exec "$@"`) + require.Equal(t, []string{"--", "/configs/datacenter.toml", "/tmp/datacenter.toml", "/bin/nova"}, container.Args[1:5]) if explicit { template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate require.Equal(t, "explicit-conductor", clique.Labels["owner"]) @@ -708,7 +733,7 @@ func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphM func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { t.Helper() - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd), dgd)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) return plan } @@ -811,7 +836,6 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { ) require.NoError(t, err) - configMap := getResource[*corev1.ConfigMap](t, resources, dynamo.PCSNameForLPX(child, &deployment)+"-lpu") var hash string for _, clique := range podCliqueSet.Spec.Template.Cliques { if value := clique.Annotations[commonconsts.AnnotationExtraResourcesHash]; value != "" { @@ -820,6 +844,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { } } require.NotEmpty(t, hash) + configMap := getResource[*corev1.ConfigMap](t, resources, lpx.LPUConfigMapName(dynamo.PCSNameForLPX(child), hash)) configs = append(configs, renderedConfig{modelConfig: configMap.Data["model_config.toml"], hash: hash}) } @@ -980,7 +1005,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { } selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment), &dynamoDeployment)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment))) require.NoError(t, err) t.Log("Render all LPX roles with the separate LPX PCS identity") got, extraResources, err := renderPodCliqueSet( @@ -1052,7 +1077,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { normalized.Data[key] = strings.ReplaceAll(value, registryRoot, "/testdata") } normalizedHash := lpx.LPUConfigMapHash(normalized) - replacements = append(replacements, actualHash, normalizedHash) + replacements = append(replacements, actualHash, normalizedHash, actualHash[:16], normalizedHash[:16]) } goldenPath, err := filepath.Abs("../../dynamo/testdata/" + name + ".yaml") require.NoError(t, err) @@ -1126,7 +1151,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test projections := selected.ModelProjections() require.Len(t, projections, 1) projection := projections[0] - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd), dgd)) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) require.Empty(t, plan.ConductorTemplate) require.Len(t, plan.Agents, 1) @@ -1202,7 +1227,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test gasDirSource := envValueSource(main.Env, "GAS_DIR") require.NotNil(t, gasDirSource) require.NotNil(t, gasDirSource.ConfigMapKeyRef) - require.Equal(t, plan.PodCliqueSetName+"-lpu", gasDirSource.ConfigMapKeyRef.Name) + lpuConfigName := lpx.LPUConfigMapName(plan.PodCliqueSetName, agent.Annotations[commonconsts.AnnotationExtraResourcesHash]) + require.Equal(t, lpuConfigName, gasDirSource.ConfigMapKeyRef.Name) require.Equal(t, "gas_dir", gasDirSource.ConfigMapKeyRef.Key) require.Equal(t, "topologies", envValueSource(main.Env, "TOPOLOGIES").ConfigMapKeyRef.Key) podIPSource := envValueSource(main.Env, "POD_IP") @@ -1240,7 +1266,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test cyborgMain := cyborg.Spec.PodSpec.Containers[0] require.Equal(t, []string{"/bin/sh", "-ec"}, cyborgMain.Command) require.Contains(t, cyborgMain.Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) - require.Equal(t, []string{"--", "/opt/gpu-runtime", "serve"}, cyborgMain.Args[1:]) + require.Equal(t, []string{"--", "/configs/lpu_servers", "/tmp/lpu_servers", "/opt/gpu-runtime", "serve"}, cyborgMain.Args[1:]) require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "config" })) require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) @@ -1250,7 +1276,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test } configVolume := volumes["config"] require.NotNil(t, configVolume.ConfigMap) - require.Equal(t, plan.PodCliqueSetName+"-decode", configVolume.ConfigMap.Name) + decodeConfigName := plan.PodCliqueSetName + "-decode-" + cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash][:16] + require.Equal(t, decodeConfigName, configVolume.ConfigMap.Name) require.Nil(t, configVolume.PersistentVolumeClaim) infinibandVolume := volumes["infiniband"] require.NotNil(t, infinibandVolume.HostPath) @@ -1263,7 +1290,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test cyborgEnv[variable.Name] = variable.Value } require.Equal(t, "19877", cyborgEnv["RDMA_PORT"]) - require.Equal(t, "/configs/lpu_servers", cyborgEnv["SERVER_HOSTS_FILE"]) + require.Equal(t, "/tmp/lpu_servers", cyborgEnv["SERVER_HOSTS_FILE"]) require.Equal(t, "1", cyborgEnv["TOTAL_REPLICAS"]) require.Equal(t, "2", cyborgEnv[lpx.CyborgBatchSizeEnv]) require.NotContains(t, cyborgEnv, "CYBORG_SWA_CACHE_IDS") @@ -1286,22 +1313,22 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NotNil(t, tokenizerEnv) require.NotNil(t, tokenizerEnv.ValueFrom) require.NotNil(t, tokenizerEnv.ValueFrom.ConfigMapKeyRef) - require.Equal(t, plan.PodCliqueSetName+"-decode", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) + require.Equal(t, decodeConfigName, tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) require.Equal(t, "tokenizer_dir", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Key) t.Log("Verify the preserved runtime and LPX projection use the same filtered physical partition view") - lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, plan.PodCliqueSetName+"-lpu") + lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, lpuConfigName) require.Equal(t, "1", lpuConfig.Data["partition_ids"]) require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", lpuConfig.Data["gas_dir"]) t.Log("Verify generated Cyborg and LPU config resource order and contents") - decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, plan.PodCliqueSetName+"-decode") + decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, decodeConfigName) require.Len(t, extraResources, 2) - require.Equal(t, plan.PodCliqueSetName+"-decode", extraResources[0].GetName()) - require.Equal(t, plan.PodCliqueSetName+"-lpu", extraResources[1].GetName()) + require.Equal(t, decodeConfigName, extraResources[0].GetName()) + require.Equal(t, lpuConfigName, extraResources[1].GetName()) require.Equal(t, dgd.Namespace, decodeConfig.Namespace) require.Len(t, decodeConfig.Data, 2) - require.Equal(t, []string{"lpu-0-lpu-wkr-m-0-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) + require.Equal(t, []string{"lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) require.NotEmpty(t, decodeConfig.Data["tokenizer_dir"]) decodeHash := lpx.LPUConfigMapHash(decodeConfig) require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) @@ -1455,23 +1482,15 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin } case lpxv1alpha1.PodRoleConductor: conductors++ - if len(intent.command)+len(intent.args) == 0 { - require.Equal(t, []string{"/bin/bash"}, main.Command) - allocation := slices.Index(main.Args, "--allocation") - require.GreaterOrEqual(t, allocation, 0) - require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) - } else { - wantCommand, argsStart := intent.command, 0 - if len(intent.command) == 0 { - wantCommand, argsStart = []string{"/bin/bash"}, 3 - } - require.Equal(t, wantCommand, main.Command) - require.GreaterOrEqual(t, len(main.Args), argsStart+len(intent.args)) - require.True(t, slices.Equal(intent.args, main.Args[argsStart:argsStart+len(intent.args)])) - allocation := slices.Index(main.Args, "--allocation") - require.GreaterOrEqual(t, allocation, 0) - require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) + command := intent.command + if len(command) == 0 { + command = []string{"/bin/nova"} } + require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) + require.Equal(t, append(slices.Clone(command), intent.args...), main.Args[4:4+len(command)+len(intent.args)]) + allocation := slices.Index(main.Args, "--allocation") + require.GreaterOrEqual(t, allocation, 0) + require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) case lpxv1alpha1.PodRoleCyborgWorker: cyborgs++ wantCommand := intent.command @@ -1480,8 +1499,11 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin wantCommand = intent.cyborgCommand wantArgs = intent.cyborgArgs } - require.Equal(t, wantCommand, main.Command) - require.Equal(t, wantArgs, main.Args) + if len(wantCommand) == 0 { + wantCommand = []string{"/usr/local/bin/cyborg"} + } + require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) + require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[4:]) require.NotNil(t, main.LivenessProbe) require.NotNil(t, main.ReadinessProbe) require.NotNil(t, main.StartupProbe) @@ -1540,6 +1562,7 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, {Name: "TOTAL_REPLICAS", Value: "9"}, {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, + {Name: lpx.CyborgBatchSizeEnv, Value: "2"}, } leader.Spec.Volumes = []corev1.Volume{{ Name: "config", @@ -1590,4 +1613,11 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { require.Equal(t, "/custom-tokenizer", env["TOKENIZER_DIR"]) require.Equal(t, "9", env["TOTAL_REPLICAS"]) require.Equal(t, "/custom-servers", env["SERVER_HOSTS_FILE"]) + + t.Log("Run SWA initialization without reading the replaced generated ConfigMap") + command := exec.CommandContext(t.Context(), "/bin/sh", "-ec", main.Args[0], "--", "/bin/sh", "-c", `printf '%s %s' "$SERVER_HOSTS_FILE" "$CYBORG_SWA_CACHE_IDS"`) + command.Env = []string{"SERVER_HOSTS_FILE=/custom-servers", "CYBORG_FPGA_GPI_REPLICA_INDEX=3", "CYBORG_BATCH_SIZE=2"} + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) + require.Equal(t, "/custom-servers 6,7", string(output)) } diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 0ca1a84beb0b..ef7a9d07095d 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -187,12 +187,12 @@ func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { statusOnly.Status.Phase = groveschedulerv1alpha1.PodGangPhase("scheduled") require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: statusOnly})) - t.Log("Wake on PodGang spec and mirrored attempt-annotation convergence") + t.Log("Wake on PodGang spec and mirrored workload-annotation convergence") specChange := gang.DeepCopy() specChange.Generation++ require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: specChange})) annotationChange := gang.DeepCopy() - annotationChange.Annotations = map[string]string{lpxDeploymentGenerationAnnotation: "2"} + annotationChange.Annotations = map[string]string{dynamolpx.WorkloadDigestAnnotation: "sha256:updated"} require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: annotationChange})) t.Log("Accept the ordinary scheduler witness and reject unrelated PodGangs") @@ -263,7 +263,7 @@ func TestLPXWorkloadEventPredicates(t *testing.T) { {"status noise", func(o client.Object) { o.SetResourceVersion("2") }, false}, {"status observed", test.observe, true}, {"stamp lost", func(o client.Object) { o.SetAnnotations(nil) }, true}, - {"attempt changed", func(o client.Object) { o.GetAnnotations()[lpxDeploymentGenerationAnnotation] = "2" }, true}, + {"workload changed", func(o client.Object) { o.GetAnnotations()[dynamolpx.WorkloadDigestAnnotation] = "sha256:updated" }, true}, {"labels changed", func(o client.Object) { o.SetLabels(map[string]string{"changed": "true"}) }, true}, {"owner changed", func(o client.Object) { o.SetOwnerReferences([]metav1.OwnerReference{{UID: "new-owner"}}) }, true}, {"deleting", func(o client.Object) { now := metav1.Now(); o.SetDeletionTimestamp(&now) }, true}, diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 8d7b36492253..96aa53bbf539 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -6,6 +6,7 @@ package dynamo import ( "context" "crypto/sha256" + "encoding/base32" "encoding/json" "fmt" "maps" @@ -86,12 +87,10 @@ func RenderLPXBasePodCliqueSet( } const ( - LPXDeploymentUIDAnnotation = dynamolpx.DeploymentUIDAnnotation - LPXDeploymentGenerationAnnotation = "lpx.nvidia.com/deployment-generation" - LPXInputRevisionAnnotation = "lpx.nvidia.com/input-revision" - LPXRestartAnnotation = "lpx.nvidia.com/restart-id" - LPXServingLabel = "lpx.nvidia.com/serving" - lpxGPUExecutionRole = "gpu" + LPXDeploymentUIDAnnotation = dynamolpx.DeploymentUIDAnnotation + LPXRestartAnnotation = "lpx.nvidia.com/restart-id" + LPXServingLabel = "lpx.nvidia.com/serving" + lpxGPUExecutionRole = "gpu" ) // LPXRestartToken advances only from the DGD's persisted restart selection. @@ -246,30 +245,18 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin func lpxSchedulingMetadata(metadata map[string]string) map[string]string { selected := make(map[string]string) for key, value := range metadata { - if strings.HasPrefix(key, "kai.scheduler/") || key == "priorityClassName" || key == "project" || key == "user" { + if strings.HasPrefix(key, "kai.scheduler/") || key == "priorityClassName" || key == "project" || key == "user" || key == commonconsts.GroveAnnotationVolcanoQueue { selected[key] = value } } return selected } -// PCSNameForLPX hashes the materialization identity within the Grove name budget, -// independently of ordinary components. The LPX planner checks materialized names. -func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment) string { - // The serving component owns all generated LPX names, including draft cliques. - groupBudget, cliqueBudget := 8, 0 - if component := dynamolpx.ServingComponent(dgd); component != nil { - groupBudget = max(groupBudget, len(component.ComponentName)) - cliqueBudget = max(cliqueBudget, longestLPXCliqueNameLength(component.ComponentName), len(component.ComponentName)+len("-engine-gpu")) - } - budget := commonconsts.MaxCombinedGroveResourceNameLength - groupBudget - cliqueBudget - budget = max(budget, 8) - name := strings.ReplaceAll(deployment.Name, ".", "-") +// PCSNameForLPX derives a stable identity from the owning LGD, not its mutable source. +func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment) string { + // Eight characters fit Grove's name budget; a leading letter also permits Services. digest := sha256.Sum256([]byte(deployment.Namespace + "/" + deployment.Name + "/" + string(deployment.UID))) - if budget <= 13 { - return fmt.Sprintf("%x", digest[:4]) - } - return strings.TrimRight(name[:min(len(name), budget-13)], "-") + fmt.Sprintf("-%x-lpx", digest[:4]) + return "l" + strings.ToLower(base32.StdEncoding.EncodeToString(digest[:5])[:7]) } func longestLPXCliqueNameLength(componentName string) int { @@ -350,8 +337,12 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m role.MinAvailable = nil defaults := ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)) if workload.BuildFamily() == dynamolpx.BuildFamilyXT { + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, materializationName, lpuTemplate.Spec) + if err != nil { + return nil, nil, err + } defaults = &selectedCyborgComponentDefaults{ - ComponentDefaults: defaults, workload: workload, dgdName: materializationName, + ComponentDefaults: defaults, workload: workload, configMapName: input.CyborgConfigMap.Name, replicas: *role.Replicas, lpxPodSpec: lpuTemplate.Spec, } } else { @@ -416,10 +407,10 @@ func (d *imageEntrypointComponentDefaults) GetBaseContainer(context ComponentCon // the user's PodTemplate is merged, preserving the normal merge strategy. type selectedCyborgComponentDefaults struct { ComponentDefaults - workload *dynamolpx.SelectedWorkload - dgdName string - replicas int32 - lpxPodSpec corev1.PodSpec + workload *dynamolpx.SelectedWorkload + configMapName string + replicas int32 + lpxPodSpec corev1.PodSpec } func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { @@ -431,7 +422,7 @@ func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentCont if err := dynamolpx.ApplySelectedCyborgContainerDefaults( &container, d.workload, - d.dgdName, + d.configMapName, d.replicas, d.lpxPodSpec, ); err != nil { @@ -445,7 +436,7 @@ func (d *selectedCyborgComponentDefaults) GetBasePodSpec(context ComponentContex if err != nil { return corev1.PodSpec{}, err } - dynamolpx.ApplySelectedCyborgPodDefaults(&podSpec, d.dgdName) + dynamolpx.ApplySelectedCyborgPodDefaults(&podSpec, d.configMapName) return podSpec, nil } diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index de7000056c3f..d29363abed00 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -107,19 +107,11 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { } func TestLPXPCSNameUsesStableMaterializationIdentity(t *testing.T) { - source := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "chat-uid"}, - Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - {ComponentName: "engine", ComponentType: v1beta1.ComponentTypeLPX, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXConductor}, {Name: v1beta1.ComponentRoleLPXAgent}}}, - }}, - } deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "materialization-uid"}} - name := PCSNameForLPX(deployment, source) + name := PCSNameForLPX(deployment) t.Log("An ordinary DGD named chat-lpx must not collide with the LPX PCS of chat") - ordinary := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat-lpx"}} - require.NotEqual(t, PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), name) - require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), name) + require.NotEqual(t, PCSNameForDGD("chat-lpx", nil), name) + require.NotEqual(t, PCSNameForDGD(deployment.Name, nil), name) for _, test := range []struct { name string @@ -131,36 +123,28 @@ func TestLPXPCSNameUsesStableMaterializationIdentity(t *testing.T) { t.Run(test.name, func(t *testing.T) { changed := deployment.DeepCopy() test.mutate(changed) - require.NotEqual(t, name, PCSNameForLPX(changed, source)) + require.NotEqual(t, name, PCSNameForLPX(changed)) }) } - t.Log("Source replacement, graph edits, and materialization bookkeeping must not rename the workload") - source.UID = "replacement-source" - source.Spec.Components[0].Replicas = ptr.To(int32(3)) + t.Log("Materialization bookkeeping must not rename the workload") + deployment.Annotations = map[string]string{"unrelated": "changed"} deployment.Generation++ deployment.Spec.InputRevision = "sha256:changed" - require.Equal(t, name, PCSNameForLPX(deployment, source)) - t.Log("A draft name cannot change the target-owned resource name budget") - source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: strings.Repeat("d", 30), ComponentType: v1beta1.ComponentTypeLPX, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent}}, - }) - require.Equal(t, name, PCSNameForLPX(deployment, source)) + require.Equal(t, name, PCSNameForLPX(deployment)) deployment.Name = "chat.example" - dotted := PCSNameForLPX(deployment, source) + dotted := PCSNameForLPX(deployment) deployment.Name = "chat-example" - require.NotEqual(t, dotted, PCSNameForLPX(deployment, source), "normalizing dots must not erase materialization identity") + require.NotEqual(t, dotted, PCSNameForLPX(deployment), "normalizing dots must not erase materialization identity") - t.Log("Both readable and compact names fit the complete Grove name budget") + t.Log("The source-independent PCS name fits every supported component's Grove and Service name budget") for componentLength := 1; componentLength <= 13; componentLength++ { - source.Spec.Components[0].ComponentName = strings.Repeat("e", componentLength) - for _, materializationName := range []string{"c", "chat.example", strings.Repeat("long.", 40) + "chat"} { + for _, materializationName := range []string{"c", strings.Repeat("long.", 40) + "chat", "chat.example"} { deployment.Name = materializationName - got := PCSNameForLPX(deployment, source) - require.Empty(t, validation.IsDNS1123Label(got), got) + got := PCSNameForLPX(deployment) + require.Empty(t, validation.IsDNS1035Label(got+"-"+strings.Repeat("e", componentLength)), got) require.LessOrEqual(t, len(got)+max(componentLength, 8)+componentLength+len("-engine-gpu"), commonconsts.MaxCombinedGroveResourceNameLength) - require.Equal(t, got, PCSNameForLPX(deployment, source)) + require.Equal(t, got, PCSNameForLPX(deployment)) } } } @@ -183,7 +167,7 @@ func TestLPXInputRevision(t *testing.T) { want, err := LPXInputRevision(source, "") require.NoError(t, err) require.Regexp(t, `^sha256:[a-f0-9]{64}$`, want) - pcsName := PCSNameForLPX(deployment, source) + pcsName := PCSNameForLPX(deployment) require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), pcsName) t.Log("Prefill-only edits and DGD bookkeeping do not alter the LPX revision or PCS name") @@ -197,7 +181,7 @@ func TestLPXInputRevision(t *testing.T) { got, err := LPXInputRevision(source, "") require.NoError(t, err) require.Equal(t, want, got) - require.Equal(t, pcsName, PCSNameForLPX(deployment, source)) + require.Equal(t, pcsName, PCSNameForLPX(deployment)) t.Log("Each LPX component or shared-input change invalidates the revision; ordinary edits do not") source.Spec.Scheduling = &v1beta1.SchedulingSpec{} diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 41250e69311e..84e57a896ada 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -90,7 +90,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source return result(false, v1beta1.DGDReadyReasonSomeResourcesNotReady, message) } if pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) || - pcs.Annotations[LPXInputRevisionAnnotation] != deployment.Spec.InputRevision || len(pcs.Spec.Template.PodCliqueScalingGroupConfigs) != 1 { + len(pcs.Spec.Template.PodCliqueScalingGroupConfigs) != 1 { return pending("Waiting for the exact LPX PodCliqueSet"), nil } hash := getAcceptedPCSRevisionHash(pcs) @@ -113,7 +113,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source return pending("Waiting for the exact observed LPX scaling group"), nil } status.ScheduledReplicas = ptr.To(group.Status.ScheduledReplicas) - replicas := ptr.Deref(component.Replicas, 1) + replicas := ptr.Deref(component.Replicas, group.Spec.Replicas) if group.Spec.Replicas != replicas || group.Status.CurrentPodCliqueSetGenerationHash == nil || *group.Status.CurrentPodCliqueSetGenerationHash != *hash { return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity"), nil } @@ -129,7 +129,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source for _, template := range pcs.Spec.Template.Cliques { name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: groupName, Replica: int(replica)}, template.Name) memberName := template.Labels[lpx.StageLabel] - readiness, err := observeLPXRole(ctx, reader, deployment, group, name, template.Spec.Replicas) + readiness, err := observeLPXRole(ctx, reader, group, name, template.Spec.Replicas) if err != nil { return GroveReadiness{}, err } @@ -166,7 +166,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source // observeLPXRole fences one clique by ownership and revision, then reports its // counts in complete model instances. The group must already be observed at the // accepted PCS revision; all pointer inputs are non-nil. -func observeLPXRole(ctx context.Context, reader client.Reader, deployment *v1alpha1.LPXGraphDeployment, group *grovev1alpha1.PodCliqueScalingGroup, name string, width int32) (groveComponentReadiness, error) { +func observeLPXRole(ctx context.Context, reader client.Reader, group *grovev1alpha1.PodCliqueScalingGroup, name string, width int32) (groveComponentReadiness, error) { role := groveComponentReadiness{} clique := &grovev1alpha1.PodClique{} if err := reader.Get(ctx, client.ObjectKey{Namespace: group.Namespace, Name: name}, clique); err != nil { @@ -175,8 +175,7 @@ func observeLPXRole(ctx context.Context, reader client.Reader, deployment *v1alp } return role, err } - if !metav1.IsControlledBy(clique, group) || !clique.DeletionTimestamp.IsZero() || - clique.Annotations[LPXInputRevisionAnnotation] != deployment.Spec.InputRevision { + if !metav1.IsControlledBy(clique, group) || !clique.DeletionTimestamp.IsZero() { return role.withResult(false, fmt.Sprintf("Waiting for exact LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil } if clique.Spec.Replicas != width || clique.Status.CurrentPodCliqueSetGenerationHash == nil || diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go index b503a2750e0e..f8fdac7e4e13 100644 --- a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go @@ -14,25 +14,17 @@ import ( ) const ( - lpuDatacenterConfigFilepath = lpuConfigMountPath + "/datacenter.toml" lpuModelConfigFilepath = lpuConfigMountPath + "/model_config.toml" lpuExpandedDatacenterConfigFilepath = runtimeTemporaryStorageMountPath + "/datacenter.toml" - lpuNovaEntrypointScript = `set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" ` + lpuDatacenterConfigFilepath + ` > ` + lpuExpandedDatacenterConfigFilepath + `; exec /bin/nova "$@"` ) func updateLPUConductorContainer( container *corev1.Container, allocation string, ) { - // Wrap image-only workloads so the role always starts Nova after config expansion. + // Expand this engine's topology before the default or explicitly authored command. container.Name = "conductor" - datacenterConfigFilepath := lpuDatacenterConfigFilepath - if len(container.Command) == 0 { - args := container.Args - container.Command = []string{"/bin/bash"} - container.Args = append([]string{"-c", lpuNovaEntrypointScript, "--"}, args...) - datacenterConfigFilepath = lpuExpandedDatacenterConfigFilepath - } + wrapRuntimeStartup(container, "/bin/nova", "datacenter.toml", "") // Nova and OpenMPI require root in the supported direct-DGD runtime. if container.SecurityContext == nil { @@ -57,7 +49,7 @@ func updateLPUConductorContainer( // configmap "--datacenter-config-filepath", - datacenterConfigFilepath, + lpuExpandedDatacenterConfigFilepath, "--model-config-filepath", lpuModelConfigFilepath, diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go new file mode 100644 index 000000000000..f47ab5685fb1 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +// Image and command from inference-engine-instances at 334110c9676479224544a3879ebf8723d04ab3c7, +// kustomize/overlays/hou2-prod1/deployments/gpt-oss-20b-lp20-b300.yaml. +// The command is /bin/bash -lc; this is its complete, unmodified argument. +const pinnedCyborgImage = "us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:0.157.0@sha256:e13b373f9a08ab9dc1d7d4c618180815aa3bbac471f0ca25426da79e8437f908" + +const pinnedCyborgLauncher = `set -euo pipefail + +if [[ -z "${GBUILD_MANIFEST_PATH:-}" ]]; then + echo "GBUILD_MANIFEST_PATH was not provided by the Dynamo operator" >&2 + exit 1 +fi +if [[ -z "${CYBORG_BATCH_SIZE:-}" ]]; then + echo "CYBORG_BATCH_SIZE was not provided by the Dynamo operator" >&2 + exit 1 +fi +MODEL_RUNTIME_DIR="${GBUILD_MANIFEST_PATH%/*}" +. /usr/local/bin/k8s_gpu_detect.sh +set_cuda_visible_devices_from_nvidia_mounts + +trap 'exit 0' TERM INT + +ulimit -l unlimited + +LPU_RDMA_ADDR_FILE="$(mktemp)" +/usr/local/bin/render_grove_lpu_addr_files \ + --server-hosts-file "${SERVER_HOSTS_FILE}" \ + --rdma-addr-file "${LPU_RDMA_ADDR_FILE}" + +cyborg_cmd=( + /usr/local/bin/cyborg + --lpu-rdma-addr-file "${LPU_RDMA_ADDR_FILE}" \ + --gpu 0 \ + --model-family gpt-oss \ + --max-inflight-tasks 1 \ + --num-layers 12 \ + --num-heads-q 64 \ + --head-dim-qk 64 \ + --head-dim-v 64 \ + --num-heads-kv 8 \ + --seqlen-kv 131072 \ + --tokens-per-page 32 \ + --hidden-size 2880 \ + --stop-tokens 200002 \ + --swa-block-size 128 \ + --swa-padding-size 128 \ + --lpu-tail-top-k 40 \ + --standalone-post-decoder \ + --embedding-path "${MODEL_RUNTIME_DIR}/runtime/text_embeddings.npz" \ + --attention-sinks-path "${MODEL_RUNTIME_DIR}/cuda/attention_sinks.npz" \ + --batch-size "${CYBORG_BATCH_SIZE}" \ + --nic auto \ + --dtype bf16 \ + --backend fpga_gpi \ + --dynamo-listen-port 9124 \ + --dynamo-readiness-port 9090 \ + --dynamo-model-name "${MODEL_NAME}" \ + --tokenizer-dir "${TOKENIZER_DIR}" \ + --dynamo-namespace "${DYN_NAMESPACE:-${DYN_PARENT_DGD_K8S_NAMESPACE:-inference-engine}}" \ + --dynamo-component "${DYN_COMPONENT:-decode}" \ + --dynamo-tool-call-parser harmony \ + --dynamo-reasoning-parser gpt_oss \ + --openai-http-port "${OPENAI_HTTP_PORT}" \ + --openai-http-head-host "${POD_IP}" +) + +cyborg_nsys_enable="${CYBORG_NSYS_ENABLE:-false}" +if [[ "${cyborg_nsys_enable,,}" == "true" || "${cyborg_nsys_enable}" == "1" ]]; then + nsys_session_name="${NSYS_SESSION_NAME:-cyborg}" + nsys_profile_dir="${NSYS_PROFILE_DIR:-/profiles}" + nsys_trace="${NSYS_TRACE:-cuda-sw,nvtx}" + nsys_cuda_graph_trace="${NSYS_CUDA_GRAPH_TRACE:-node}" + command -v nsys >/dev/null + mkdir -p "${nsys_profile_dir}" + exec nsys launch \ + --session-new "${nsys_session_name}" \ + --trace="${nsys_trace}" \ + --cuda-graph-trace="${nsys_cuda_graph_trace}" \ + --resolve-symbols=false \ + "${cyborg_cmd[@]}" +fi + +exec "${cyborg_cmd[@]}" +` diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 465a28d45464..63b88b1aaa3d 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -6,6 +6,7 @@ package lpx import ( + "slices" "strconv" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -47,9 +48,16 @@ func configureHybridCyborg( if err := applyCyborgManifestPath(container, projection, modelStorage.mount.MountPath); err != nil { return err } - if err := wrapCyborgDecodeForSwaBatch(container, cyborgBatchSize); err != nil { - return err + configFile := "" + if cyborgConfigMap != nil && slices.ContainsFunc(container.Env, func(variable corev1.EnvVar) bool { + return variable.Name == selectedCyborgServerHostsFileEnv && variable.Value == runtimeTemporaryStorageMountPath+"/lpu_servers" && variable.ValueFrom == nil + }) { + configFile = "lpu_servers" + if err := addRuntimeConfigStorage(&cyborg.Spec.PodSpec, container, configFile); err != nil { + return err + } } + wrapCyborgStartup(container, cyborgBatchSize, configFile) cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index f264ce2d1fa0..6c3a573c2fd0 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -86,7 +86,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, renderTestPodSpec().Containers[0].VolumeMounts[0]) require.Equal(t, []string{"/bin/sh", "-ec"}, cyborg.Spec.PodSpec.Containers[0].Command) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) - require.Equal(t, []string{"--", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args[1:]) + require.Equal(t, []string{"--", "/configs/lpu_servers", "/tmp/lpu_servers", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args[1:]) t.Log("Preserve an image-owned entrypoint") imageEntrypointPCS := renderTestPCS(true) @@ -99,10 +99,11 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) - t.Log("Render and verify rejection of an unwrappable image-owned entrypoint") + t.Log("Render the agreed Cyborg binary when the command is omitted") input.Stages = map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}} _, err = renderSelectedForTest(imageEntrypointPCS, []*ModelProjection{projection}, input) - require.ErrorContains(t, err, "cyborg command is empty; cannot wrap image entrypoint for SWA batch ID generation") + require.NoError(t, err) + require.Equal(t, []string{"/usr/local/bin/cyborg", "serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args[4:]) t.Log("Reject incomplete endpoint and fanout coverage of the same split-I/O runtime") diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index a6c4a23eb025..f07a5538251c 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -8,7 +8,6 @@ package lpx import ( "fmt" "path/filepath" - "slices" "strconv" "strings" @@ -64,7 +63,7 @@ func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { } } -const cyborgSwaBatchIDsEntrypointScript = `if [ -z "${CYBORG_SWA_CACHE_IDS+x}" ]; then +const cyborgSwaBatchIDsInitialization = `if [ -z "${CYBORG_SWA_CACHE_IDS+x}" ]; then : "${CYBORG_FPGA_GPI_REPLICA_INDEX:?CYBORG_FPGA_GPI_REPLICA_INDEX is required}" : "${CYBORG_BATCH_SIZE:?CYBORG_BATCH_SIZE is required}" base=$((CYBORG_FPGA_GPI_REPLICA_INDEX * CYBORG_BATCH_SIZE)) @@ -76,28 +75,19 @@ const cyborgSwaBatchIDsEntrypointScript = `if [ -z "${CYBORG_SWA_CACHE_IDS+x}" ] done export CYBORG_SWA_CACHE_IDS="${ids}" fi -exec "$@" ` -// wrapCyborgDecodeForSwaBatch derives each batched replica's contiguous SWA cache IDs. -func wrapCyborgDecodeForSwaBatch(container *corev1.Container, cyborgBatchSize int) error { - if cyborgBatchSize == 1 { - return nil - } - if len(container.Command) == 0 { - return fmt.Errorf("cyborg command is empty; cannot wrap image entrypoint for SWA batch ID generation") +// wrapCyborgStartup prepares engine-local hosts and SWA IDs in one wrapper. +func wrapCyborgStartup(container *corev1.Container, cyborgBatchSize int, configFile string) { + if cyborgBatchSize == 1 && configFile == "" { + return } - // Preserve the complete image command while installing the batch-ID wrapper. - args := slices.Grow( - []string{cyborgSwaBatchIDsEntrypointScript, "--"}, - len(container.Command)+len(container.Args), - ) - args = append(args, container.Command...) - args = append(args, container.Args...) - container.Command = []string{"/bin/sh", "-ec"} - container.Args = args - return nil + initialization := "" + if cyborgBatchSize > 1 { + initialization = cyborgSwaBatchIDsInitialization + } + wrapRuntimeStartup(container, "/usr/local/bin/cyborg", configFile, initialization) } // cyborgRuntimeIO requires a nonnil normalized build and validates its Cyborg replica domain. diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go index 3ab0074c016c..94c7f6ff4687 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go @@ -24,7 +24,7 @@ func TestWrapCyborgDecodeForSwaBatchSkipsBatchOne(t *testing.T) { original := container.DeepCopy() t.Log("Apply the batch-one runtime contract") - require.NoError(t, wrapCyborgDecodeForSwaBatch(container, 1)) + wrapCyborgStartup(container, 1, "") t.Log("Verify batch one leaves the container unchanged") require.Equal(t, original, container) @@ -66,7 +66,7 @@ func TestWrapCyborgDecodeForSwaBatch(t *testing.T) { command := []string{"/bin/sh", "-c"} args := []string{`printf '%s\n' "${CYBORG_SWA_CACHE_IDS}" "$@"`, "--", "argument with spaces", "literal '$HOME'", ""} container := &corev1.Container{Command: command, Args: args} - require.NoError(t, wrapCyborgDecodeForSwaBatch(container, 2)) + wrapCyborgStartup(container, 2, "") t.Log("Changing caller-owned slices must not change the wrapped invocation") command[0] = "mutated-command" @@ -95,3 +95,12 @@ func TestConfiguredCyborgBatchSizeRejectsInvalidEnvironment(t *testing.T) { require.Error(t, err) } } + +func TestPinnedCyborgLauncher(t *testing.T) { + t.Log("Preserve the linked deployment's exact image and complete bash argument") + container := &corev1.Container{Image: pinnedCyborgImage, Command: []string{"/bin/bash", "-lc"}, Args: []string{pinnedCyborgLauncher}} + wrapCyborgStartup(container, 2, "lpu_servers") + require.Equal(t, pinnedCyborgImage, container.Image) + require.Equal(t, []string{"/bin/bash", "-lc", pinnedCyborgLauncher}, container.Args[4:]) + require.Contains(t, container.Args[0], "export CYBORG_SWA_CACHE_IDS") +} diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 058aca384c5b..1e16c71ac266 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -13,14 +13,12 @@ import ( "strings" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) const ( - selectedCyborgConfigSuffix = "-decode" selectedCyborgInfiniBandName = "infiniband" selectedCyborgInfiniBandPath = "/dev/infiniband" selectedCyborgServerHostsFileEnv = "SERVER_HOSTS_FILE" @@ -28,10 +26,6 @@ const ( selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" ) -func selectedCyborgConfigMapName(dgdName string) string { - return boundedAuxiliaryName(dgdName, selectedCyborgConfigSuffix) -} - // ApplySelectedCyborgContainerDefaults installs the operator-owned V2 Cyborg // bindings before the user's main-container override is merged. container and // workload must be non-nil, and workload must contain a projection. The @@ -39,13 +33,12 @@ func selectedCyborgConfigMapName(dgdName string) string { func ApplySelectedCyborgContainerDefaults( container *corev1.Container, workload *SelectedWorkload, - dgdName string, + configMapName string, totalReplicas int32, lpxPodSpec corev1.PodSpec, ) error { projection := workload.modelProjections[0] - configMapName := selectedCyborgConfigMapName(dgdName) container.VolumeMounts = append( container.VolumeMounts, corev1.VolumeMount{Name: selectedCyborgInfiniBandName, MountPath: selectedCyborgInfiniBandPath}, @@ -83,21 +76,9 @@ func ApplySelectedCyborgContainerDefaults( if err := applyCyborgManifestPath(container, projection, modelStoragePath); err != nil { return err } - hostsFile := lpuConfigMountPath + "/lpu_servers" - if workload.scalingGroupReplicas > 1 { - // Resolve the replica through the downward API before Kubernetes expands - // SERVER_HOSTS_FILE. The image entrypoint remains untouched. - container.Env = append(container.Env, corev1.EnvVar{ - Name: "LPX_ENGINE_REPLICA", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.labels['" + grovecommon.LabelPodCliqueScalingGroupReplicaIndex + "']", - }}, - }) - hostsFile += "-$(LPX_ENGINE_REPLICA)" - } container.Env = append(container.Env, corev1.EnvVar{ Name: selectedCyborgServerHostsFileEnv, - Value: hostsFile, + Value: runtimeTemporaryStorageMountPath + "/lpu_servers", }) return nil } @@ -105,13 +86,13 @@ func ApplySelectedCyborgContainerDefaults( // ApplySelectedCyborgPodDefaults installs the operator-owned V2 Cyborg // volumes before the user's PodSpec override is merged. podSpec must be non-nil // and is mutated in place. -func ApplySelectedCyborgPodDefaults(podSpec *corev1.PodSpec, dgdName string) { +func ApplySelectedCyborgPodDefaults(podSpec *corev1.PodSpec, configMapName string) { podSpec.Volumes = append( podSpec.Volumes, corev1.Volume{ Name: lpuConfigVolumeName, VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: selectedCyborgConfigMapName(dgdName)}, + LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, }}, }, corev1.Volume{ @@ -124,53 +105,51 @@ func ApplySelectedCyborgPodDefaults(podSpec *corev1.PodSpec, dgdName string) { ) } -// renderSelectedCyborgConfigMap requires a nonnil build and at least one Agent template name. -func renderSelectedCyborgConfigMap( +// RenderCyborgConfigMap renders the XT hybrid configuration before Cyborg defaults are merged. +// The workload must contain the selected hybrid model; agentPodSpec supplies its merged storage. +func (w *SelectedWorkload) RenderCyborgConfigMap( namespace string, - dgdName string, - plan *MaterializationPlan, - modelStoragePath string, - build *Build, + root string, + agentPodSpec corev1.PodSpec, ) (*corev1.ConfigMap, error) { + storage, err := lpuModelStorageBinding(agentPodSpec) + if err != nil { + return nil, err + } + plan, err := w.PlanNodeLocalMaterialization(root) + if err != nil { + return nil, err + } + build := &w.modelProjections[0].configuredBuild tokenizerDir := "" if build.Path != "" { tokenizerPath := build.RuntimeTokenizerPath if strings.TrimSpace(tokenizerPath) == "" { return nil, fmt.Errorf("capnp manifest build is missing model.tokenizer.path") } - runtimePath, runtimeErr := buildRuntimePath(build.Path, modelStoragePath) + runtimePath, runtimeErr := buildRuntimePath(build.Path, storage.mount.MountPath) if runtimeErr != nil { return nil, fmt.Errorf("failed to determine tokenizer_dir: %w", runtimeErr) } tokenizerDir = filepath.Join(runtimePath, tokenizerPath) } - data := map[string]string{"tokenizer_dir": tokenizerDir} - for replica := int32(0); replica < plan.Replicas; replica++ { - // Cyborg supplies the PCS prefix, so only materialize the relative Agent name. - serverPrefix := materializedCliqueNameForReplica(plan.LPXScalingGroupTemplate, plan.Agents[0].TemplateName, replica) + "-" - var servers strings.Builder - offset := 0 - for index, partition := range build.Partitions { - if index != 0 { - servers.WriteByte('\n') - } - servers.WriteString(serverPrefix) - servers.WriteString(strconv.Itoa(offset)) - offset += partition.effectiveNodeCount() - } - key := "lpu_servers" - if plan.Replicas > 1 { - key += "-" + strconv.Itoa(int(replica)) - } - data[key] = servers.String() + // Cyborg supplies the PCS prefix; startup supplies this engine's Grove index. + serverPrefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" + servers := make([]string, len(build.Partitions)) + offset := 0 + for index, partition := range build.Partitions { + servers[index] = serverPrefix + strconv.Itoa(offset) + offset += partition.effectiveNodeCount() } - return &corev1.ConfigMap{ + configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: selectedCyborgConfigMapName(dgdName), Namespace: namespace, }, - Data: data, - }, nil + Immutable: ptr.To(true), + Data: map[string]string{"tokenizer_dir": tokenizerDir, "lpu_servers": strings.Join(servers, "\n")}, + } + configMap.Name = boundedAuxiliaryName(root, fmt.Sprintf("-decode-%.16s", LPUConfigMapHash(configMap))) + return configMap, nil } diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 195420b74acf..8a8181eaeb8c 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -39,44 +39,41 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { } t.Log("Resolve the projected runtime path") - plan, err := workload.PlanNodeLocalMaterialization("test-dgd-lpx") + plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) runtimePath, err := buildRuntimePath(projection.configuredBuild.Path, "/models") require.NoError(t, err) + podSpec := renderTestPodSpec() + initial, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + require.NoError(t, err) t.Log("Verify every engine replica addresses only its own Agents") - for _, replicas := range []int32{1, 3, 9} { + for _, replicas := range []int32{1, 2, 10, 12} { t.Run(strconv.Itoa(int(replicas)), func(t *testing.T) { workload.scalingGroupReplicas = replicas - plan, err := workload.PlanNodeLocalMaterialization("test-dgd-lpx") - require.NoError(t, err) - configMap, err := renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + configMap, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) require.NoError(t, err) - require.Len(t, configMap.Data, int(replicas)+1) - for replica := int32(0); replica < replicas; replica++ { - key := "lpu_servers" - if replicas > 1 { - key += "-" + strconv.Itoa(int(replica)) - } - prefix := plan.LPXScalingGroupTemplate + "-" + strconv.Itoa(int(replica)) + "-" - require.Equal(t, prefix+"lpu-wkr-m-0-0\n"+prefix+"lpu-wkr-m-0-2", configMap.Data[key]) - } + require.Equal(t, initial, configMap) + require.True(t, *configMap.Immutable) + prefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + require.Equal(t, prefix+"lpu-wkr-m-0-0\n"+prefix+"lpu-wkr-m-0-2", configMap.Data["lpu_servers"]) require.Equal(t, filepath.Join(runtimePath, "tokenizer"), configMap.Data["tokenizer_dir"]) }) } t.Log("Render a Cap'n Proto build with a nested tokenizer path") projection.configuredBuild.RuntimeTokenizerPath = "metadata/tokenizer" - configMap, err := renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + configMap, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) t.Log("Verify the nested tokenizer path is rooted in the runtime build") require.NoError(t, err) require.Equal(t, filepath.Join(runtimePath, "metadata/tokenizer"), configMap.Data["tokenizer_dir"]) + require.NotEqual(t, initial.Name, configMap.Name) t.Log("Render a Cap'n Proto build without tokenizer metadata") projection.configuredBuild.RuntimeTokenizerPath = "" projection.configuredBuild.Path = "relative/build" - _, err = renderSelectedCyborgConfigMap("test-namespace", "test-dgd", plan, "/models", &projection.configuredBuild) + _, err = workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) t.Log("Verify missing tokenizer metadata is rejected") require.EqualError(t, err, "capnp manifest build is missing model.tokenizer.path") diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index b57e60edf4db..18523ff154d7 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -22,6 +22,7 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" ) const ( @@ -47,9 +48,9 @@ func boundedAuxiliaryName(root, suffix string) string { return prefix + hashSuffix } -// LPUConfigMapName returns the generated runtime ConfigMap name within the Kubernetes name limit. -func LPUConfigMapName(root string) string { - return boundedAuxiliaryName(root, "-lpu") +// LPUConfigMapName names the immutable runtime table using its Pod-template content hash. +func LPUConfigMapName(root, configHash string) string { + return boundedAuxiliaryName(root, fmt.Sprintf("-lpu-%.16s", configHash)) } func renderLPUConfigMap( @@ -96,13 +97,15 @@ func renderLPUConfigMap( } data["model_config.toml"] = modelTOML.String() data["datacenter.toml"] = datacenterTOML.String() - return &corev1.ConfigMap{ + configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: LPUConfigMapName(materializationName), Namespace: namespace, }, - Data: data, - }, nil + Immutable: ptr.To(true), + Data: data, + } + configMap.Name = LPUConfigMapName(materializationName, LPUConfigMapHash(configMap)) + return configMap, nil } // LPUConfigMapHash returns the hash stamped on LPU runtime Pod templates. diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 4cbb969ac149..86a833e5d810 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -19,8 +19,8 @@ func TestLPXAuxiliaryNamesAreBounded(t *testing.T) { root := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) t.Log("Render the LPU and decode ConfigMap names") - lpuName := LPUConfigMapName(root) - decodeName := selectedCyborgConfigMapName(root) + lpuName := LPUConfigMapName(root, "0123456789abcdef") + decodeName := boundedAuxiliaryName(root, "-decode-0123456789abcdef") t.Log("Keep names valid and distinct while leaving short names unchanged") require.Len(t, lpuName, validation.DNS1123SubdomainMaxLength) @@ -28,7 +28,7 @@ func TestLPXAuxiliaryNamesAreBounded(t *testing.T) { require.Empty(t, validation.IsDNS1123Subdomain(lpuName)) require.Empty(t, validation.IsDNS1123Subdomain(decodeName)) require.NotEqual(t, lpuName, decodeName) - require.Equal(t, "short-lpu", LPUConfigMapName("short")) + require.Equal(t, "short-lpu-0123456789abcdef", LPUConfigMapName("short", "0123456789abcdef")) } func TestLPURuntimeBuildRef(t *testing.T) { @@ -95,6 +95,8 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { require.NoError(t, err) require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") require.Equal(t, "/models/model-build", configMap.Data["gas_dir"]) + require.True(t, *configMap.Immutable) + require.Equal(t, LPUConfigMapName("test-dgd", LPUConfigMapHash(configMap)), configMap.Name) t.Log("Render the preserved runtime with an invalid snapshot build reference") projection.configuredBuild.Path = "relative-build" diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 88cc638efeb9..e839c9dbf190 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -22,6 +22,15 @@ const minimumHashedCliqueNameLength = 8 const maximumCyborgPodIndex = math.MaxInt32 +// LPXAttemptRequestStatus stores a name (lpxRequestName uses at most 60 bytes), +// sha256 digest, and server-issued UUID. Reserve a full DNS label plus JSON syntax +// and an array separator; all generated values are ASCII. +const maximumSchedulingAttemptRequestBytes = len(`{"name":"","attemptDigest":"","uid":""},`) + + validation.DNS1123LabelMaxLength + len("sha256:") + sha256.Size*2 + 36 + +// Leave half the object allowance for the enclosing status, spec, and metadata. +const schedulingAttemptRequestBytesBudget = MaxRenderedPodCliqueSetBytes / 2 + // ExpectedAgent binds one projected model to the exact Grove // PodClique identity that must materialize it. The controller uses this plan // to observe objects; it never reconstructs a row from a generated Pod name. @@ -122,24 +131,32 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi } plan = plan.ForReplica(0) - // All admitted replica indices are 0..8, so their names have the same length. - // Validate final hostnames once, retaining conductor/Agent/Cyborg error order. + return plan, plan.ValidateReplicaCount() +} + +// ValidateReplicaCount checks the longest engine names and bounds request/status allocation. +func (p *MaterializationPlan) ValidateReplicaCount() error { + // Charge every model/replica for its complete persisted request identity. + if p.Replicas < 0 || int64(p.Replicas)*int64(len(p.Agents)) > int64(schedulingAttemptRequestBytesBudget/maximumSchedulingAttemptRequestBytes) { + return fmt.Errorf("LPX replica count exceeds the scheduling status size budget") + } + plan := p.ForReplica(max(0, p.Replicas-1)) if plan.ConductorClique != "" { if err := validateMaterializedPodHostnameAtIndex("conductor", plan.ConductorClique, 0); err != nil { - return nil, err + return err } } for _, agent := range plan.Agents { if err := validateMaterializedPodHostnameAtIndex("Agent", agent.CliqueName, agent.Replicas-1); err != nil { - return nil, err + return err } } if plan.CyborgClique != "" { if err := validateMaterializedPodHostnameAtIndex("Cyborg", plan.CyborgClique, maximumCyborgPodIndex); err != nil { - return nil, err + return err } } - return plan, nil + return nil } // ForReplica returns the child clique identities for one scaling-group replica. diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index b6c3c020d93e..572d3f078f66 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -6,15 +6,66 @@ package lpx import ( + "encoding/json" + "fmt" "strings" "testing" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" ) +func TestMaterializationBoundsCompleteSchedulingAttempt(t *testing.T) { + t.Parallel() + + t.Log("Account for every populated API field and the array separator") + row := nvidiacomv1beta1.LPXAttemptRequestStatus{ + Name: strings.Repeat("a", validation.DNS1123LabelMaxLength), + AttemptDigest: "sha256:" + strings.Repeat("f", 64), + UID: types.UID("01234567-89ab-cdef-0123-456789abcdef"), + } + encoded, err := json.Marshal(row) + require.NoError(t, err) + require.Equal(t, maximumSchedulingAttemptRequestBytes, len(encoded)+1) + + const maximumRequests = int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes) + + t.Log("Accept the exact row limit and reject larger single- and multi-model allocations") + for _, test := range []struct { + name string + replicas int32 + models int + wantErr bool + }{ + {name: "zero", models: 1}, + {name: "nine replicas", replicas: 9, models: 1}, + {name: "twelve replicas", replicas: 12, models: 1}, + {name: "limit", replicas: maximumRequests, models: 1}, + {name: "one over", replicas: maximumRequests + 1, models: 1, wantErr: true}, + {name: "two models at limit", replicas: maximumRequests / 2, models: 2}, + {name: "two models over limit", replicas: maximumRequests/2 + 1, models: 2, wantErr: true}, + {name: "16384 replicas", replicas: 16384, models: 1, wantErr: true}, + {name: "negative", replicas: -1, models: 1, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Validate the complete identity storage before materializing replicas") + plan := &MaterializationPlan{LPXScalingGroup: "group", Replicas: test.replicas, Agents: make([]ExpectedAgent, test.models)} + for i := range plan.Agents { + plan.Agents[i] = ExpectedAgent{TemplateName: fmt.Sprintf("agent-%d", i), Replicas: 1} + } + if test.wantErr { + require.ErrorContains(t, plan.ValidateReplicaCount(), "scheduling status size budget") + } else { + require.NoError(t, plan.ValidateReplicaCount()) + } + }) + } +} + func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { t.Parallel() diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 50b736fe244f..b177cc4dd4cc 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -261,6 +261,10 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { agent := *base.DeepCopy() t.Log("Lower the image into the Nova conductor and privileged SSH worker roles") + require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret"), "writable storage") + conductor = *base.DeepCopy() + conductor.Containers[0].VolumeMounts[0].ReadOnly = false + conductor.Volumes[2].VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret")) require.NoError(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret")) @@ -268,8 +272,8 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { require.Len(t, conductor.Containers, 1) conductorContainer := conductor.Containers[0] require.Equal(t, "conductor", conductorContainer.Name) - require.Equal(t, []string{"/bin/bash"}, conductorContainer.Command) - require.Equal(t, []string{"-c", lpuNovaEntrypointScript, "--"}, conductorContainer.Args[:3]) + require.Equal(t, []string{"/bin/sh", "-ec"}, conductorContainer.Command) + require.Equal(t, []string{runtimeConfigExpansion + "exec \"$@\"\n", "--", "/configs/datacenter.toml", "/tmp/datacenter.toml", "/bin/nova"}, conductorContainer.Args[:5]) require.Contains(t, conductorContainer.Args, "--instance-model-name") requireFlagValue(t, conductorContainer.Args, "--allocation", "lpu-wkr-m-0") require.GreaterOrEqual(t, len(conductorContainer.Args), len(nodeLocalHXAgentEnvironmentArgs)) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index ae8e8021afe3..d0399358c3a0 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -34,6 +34,8 @@ const ( type RenderInput struct { // MaterializationName is the owning LPXGraphDeployment's bounded identity root. MaterializationName string + // CyborgConfigMap is rendered before the user's Cyborg defaults are merged. + CyborgConfigMap *corev1.ConfigMap // MinAvailable is the minimum number of complete engine replicas in the gang. MinAvailable *int32 // Stages contains an independently merged LPU template for every projected stage. @@ -99,10 +101,7 @@ func RenderSelectedNodeLocal( extraResources []client.Object ) if v2HybridRuntime { - cyborgConfigMap, err = renderSelectedCyborgConfigMap(namespace, input.MaterializationName, plan, modelStorage.mount.MountPath, &projections[0].configuredBuild) - if err != nil { - return nil, err - } + cyborgConfigMap = input.CyborgConfigMap // Preserve the legacy graph order: Cyborg config first, LPU config last. extraResources = []client.Object{cyborgConfigMap, configMap} } else { diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 7de0e888ea56..f20080945b5b 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -46,6 +46,12 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model if err != nil { return nil, err } + if workload.BuildFamily() == BuildFamilyXT && workload.Pipeline() == PipelineLPX { + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, input.MaterializationName, input.Stages[testRenderComponentName].Spec) + if err != nil { + return nil, err + } + } if _, err := RenderSelectedNodeLocal(pcs, workload, plan, input); err != nil { return nil, err } @@ -190,8 +196,8 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { conductor := namedClique(t, rendered, "lpu-ldr") require.Nil(t, conductor.Spec.PodSpec.Affinity) require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) - require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) - require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) + require.Equal(t, []string{"/bin/sh", "-ec"}, conductor.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"/bin/bash", "-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[4:7]) require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) @@ -323,7 +329,9 @@ func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { RoleName: "lpu-engine-gpu", Replicas: 1, MinAvailable: &one, PodSpec: corev1.PodSpec{ SchedulerName: corev1.DefaultSchedulerName, - Containers: []corev1.Container{{Name: "main", Image: "cyborg"}}, + Containers: []corev1.Container{{Name: "main", Image: "cyborg", Env: []corev1.EnvVar{ + {Name: selectedCyborgServerHostsFileEnv, Value: "/tmp/lpu_servers"}, + }}}, ResourceClaims: []corev1.PodResourceClaim{{ Name: "candidate", ResourceClaimTemplateName: ptr.To("candidate"), diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index 8a13171b605c..f7dcde489f8e 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -122,10 +122,6 @@ func ResolveSelectedWorkload( // A sole component can flatten its replica axis into the outer scaling group. scalingGroupReplicas = ptr.Deref(components[0].Replicas, 1) } - if pipeline != PipelineLPX && scalingGroupReplicas != 1 { - return nil, fmt.Errorf("%w: scaling-group replicas above one currently require a hybrid engine", ErrUnsupportedRuntime) - } - // Canonical roles expand into default or draft0..draft7 followed by target. digest, err := workloadSetDigest(projections) if err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index d1449a467676..db6064975701 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -46,6 +46,16 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Equal(t, "lpx-ldr", plan.ConductorTemplate) require.NotEmpty(t, plan.ConductorClique) + t.Log("Scale Nova engines without changing their model or workload digest") + for _, replicas := range []int32{2, 10, 12, 123} { + dgd.Spec.Components[0].Replicas = ptr.To(replicas) + scaled, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": hxSnapshot}) + require.NoError(t, err) + require.Equal(t, hx.Digest(), scaled.Digest()) + require.Equal(t, replicas, scaled.scalingGroupReplicas) + } + dgd.Spec.Components[0].Replicas = nil + t.Log("Require hybrid conductor resources in either the explicit or inherited template") fixture := newV2CompilerFixture() fixture.compilationMode = manifestcapnp.CompilationMode_lpx diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 8304e927dd79..b8c0be7346b4 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -199,7 +199,9 @@ func configureNodeLocalConductorRuntime( // Materialize the GPC conductor around the image's Nova binary. setNodeLocalPodIPEnv(conductor, isXT) - addRuntimeTemporaryStorage(conductorPodSpec, conductor, !isXT) + if err := addRuntimeConfigStorage(conductorPodSpec, conductor, "datacenter.toml"); err != nil { + return err + } updateLPUConductorContainer(conductor, allocation) if targetFamily == BuildFamilyHX { conductor.Args = append(conductor.Args, nodeLocalHXAgentEnvironmentArgs...) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup.go b/deploy/operator/internal/dynamo/lpx/runtime_startup.go new file mode 100644 index 000000000000..ee6b7d88c86c --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +const runtimeConfigExpansion = `: "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" +sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" +shift 2 +` + +// wrapRuntimeStartup prepares an optional fixed config and executes the original argument vector. +// configFile and initialization are operator-owned, never user-supplied shell fragments. +func wrapRuntimeStartup(container *corev1.Container, defaultCommand, configFile, initialization string) { + // Keep paths and the complete caller argument vector out of the shell source. + args := []string{initialization + "exec \"$@\"\n", "--"} + if configFile != "" { + args[0] = runtimeConfigExpansion + args[0] + args = append(args, lpuConfigMountPath+"/"+configFile, runtimeTemporaryStorageMountPath+"/"+configFile) + } + if len(container.Command) == 0 { + args = append(args, defaultCommand) + } else { + args = append(args, container.Command...) + } + args = append(args, container.Args...) + container.Command, container.Args = []string{"/bin/sh", "-ec"}, args +} + +// addRuntimeConfigStorage keeps expanded configuration writable and local to this Pod. +func addRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, configFile string) error { + // Reject incompatible authored bindings instead of replacing their volumes or mounts. + mount := corev1.VolumeMount{Name: runtimeTemporaryStorageVolumeName, MountPath: runtimeTemporaryStorageMountPath} + for _, existing := range container.VolumeMounts { + if existing.MountPath == runtimeTemporaryStorageMountPath+"/"+configFile { + return fmt.Errorf("LPX runtime configuration cannot overwrite a volume mounted at %q", existing.MountPath) + } + if existing.MountPath == mount.MountPath { + mount = existing + } + } + if mount.ReadOnly { + return fmt.Errorf("LPX runtime configuration requires writable storage at %q", mount.MountPath) + } + for _, volume := range podSpec.Volumes { + if volume.Name == mount.Name { + if volume.EmptyDir == nil { + return fmt.Errorf("LPX runtime configuration requires Pod-local emptyDir storage at %q", mount.MountPath) + } + container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) + return nil + } + } + if mount.Name != runtimeTemporaryStorageVolumeName { + return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) + } + addRuntimeTemporaryStorage(podSpec, container, false) + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go new file mode 100644 index 000000000000..307f5090ce05 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" +) + +func TestRuntimeStartup(t *testing.T) { + for _, file := range []string{"datacenter.toml", "lpu_servers"} { + for _, index := range []string{"0", "42", ""} { + t.Run(file+index, func(t *testing.T) { + t.Log("Render fixed Grove variables before an explicit command with literal arguments") + dir := t.TempDir() + input, output := filepath.Join(dir, "input"), filepath.Join(dir, "output") + require.NoError(t, os.WriteFile(input, []byte("group-${GROVE_PCSG_INDEX}-agent-0\n${GROVE_PCSG_NAME}.${GROVE_HEADLESS_SERVICE}"), 0600)) + container := &corev1.Container{Command: []string{"/bin/sh", "-c"}, Args: []string{`printf '%s\n' "$@"; exit 17`, "--", "argument with spaces", "literal '$HOME'", ""}} + wrapRuntimeStartup(container, "/bin/nova", file, "") + container.Args[2], container.Args[3] = input, output + cmd := exec.CommandContext(t.Context(), container.Command[0], append(container.Command[1:], container.Args...)...) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GROVE_PCSG_NAME=pcs-0-group", "GROVE_PCSG_INDEX=" + index, "GROVE_HEADLESS_SERVICE=service"} + actual, err := cmd.CombinedOutput() + if index == "" { + require.Error(t, err) + require.Contains(t, string(actual), "missing GROVE_PCSG_INDEX") + require.NoFileExists(t, output) + return + } + t.Log("Keep exit status, argument boundaries and this engine's expanded file") + require.EqualError(t, err, "exit status 17") + require.Equal(t, "argument with spaces\nliteral '$HOME'\n\n", string(actual)) + data, err := os.ReadFile(output) + require.NoError(t, err) + require.Equal(t, "group-"+index+"-agent-0\npcs-0-group.service", string(data)) + }) + } + } +} + +func TestRuntimeConfigStorage(t *testing.T) { + for _, source := range []corev1.VolumeSource{ + {EmptyDir: &corev1.EmptyDirVolumeSource{}}, + {PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "shared"}}, + {Secret: &corev1.SecretVolumeSource{SecretName: "readonly"}}, + } { + t.Log("Reuse only a writable Pod-local workspace for expanded runtime configuration") + pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "scratch", VolumeSource: source}}} + container := corev1.Container{VolumeMounts: []corev1.VolumeMount{{Name: "scratch", MountPath: "/tmp"}}} + err := addRuntimeConfigStorage(&pod, &container, "lpu_servers") + if source.EmptyDir == nil { + require.ErrorContains(t, err, "Pod-local emptyDir") + } else { + require.NoError(t, err) + require.Len(t, pod.Volumes, 1) + } + container.VolumeMounts[0].ReadOnly = true + require.ErrorContains(t, addRuntimeConfigStorage(&pod, &container, "lpu_servers"), "writable storage") + } +} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 3aceedc3bbc2..6d295d912150 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -1,50 +1,30 @@ data: - lpu_servers-0: |- - lpu-0-lpu-wkr-m-0-0 - lpu-0-lpu-wkr-m-0-1 - lpu-0-lpu-wkr-m-0-2 - lpu-0-lpu-wkr-m-0-3 - lpu-0-lpu-wkr-m-0-4 - lpu-0-lpu-wkr-m-0-5 - lpu-0-lpu-wkr-m-0-6 - lpu-0-lpu-wkr-m-0-7 - lpu-0-lpu-wkr-m-0-8 - lpu-0-lpu-wkr-m-0-9 - lpu-0-lpu-wkr-m-0-10 - lpu-0-lpu-wkr-m-0-11 - lpu-0-lpu-wkr-m-0-12 - lpu-0-lpu-wkr-m-0-13 - lpu-0-lpu-wkr-m-0-14 - lpu-0-lpu-wkr-m-0-15 - lpu-0-lpu-wkr-m-0-16 - lpu_servers-1: |- - lpu-1-lpu-wkr-m-0-0 - lpu-1-lpu-wkr-m-0-1 - lpu-1-lpu-wkr-m-0-2 - lpu-1-lpu-wkr-m-0-3 - lpu-1-lpu-wkr-m-0-4 - lpu-1-lpu-wkr-m-0-5 - lpu-1-lpu-wkr-m-0-6 - lpu-1-lpu-wkr-m-0-7 - lpu-1-lpu-wkr-m-0-8 - lpu-1-lpu-wkr-m-0-9 - lpu-1-lpu-wkr-m-0-10 - lpu-1-lpu-wkr-m-0-11 - lpu-1-lpu-wkr-m-0-12 - lpu-1-lpu-wkr-m-0-13 - lpu-1-lpu-wkr-m-0-14 - lpu-1-lpu-wkr-m-0-15 - lpu-1-lpu-wkr-m-0-16 + lpu_servers: |- + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-1 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-2 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-3 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-4 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-5 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-6 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-7 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-8 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-9 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-10 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-11 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-12 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-13 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-14 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-15 + lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-16 tokenizer_dir: /testdata/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-lpx-v2-94cdbc7c-lpx-decode + name: lstg3y7d-decode-cd1df68308eaf508 namespace: inference-engine --- data: @@ -196,29 +176,24 @@ data: URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-lpx-v2-94cdbc7c-lpx-lpu + name: lstg3y7d-lpu-70f8ece55681fe1b namespace: inference-engine --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 labels: nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm - name: dgd-lpx-v2-94cdbc7c-lpx + name: lstg3y7d namespace: inference-engine spec: replicas: 1 @@ -226,12 +201,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - nvidia.com/extra-resources-hash: a9c40c6452f37713460176f888205de0185c5478ab3a4aa2a06936caa86ff0c4 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: cd1df68308eaf508bdf4d2ab25abbdc0c374769b498191e1d9dcca12dbfa8d88 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -260,6 +232,15 @@ spec: - c1v2-b300n1-nic1 containers: - args: + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" + - -- + - /configs/lpu_servers + - /tmp/lpu_servers + - /usr/local/bin/cyborg - --dynamo-model-name - meta-llama/Llama-3.2-1B-Instruct - --model-card @@ -269,6 +250,9 @@ spec: - HND - --num-layers - "16" + command: + - /bin/sh + - -ec env: - name: CYBORG_BATCH_SIZE value: "1" @@ -308,10 +292,6 @@ spec: value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin - name: GLUE_RDMA_PATH_MTU value: "1024" - - name: LPX_ENGINE_REPLICA - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podcliquescalinggroup-replica-index'] - name: NATS_SERVER value: nats-address - name: NIXL_TELEMETRY_ENABLE @@ -335,12 +315,12 @@ spec: - name: RDMA_PORT value: "19877" - name: SERVER_HOSTS_FILE - value: /configs/lpu_servers-$(LPX_ENGINE_REPLICA) + value: /tmp/lpu_servers - name: TOKENIZER_DIR valueFrom: configMapKeyRef: key: tokenizer_dir - name: dgd-lpx-v2-94cdbc7c-lpx-decode + name: lstg3y7d-decode-cd1df68308eaf508 - name: TOTAL_REPLICAS value: "1" image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 @@ -389,6 +369,8 @@ spec: name: config - mountPath: /dev/shm name: shared-memory + - mountPath: /tmp + name: tmp restartPolicy: Always schedulerName: default-scheduler securityContext: @@ -410,23 +392,22 @@ spec: type: Directory name: infiniband - configMap: - name: dgd-lpx-v2-94cdbc7c-lpx-decode + name: lstg3y7d-decode-cd1df68308eaf508 name: config - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory + - emptyDir: {} + name: tmp replicas: 1 roleName: lpu-engine-gpu startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - nvidia.com/extra-resources-hash: d782ea49a4ae530a65fb915ae77c6d4b603d6bee3f5b336a52d026439b8f2d41 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 70f8ece55681fe1ba61a15bbecddf456fe9857e535d513e6ca0ac2fd00694a00 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/model: default @@ -516,7 +497,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: dgd-lpx-v2-94cdbc7c-lpx-lpu + name: lstg3y7d-lpu-70f8ece55681fe1b - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -550,7 +531,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: dgd-lpx-v2-94cdbc7c-lpx-lpu + name: lstg3y7d-lpu-70f8ece55681fe1b - name: UCX_TLS value: tcp image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f @@ -629,7 +610,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-94cdbc7c-lpx-lpu + name: lstg3y7d-lpu-70f8ece55681fe1b name: config - emptyDir: {} name: tmp @@ -654,11 +635,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:7daf91bc2a789a171c747e9d93c3fed8cbd748b60888dc98cdcd795ee463a61b - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 75cb7d5bbfc3..2dcf9f9e1033 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -1,15 +1,13 @@ data: - lpu_servers: lpu-0-lpu-wkr-m-0-0 + lpu_servers: lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 tokenizer_dir: /testdata/node-local-v2-connected-lpx/tokenizer +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: node-local-9d06c48a-lpx-decode + name: ltudmjcv-decode-97936407dc305636 namespace: test --- data: @@ -42,29 +40,24 @@ data: partition_node_offsets: "0" partition_paths: part-0 topologies: URSA_V2__Q8__32C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: node-local-9d06c48a-lpx-lpu + name: ltudmjcv-lpu-6bebe5759af346e9 namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid - name: node-local-9d06c48a-lpx + name: ltudmjcv namespace: test spec: replicas: 1 @@ -72,12 +65,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - nvidia.com/extra-resources-hash: d410b2a3efc122a1a232c8d3d209b74eda592811c79dad628c5d46d8fac52264 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 97936407dc3056369df53ef5ff8f62f4b4f451ae1af986e6fc78ad4d75c37c31 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -97,9 +87,19 @@ spec: podSpec: containers: - args: + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" + - -- + - /configs/lpu_servers + - /tmp/lpu_servers + - /opt/gpu-runtime - serve command: - - /opt/gpu-runtime + - /bin/sh + - -ec env: - name: CYBORG_BATCH_SIZE value: "1" @@ -160,12 +160,12 @@ spec: - name: RDMA_PORT value: "19877" - name: SERVER_HOSTS_FILE - value: /configs/lpu_servers + value: /tmp/lpu_servers - name: TOKENIZER_DIR valueFrom: configMapKeyRef: key: tokenizer_dir - name: node-local-9d06c48a-lpx-decode + name: ltudmjcv-decode-97936407dc305636 - name: TOTAL_REPLICAS value: "1" image: example.invalid/gpu-runtime:test @@ -210,6 +210,8 @@ spec: name: shared-memory - mountPath: /nfs name: model-storage + - mountPath: /tmp + name: tmp resourceClaims: - name: gpu resourceClaimTemplateName: decode-gpu @@ -221,7 +223,7 @@ spec: terminationGracePeriodSeconds: 60 volumes: - configMap: - name: node-local-9d06c48a-lpx-decode + name: ltudmjcv-decode-97936407dc305636 name: config - hostPath: path: /dev/infiniband @@ -234,17 +236,16 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: tmp replicas: 1 roleName: lpu-engine-gpu startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - nvidia.com/extra-resources-hash: 24aefcdb8d31a8d6357564441af523737eca563784d5fbe2d770e94f5c2b9c05 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 6bebe5759af346e96ad5bf5bb097effd88b0b56bb7339adbda3868add7aac2e9 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/model: default @@ -285,7 +286,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-9d06c48a-lpx-lpu + name: ltudmjcv-lpu-6bebe5759af346e9 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -319,7 +320,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: node-local-9d06c48a-lpx-lpu + name: ltudmjcv-lpu-6bebe5759af346e9 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -398,7 +399,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-9d06c48a-lpx-lpu + name: ltudmjcv-lpu-6bebe5759af346e9 name: config - emptyDir: {} name: tmp @@ -423,11 +424,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:9d3faab86f714a9c410f67e9b8b85dbfb120b41332690be3f32826897497a89d - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 9dc7309b9f01..b3b0fba58bcf 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -44,29 +44,24 @@ data: topologies: |- URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: node-local-33b11b4d-lpx-lpu + name: lgoyrwtj-lpu-62ffb8651b497e5f namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only - name: node-local-33b11b4d-lpx + name: lgoyrwtj namespace: test spec: replicas: 1 @@ -74,12 +69,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e - nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 62ffb8651b497e5f141bbf328eecc74fc4f1729d8314b66928832afd105631ee scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/role: conductor @@ -99,6 +91,15 @@ spec: podSpec: containers: - args: + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" + - -- + - /configs/datacenter.toml + - /tmp/datacenter.toml + - /opt/dynamo-lpu - serve - --agent-path - /bin/agent @@ -111,7 +112,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /configs/datacenter.toml + - /tmp/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -144,7 +145,8 @@ spec: - --store-backend - kubernetes command: - - /opt/dynamo-lpu + - /bin/sh + - -ec env: - name: DYN_COMPONENT value: lpx @@ -247,7 +249,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-33b11b4d-lpx-lpu + name: lgoyrwtj-lpu-62ffb8651b497e5f name: config - emptyDir: {} name: tmp @@ -262,12 +264,9 @@ spec: startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e - nvidia.com/extra-resources-hash: c54666a901d96415ed0c340891f762a480f3a78ca4a04755d4ae3d02a1125512 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 62ffb8651b497e5f141bbf328eecc74fc4f1729d8314b66928832afd105631ee scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/model: default @@ -393,7 +392,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-33b11b4d-lpx-lpu + name: lgoyrwtj-lpu-62ffb8651b497e5f name: config - emptyDir: {} name: tmp @@ -418,11 +417,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:3a66c573439fcd5c10ee0ac67f9e82ebee1cf652f15afc89847fe634151f562e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index b63aa924614c..fb5128c5b013 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -113,29 +113,24 @@ data: URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: node-local-63df422c-lpx-lpu + name: lmppuelh-lpu-29156c984c942fc4 namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode - name: node-local-63df422c-lpx + name: lmppuelh namespace: test spec: replicas: 1 @@ -143,12 +138,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 scheduling.lpu.nvidia.com/role: conductor @@ -168,13 +160,15 @@ spec: podSpec: containers: - args: - - -c - - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" - "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing - GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" - -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" - /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" - -- + - /configs/datacenter.toml + - /tmp/datacenter.toml + - /bin/nova - --agent-path - /bin/agent - --agent-wd @@ -219,7 +213,8 @@ spec: - --store-backend - kubernetes command: - - /bin/bash + - /bin/sh + - -ec env: - name: DYN_COMPONENT value: lpx @@ -322,7 +317,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-63df422c-lpx-lpu + name: lmppuelh-lpu-29156c984c942fc4 name: config - emptyDir: {} name: tmp @@ -339,12 +334,9 @@ spec: - lpu-wkr-m-1 - lpu-wkr-m-2 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 scheduling.lpu.nvidia.com/model: draft0 @@ -552,7 +544,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-63df422c-lpx-lpu + name: lmppuelh-lpu-29156c984c942fc4 name: config - emptyDir: {} name: tmp @@ -574,12 +566,9 @@ spec: replicas: 4 roleName: lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b scheduling.lpu.nvidia.com/model: draft1 @@ -787,7 +776,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-63df422c-lpx-lpu + name: lmppuelh-lpu-29156c984c942fc4 name: config - emptyDir: {} name: tmp @@ -809,12 +798,9 @@ spec: replicas: 4 roleName: lpu-wkr-m-1 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - nvidia.com/extra-resources-hash: 894a9b0b4bf260c4f77600b2674f4fcbd9cb17eb31aed93e1a29317d03049e2a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e scheduling.lpu.nvidia.com/model: target @@ -1022,7 +1008,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-63df422c-lpx-lpu + name: lmppuelh-lpu-29156c984c942fc4 name: config - emptyDir: {} name: tmp @@ -1047,11 +1033,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:f7d66b8ad0e258f4345d853caa98c07d18a58753a5bb4d9de73436d1c41416c4 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 2c9ac369f0f5..b3b0882190ad 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -37,29 +37,24 @@ data: partition_node_offsets: "0" partition_paths: part-1 topologies: opaque-v3-topology +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: node-local-be18433d-lpx-lpu + name: lxymegpm-lpu-40e022e2d45905fb namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid - name: node-local-be18433d-lpx + name: lxymegpm namespace: test spec: replicas: 1 @@ -67,11 +62,8 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -211,12 +203,9 @@ spec: startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e - nvidia.com/extra-resources-hash: 8803dd921b1a97775b917767487276baa0462067b92142d4456e4409df78ee42 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 40e022e2d45905fb8bce9ead674fe7e847e78a5785b481ef8dbff443218d20e5 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 scheduling.lpu.nvidia.com/model: default @@ -377,7 +366,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-be18433d-lpx-lpu + name: lxymegpm-lpu-40e022e2d45905fb - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -411,7 +400,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: node-local-be18433d-lpx-lpu + name: lxymegpm-lpu-40e022e2d45905fb - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -564,7 +553,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-be18433d-lpx-lpu + name: lxymegpm-lpu-40e022e2d45905fb name: config - emptyDir: {} name: tmp @@ -589,11 +578,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:2c69578d3223855a1909b27ea91ff120daa8d46c56d97eff4d20c89faa91439e - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index d2861339e335..49d14c08cb96 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -36,29 +36,24 @@ data: partition_node_offsets: "0" partition_paths: part-1 topologies: opaque-v3-topology +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: node-local-d43cad47-lpx-lpu + name: l2q6k2r6-lpu-68b42744ed35823d namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only - name: node-local-d43cad47-lpx + name: l2q6k2r6 namespace: test spec: replicas: 1 @@ -66,12 +61,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c - nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 68b42744ed35823df3d0ff18f87c164e682cf5681ce0d76ab40c410b19c997d2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/role: conductor @@ -91,13 +83,15 @@ spec: podSpec: containers: - args: - - -c - - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" - "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing - GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" - -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" - /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" - -- + - /configs/datacenter.toml + - /tmp/datacenter.toml + - /bin/nova - --instance-model-name - test/llama32-1b-lp30 - --agent-path @@ -182,7 +176,8 @@ spec: - --agent-env-vars - REDFISH_BMC_ENABLED=false command: - - /bin/bash + - /bin/sh + - -ec env: - name: DYN_COMPONENT value: lpx @@ -285,7 +280,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-d43cad47-lpx-lpu + name: l2q6k2r6-lpu-68b42744ed35823d name: config - emptyDir: {} name: tmp @@ -300,12 +295,9 @@ spec: startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c - nvidia.com/extra-resources-hash: 53688bf068eaa1ba7ecc8f8c3b54423d0eefc750bfb59627ecfa1551ba06a0b6 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 68b42744ed35823df3d0ff18f87c164e682cf5681ce0d76ab40c410b19c997d2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/model: default @@ -515,7 +507,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-d43cad47-lpx-lpu + name: l2q6k2r6-lpu-68b42744ed35823d name: config - emptyDir: {} name: tmp @@ -537,11 +529,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:6e4c26b8adca9f277bc319af32cdca52142b1729509f5b705e2f542c41bf8b7c - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 47e27fde7bfc..440239bc863b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -92,29 +92,24 @@ data: opaque-v3-topology opaque-v3-topology opaque-v3-topology +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: node-local-1f7af409-lpx-lpu + name: ld55picp-lpu-3c4a4d4af295f6cc namespace: test --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode - name: node-local-1f7af409-lpx + name: ld55picp namespace: test spec: replicas: 1 @@ -122,12 +117,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 scheduling.lpu.nvidia.com/role: conductor @@ -147,13 +139,15 @@ spec: podSpec: containers: - args: - - -c - - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" - "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing - GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" - -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" - /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" - -- + - /configs/datacenter.toml + - /tmp/datacenter.toml + - /bin/nova - --instance-model-name - test/v3-specdecode - --agent-path @@ -238,7 +232,8 @@ spec: - --agent-env-vars - REDFISH_BMC_ENABLED=false command: - - /bin/bash + - /bin/sh + - -ec env: - name: DYN_COMPONENT value: lpx @@ -341,7 +336,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-1f7af409-lpx-lpu + name: ld55picp-lpu-3c4a4d4af295f6cc name: config - emptyDir: {} name: tmp @@ -358,12 +353,9 @@ spec: - lpu-wkr-m-1 - lpu-wkr-m-2 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea scheduling.lpu.nvidia.com/model: draft0 @@ -575,7 +567,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-1f7af409-lpx-lpu + name: ld55picp-lpu-3c4a4d4af295f6cc name: config - emptyDir: {} name: tmp @@ -594,12 +586,9 @@ spec: replicas: 1 roleName: lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed scheduling.lpu.nvidia.com/model: draft1 @@ -811,7 +800,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-1f7af409-lpx-lpu + name: ld55picp-lpu-3c4a4d4af295f6cc name: config - emptyDir: {} name: tmp @@ -830,12 +819,9 @@ spec: replicas: 1 roleName: lpu-wkr-m-1 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - nvidia.com/extra-resources-hash: 8124c7e03767260567b73047785133839c574805c421f4aa3ba230b818929ef9 - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a scheduling.lpu.nvidia.com/model: target @@ -1047,7 +1033,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-1f7af409-lpx-lpu + name: ld55picp-lpu-3c4a4d4af295f6cc name: config - emptyDir: {} name: tmp @@ -1069,11 +1055,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:d92b7b04bae632de73eda61c7a56135debdc73f82eded18bb0f9fe563de31420 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 cliqueNames: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 1dac01d95906..84fd350f47b0 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -120,29 +120,24 @@ data: URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106 +immutable: true metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: dgd-single-3a98fe5c-lpx-lpu + name: lhkmp4xg-lpu-1575485f621480a4 namespace: inference-engine --- metadata: annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 labels: nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 - name: dgd-single-3a98fe5c-lpx + name: lhkmp4xg namespace: inference-engine spec: replicas: 1 @@ -150,12 +145,9 @@ spec: cliqueStartupType: CliqueStartupTypeExplicit cliques: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 - nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 1575485f621480a46d343ddb9e07a6e00472a124e8ba24f535befac176ae9936 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/role: conductor @@ -175,13 +167,15 @@ spec: podSpec: containers: - args: - - -c - - 'set -euo pipefail; : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" - "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing - GROVE_HEADLESS_SERVICE}"; sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" - -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" - /configs/datacenter.toml > /tmp/datacenter.toml; exec /bin/nova "$@"' + - | + : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" + sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" + shift 2 + exec "$@" - -- + - /configs/datacenter.toml + - /tmp/datacenter.toml + - /bin/nova - --agent-env-vars - EXTRA_RUNTIME_KNOB - --agent-path @@ -228,7 +222,8 @@ spec: - --store-backend - kubernetes command: - - /bin/bash + - /bin/sh + - -ec env: - name: DYN_COMPONENT value: lpx @@ -335,7 +330,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-3a98fe5c-lpx-lpu + name: lhkmp4xg-lpu-1575485f621480a4 name: config - emptyDir: {} name: tmp @@ -350,12 +345,9 @@ spec: startsAfter: - lpu-wkr-m-0 - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 - nvidia.com/extra-resources-hash: b1b416f11c06242b55f1a64835b60ada9eb806c05a29a814f953d2f53b99a18a - scheduling.lpu.nvidia.com/dgd-generation: "1" + nvidia.com/extra-resources-hash: 1575485f621480a46d343ddb9e07a6e00472a124e8ba24f535befac176ae9936 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/model: default @@ -565,7 +557,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-3a98fe5c-lpx-lpu + name: lhkmp4xg-lpu-1575485f621480a4 name: config - emptyDir: {} name: tmp @@ -590,11 +582,8 @@ spec: publishNotReadyAddresses: true podCliqueScalingGroups: - annotations: - lpx.nvidia.com/deployment-generation: "1" lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - lpx.nvidia.com/input-revision: sha256:726009979980e74ba17dbdff96de4f43e56a74de84b2f930efe1b2b7adaae550 - scheduling.lpu.nvidia.com/dgd-generation: "1" scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 cliqueNames: diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index c85981d60c28..0374453a6ad0 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -68,11 +68,10 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantSchemaErr: `spec.components[0].roles[1].podTemplate.spec.containers[1]: Duplicate value: map[string]interface {}{"name":"main"}`, }, { - name: "LPX replica bounds are enforced by the schema", + name: "LPX supports more than nine replicas", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Replicas = k8sptr.To(int32(10)) }), - wantCELErr: "spec.components[0]: Invalid value: replicas must be between 1 and 9 when type is lpx", }, { name: "LPX autoscaling is rejected by the schema", @@ -240,7 +239,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { setAlphaLPXSpecDec(dgd, k8sptr.To(int32(0))) }), - wantCELErr: "spec.services[draft]: Invalid value: replicas must be between 1 and 9 when componentType is lpx", + wantCELErr: "spec.services[draft]: Invalid value: replicas must be positive when componentType is lpx", }, // Engine-local template and build validation. { From 86e11693b4647a0708caa6221616e29c849ca6c1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:17:49 +0300 Subject: [PATCH 017/196] fix(operator): evict LPX failure-domain neighbors (#435) Signed-off-by: Andrew Paprotsky (cherry picked from commit 195b1cb4016664716b80177b5b468e41e57e1960) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/eviction.go | 107 +++------- .../internal/controller/lpx/eviction_test.go | 190 +++++++++--------- 2 files changed, 118 insertions(+), 179 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index 1c05ca25cc61..005f1cb029be 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -9,11 +9,11 @@ import ( "context" "fmt" "slices" - "strconv" - "strings" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,7 +42,6 @@ type lpuEvictionReconciler struct { } // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;delete;deletecollection -// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get func (r *lpuEvictionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var pod corev1.Pod if err := r.Get(ctx, req.NamespacedName, &pod); err != nil { @@ -75,6 +74,19 @@ func setupLPUEviction(mgr ctrl.Manager) error { } func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { + owner := metav1.GetControllerOf(trigger) + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || + owner.Kind != groveconstants.KindPodClique || owner.Name == "" || owner.UID == "" { + return nil, nil + } + clique := &grovev1alpha1.PodClique{} + if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: owner.Name}, clique); err != nil { + return nil, err + } + if clique.UID != owner.UID || !dynamolpx.OwnsPodClique(ctx, r.Client, clique) { + return nil, fmt.Errorf("cannot verify ownership of LPU eviction trigger pod %s/%s", trigger.Namespace, trigger.Name) + } + mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[dynamolpx.WorkloadModeAnnotation]) if mode == "" { return nil, nil @@ -86,29 +98,18 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor switch mode { case lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly: case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: - // Resolve the trigger's runtime partition from Dynamo's generated ConfigMap. - partitionByPodIndex, err := r.runtimePartitionByPodIndex(ctx, trigger) + triggerRow, err := lpxv1alpha1.ParsePodLogicalRow(trigger.Annotations) if err != nil { - return nil, err - } - triggerPartition, ok := partitionByPodIndex[trigger.Labels[grovecommon.LabelPodCliquePodIndex]] - if !ok { - return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime partition for Grove pod index %q", trigger.Namespace, trigger.Name, trigger.Labels[grovecommon.LabelPodCliquePodIndex]) + return nil, fmt.Errorf("parse LPU-GPU eviction trigger pod %s/%s: %w", trigger.Namespace, trigger.Name, err) } - // Keep only Agent Pods in the trigger's Dynamo-authored runtime partition. selected := pods[:0] - triggerConfigHash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] for _, pod := range pods { - // Ignore Pods from a concurrent rollout with a different runtime table. - if pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != triggerConfigHash { - continue + row, err := lpxv1alpha1.ParsePodLogicalRow(pod.Annotations) + if err != nil { + return nil, fmt.Errorf("parse LPU-GPU eviction candidate pod %s/%s: %w", pod.Namespace, pod.Name, err) } - partition, ok := partitionByPodIndex[pod.Labels[grovecommon.LabelPodCliquePodIndex]] - if !ok { - return nil, fmt.Errorf("LPU-GPU eviction candidate pod %s/%s has no runtime partition for Grove pod index %q", pod.Namespace, pod.Name, pod.Labels[grovecommon.LabelPodCliquePodIndex]) - } - if partition == triggerPartition { + if row.ModelPartitionID == triggerRow.ModelPartitionID { selected = append(selected, pod) } } @@ -116,62 +117,9 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor default: return nil, nil } - if len(pods) == 0 { - return nil, fmt.Errorf("matched LPU eviction mode returned no pods for %s/%s", trigger.Namespace, trigger.Name) - } return pods, nil } -func (r *lpuEvictionReconciler) runtimePartitionByPodIndex(ctx context.Context, trigger *corev1.Pod) (map[string]int, error) { - // Require the immutable runtime-table identity stamped on the trigger Pod. - triggerConfigHash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] - if triggerConfigHash == "" { - return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap hash", trigger.Namespace, trigger.Name) - } - - // Read the generated runtime partition table independently of authored volume overrides. - pcsName := trigger.Labels[grovecommon.LabelPartOfKey] - if pcsName == "" { - return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no PodCliqueSet identity", trigger.Namespace, trigger.Name) - } - configName := dynamolpx.LPUConfigMapName(pcsName, triggerConfigHash) - var config corev1.ConfigMap - if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: configName}, &config); err != nil { - return nil, fmt.Errorf("get LPU runtime ConfigMap %s/%s: %w", trigger.Namespace, configName, err) - } - - // Fail closed when a rollout has already replaced the trigger's runtime table. - configHash := dynamolpx.LPUConfigMapHash(&config) - if configHash != triggerConfigHash { - return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s does not match trigger pod %s/%s", config.Namespace, config.Name, trigger.Namespace, trigger.Name) - } - - // Require one node count and offset for every runtime partition row. - counts := strings.Fields(config.Data["nodes_per_partition"]) - offsets := strings.Fields(config.Data["partition_node_offsets"]) - if len(counts) == 0 || len(counts) != len(offsets) { - return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s has inconsistent partition node counts and offsets", config.Namespace, config.Name) - } - - // Expand each non-overlapping row into the Grove pod indexes it owns. - partitionByPodIndex := make(map[string]int) - for partition := range counts { - count, countErr := strconv.Atoi(counts[partition]) - offset, offsetErr := strconv.Atoi(offsets[partition]) - if countErr != nil || count < 1 || offsetErr != nil || offset < 0 { - return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s has invalid partition row %d", config.Namespace, config.Name, partition) - } - for rank := 0; rank < count; rank++ { - podIndex := strconv.Itoa(offset + rank) - if _, duplicate := partitionByPodIndex[podIndex]; duplicate { - return nil, fmt.Errorf("LPU runtime ConfigMap %s/%s maps Grove pod index %s more than once", config.Namespace, config.Name, podIndex) - } - partitionByPodIndex[podIndex] = partition - } - } - return partitionByPodIndex, nil -} - func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { labels := trigger.GetLabels() dgdName := labels[commonconsts.KubeLabelDynamoGraphDeploymentName] @@ -198,8 +146,10 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev return slices.DeleteFunc(pods.Items, func(pod corev1.Pod) bool { // Native updates can replace the clique or individual Pods within it. candidateOwner := metav1.GetControllerOf(&pod) - return !r.isLPUAgentPod(&pod) || - owner == nil || owner.UID == "" || candidateOwner == nil || owner.UID != candidateOwner.UID || + return pod.DeletionTimestamp != nil || !r.isLPUAgentPod(&pod) || + owner == nil || candidateOwner == nil || + owner.APIVersion != candidateOwner.APIVersion || owner.Kind != candidateOwner.Kind || + owner.Name != candidateOwner.Name || owner.UID != candidateOwner.UID || pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model }), nil @@ -209,11 +159,8 @@ func (r *lpuEvictionReconciler) deletePods(ctx context.Context, trigger *corev1. deleted := 0 for i := range pods { pod := &pods[i] - if pod.DeletionTimestamp != nil { - continue - } - uid := pod.UID - if err := r.Delete(ctx, pod, client.GracePeriodSeconds(0), client.Preconditions{UID: &uid}); err != nil { + uid, resourceVersion := pod.UID, pod.ResourceVersion + if err := r.Delete(ctx, pod, client.GracePeriodSeconds(0), client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}); err != nil { if errors.IsNotFound(err) { continue } diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index e98ef44df7bc..794077be077f 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -8,18 +8,18 @@ package lpx import ( "context" "sort" - "strings" "testing" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -35,6 +35,7 @@ import ( const ( lpuEvictionTestNamespace = "test-ns" lpuEvictionTestDGD = "test-dgd" + lpuEvictionTestLGD = "test-lgd" lpuEvictionTestPCS = "materialization-12345678" lpuEvictionTestComponent = "lpx-worker" lpuEvictionTestPCSG = "lpx-worker" @@ -44,11 +45,35 @@ const ( func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { scheme := runtime.NewScheme() _ = corev1.AddToScheme(scheme) + _ = nvidiacomv1alpha1.AddToScheme(scheme) + _ = grovev1alpha1.AddToScheme(scheme) + + deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: lpuEvictionTestLGD, Namespace: lpuEvictionTestNamespace, UID: "test-lgd-uid", + }} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: lpuEvictionTestPCS, Namespace: lpuEvictionTestNamespace, UID: "test-pcs-uid", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + }} + group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ + Name: lpuEvictionTestPCSG, Namespace: lpuEvictionTestNamespace, UID: "test-pcsg-uid", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }} + clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ + Name: "agents", Namespace: lpuEvictionTestNamespace, UID: "agents-uid", + Annotations: map[string]string{ + dynamolpx.DeploymentNameAnnotation: lpuEvictionTestLGD, + dynamolpx.DeploymentUIDAnnotation: "test-lgd-uid", + dynamolpx.WorkloadDigestAnnotation: "test-digest", + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + }, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }} cb := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&corev1.Pod{}). - WithObjects(newLPUEvictionConfigMap()) + WithObjects(deployment, pcs, group, clique) for _, o := range objs { cb = cb.WithObjects(o) } @@ -57,19 +82,6 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl return &lpuEvictionReconciler{Client: c, Recorder: events.NewFakeRecorder(16)}, c } -func newLPUEvictionConfigMap() *corev1.ConfigMap { - config := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: lpuEvictionTestNamespace}, - Immutable: ptr.To(true), - Data: map[string]string{ - "nodes_per_partition": "2\n1", - "partition_node_offsets": "0\n2", - }, - } - config.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, dynamolpx.LPUConfigMapHash(config)) - return config -} - func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting bool, conditionReason string) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -86,9 +98,10 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, }, Annotations: map[string]string{ - commonconsts.AnnotationExtraResourcesHash: dynamolpx.LPUConfigMapHash(newLPUEvictionConfigMap()), - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + lpxv1alpha1.PodPartitionIDAnnotation: "0", + lpxv1alpha1.PodRankInPartitionAnnotation: podIndex, + dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), }, }, Spec: corev1.PodSpec{SchedulerName: dynamolpx.SchedulerName}, @@ -97,6 +110,9 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting if podIndex != "" { pod.Labels[grovecommon.LabelPodCliquePodIndex] = podIndex } + if podIndex == "2" { + pod.Annotations[lpxv1alpha1.PodPartitionIDAnnotation] = "1" + } pod.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel if conditionReason != "" { pod.Status.Conditions = []corev1.PodCondition{{ @@ -156,35 +172,10 @@ func lpuEvictionPodNames(pods []corev1.Pod) []string { } func TestLPUEviction_PodsForTrigger(t *testing.T) { - t.Run("lpu-gpu rejects a runtime table newer than the trigger", func(t *testing.T) { + t.Run("lpu-gpu pod without partition returns error", func(t *testing.T) { trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = "missing-config-version" - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, _ := newLPUEvictionReconciler(trigger, sibling) - - pods, err := r.podsForTrigger(context.Background(), trigger) - - require.ErrorContains(t, err, "not found") - assert.Nil(t, pods) - }) - - t.Run("lpu-gpu excludes Pods from a concurrent rollout", func(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - samePartition := newLPUEvictionPod("agent-1", "0", "1", false, "") - samePartition.Annotations[commonconsts.AnnotationExtraResourcesHash] = "other" - r, _ := newLPUEvictionReconciler(trigger, samePartition) - - pods, err := r.podsForTrigger(context.Background(), trigger) - - require.NoError(t, err) - assert.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) - }) - - t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + delete(trigger.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") r, _ := newLPUEvictionReconciler(trigger, sibling) @@ -193,63 +184,15 @@ func TestLPUEviction_PodsForTrigger(t *testing.T) { require.Error(t, err) assert.Nil(t, pods) }) - - t.Run("lpu-gpu pod without model filters only by partition", func(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) - sibling := newLPUEvictionPod("agent-1", "0", "0", false, "") - r, _ := newLPUEvictionReconciler(trigger, sibling) - - pods, err := r.podsForTrigger(context.Background(), trigger) - - require.NoError(t, err) - require.NotNil(t, pods) - assert.Equal(t, []string{"agent-0", "agent-1"}, lpuEvictionPodNames(pods)) - }) -} - -func TestLPUEviction_RuntimePartitionTableUsesBoundedGeneratedName(t *testing.T) { - t.Log("Create the generated runtime table for a bounded materialization name unrelated to the source DGD") - pcsName := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) - config := newLPUEvictionConfigMap() - config.Name = dynamolpx.LPUConfigMapName(pcsName, dynamolpx.LPUConfigMapHash(config)) - - t.Log("Override the Agent config volume without replacing its generated partition table") - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Labels[grovecommon.LabelPartOfKey] = pcsName - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] = dynamolpx.LPUConfigMapHash(config) - trigger.Spec.Volumes = []corev1.Volume{{ - Name: "config", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }} - r, _ := newLPUEvictionReconciler(trigger, config) - - t.Log("Read the bounded generated ConfigMap despite the non-ConfigMap volume override") - partitions, err := r.runtimePartitionByPodIndex(t.Context(), trigger) - require.NoError(t, err) - require.Equal(t, map[string]int{"0": 0, "1": 0, "2": 1}, partitions) - - t.Log("Reject a missing materialization identity instead of falling back to the source DGD name") - delete(trigger.Labels, grovecommon.LabelPartOfKey) - _, err = r.runtimePartitionByPodIndex(t.Context(), trigger) - require.ErrorContains(t, err, "has no PodCliqueSet identity") } func TestLPUEvictionPreservesNativeReplacement(t *testing.T) { - t.Log("Keep the original snapshot while a new configuration and native clique coexist") + t.Log("Keep the original snapshot while a new native clique coexists") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) replacement := newLPUEvictionPod("agent-1", "0", "1", false, "") replacement.OwnerReferences[0].UID = "replacement-clique" - newConfig := newLPUEvictionConfigMap() - newConfig.Data["nodes_per_partition"] = "1\n2" - newConfig.Data["partition_node_offsets"] = "0\n1" - newConfig.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, dynamolpx.LPUConfigMapHash(newConfig)) - r, _ := newLPUEvictionReconciler(trigger, replacement, newConfig) - partitions, err := r.runtimePartitionByPodIndex(t.Context(), trigger) - require.NoError(t, err) - require.Equal(t, map[string]int{"0": 0, "1": 0, "2": 1}, partitions) + r, _ := newLPUEvictionReconciler(trigger, replacement) pods, err := r.podsForTrigger(t.Context(), trigger) require.NoError(t, err) require.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) @@ -261,7 +204,7 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid, } { t.Run(string(mode), func(t *testing.T) { - t.Log("OnDelete can retain old Pods and replace others under the same clique and ConfigMap") + t.Log("OnDelete can retain old Pods and replace others under the same clique") trigger := newLPUEvictionPod("agent-old-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) sibling := newLPUEvictionPod("agent-old-1", "0", "1", false, "") @@ -273,7 +216,6 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { sibling.Spec.Containers = trigger.Spec.Containers replacement.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:new"}} require.Equal(t, trigger.OwnerReferences, replacement.OwnerReferences) - require.Equal(t, trigger.Annotations[commonconsts.AnnotationExtraResourcesHash], replacement.Annotations[commonconsts.AnnotationExtraResourcesHash]) r, c := newLPUEvictionReconciler(trigger, sibling, replacement) t.Log("Evict the old revision's sibling without deleting the replacement") @@ -337,7 +279,50 @@ func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { } } +func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { + t.Log("Observe an ownership snapshot with a stale LGD UID") + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, c := newLPUEvictionReconciler(trigger, sibling) + + clique := &grovev1alpha1.PodClique{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: "agents"}, clique)) + clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "stale-lgd-uid" + require.NoError(t, c.Update(t.Context(), clique)) + + t.Log("Return a retryable error without deleting the sibling") + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) + require.ErrorContains(t, err, "cannot verify ownership") + assert.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) + + t.Log("Evict the sibling when the owner snapshot catches up") + clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "test-lgd-uid" + require.NoError(t, c.Update(t.Context(), clique)) + reconcileLPUEviction(t, r, trigger.Name) + assert.Empty(t, remainingLPUEvictionPods(t, c)) +} + +func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, c := newLPUEvictionReconciler(trigger, sibling) + r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + current := &corev1.Pod{} + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(object), current)) + current.Labels["changed"] = "true" + require.NoError(t, delegated.Update(ctx, current)) + return delegated.Delete(ctx, object, opts...) + }, + }) + + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) + require.Error(t, err) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(sibling), &corev1.Pod{})) +} + func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { + t.Log("Observe live neighbors alongside a terminating Agent without scheduler partition metadata") trigger := newLPUEvictionPod("stage-0-partition-3", "0", "0", true, podDisruptionReasonTaintManagerDeletion) samePartition := newLPUEvictionPod("stage-1-partition-3", "0", "1", false, "") otherPartition := newLPUEvictionPod("stage-0-partition-4", "0", "2", false, "") @@ -346,7 +331,9 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { noPartition := newLPUOnlyConductor() otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - require.NotContains(t, trigger.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") + delete(terminating.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + delete(terminating.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) r, c := newLPUEvictionReconciler( trigger, @@ -355,11 +342,16 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { otherModel, noPartition, otherReplica, + terminating, ) + t.Log("Evict the live neighbor in the trigger's partition") result := reconcileLPUEviction(t, r, "stage-0-partition-3") assert.Equal(t, ctrl.Result{}, result) assert.Equal(t, []string{"conductor-0", "replica-1-partition-3", "stage-0-draft-partition-3", "stage-0-partition-4"}, remainingLPUEvictionPods(t, c)) + + t.Log("Reconcile successfully after the partition's live neighbors are gone") + reconcileLPUEviction(t, r, trigger.Name) } func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t *testing.T) { From 9153741ef7a36bea74794aa547ee663a53e9397b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:17:49 +0300 Subject: [PATCH 018/196] fix(docs): support cross-version API type references (#437) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7b484829a90e322acceea3788d7ac9a148d4abd0) Signed-off-by: Andrew Paprotsky --- deploy/operator/docs/crd-ref-docs-config.yaml | 21 ++++++ deploy/operator/docs/fix-api-anchors.py | 18 ++--- .../additional-resources/api-reference-k8s.md | 66 +++++++++---------- .../kubernetes-api/full-api-reference.mdx | 14 ++-- .../tests/test_api_reference_regressions.py | 28 ++++++-- .../scripts/tests/test_gen_kubernetes_api.py | 50 ++++++++++++-- 6 files changed, 138 insertions(+), 59 deletions(-) diff --git a/deploy/operator/docs/crd-ref-docs-config.yaml b/deploy/operator/docs/crd-ref-docs-config.yaml index 517302928762..a39bfd6eec9b 100644 --- a/deploy/operator/docs/crd-ref-docs-config.yaml +++ b/deploy/operator/docs/crd-ref-docs-config.yaml @@ -42,6 +42,27 @@ processor: - "ExcludedNamespacesChecker" render: + # Preserve package identity before the Markdown renderer drops it. + knownTypes: + - name: DynamoGraphDeploymentRequest + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamographdeploymentrequest" + - name: DynamoGraphDeploymentRequestSpec + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamographdeploymentrequestspec" + - name: DynamoGraphDeploymentRequestStatus + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamographdeploymentrequeststatus" + - name: DynamoGraphDeploymentStatus + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamographdeploymentstatus" + - name: DynamoGraphDeploymentSpec + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamographdeploymentspec" + - name: DynamoComponentDeploymentSharedSpec + package: github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1 + link: "#v1beta1-dynamocomponentdeploymentsharedspec" + # Output format - use markdown instead of default asciidoc format: markdown diff --git a/deploy/operator/docs/fix-api-anchors.py b/deploy/operator/docs/fix-api-anchors.py index 87b2bb4e6e5c..0d0bd0bb6ad6 100644 --- a/deploy/operator/docs/fix-api-anchors.py +++ b/deploy/operator/docs/fix-api-anchors.py @@ -19,7 +19,7 @@ API versions get identical anchors (e.g. #dynamographdeploymentrequest). In standard Markdown renderers the first occurrence wins, meaning v1beta1 links resolve to the v1alpha1 section. This script prepends "v1beta1 " to the affected headings in the -v1beta1 section and updates all intra-section links to match the new anchors. +v1beta1 section. crd-ref-docs-config.yaml supplies package-aware links to them. crd-ref-docs also renders links for some external dangerous types that are referenced from the CRD but not emitted as sections. Strip those links so the published @@ -31,6 +31,7 @@ """ import re import sys +from pathlib import Path TYPE_HEADING_RE = re.compile(r"^####\s+(?:v1beta1\s+)?(?P\S+)\s*$") DCD_SPEC = "DynamoComponentDeploymentSpec" @@ -82,8 +83,8 @@ def project_standalone_dcd_schema(markdown: str) -> str: print(f"Usage: {sys.argv[0]} ", file=sys.stderr) sys.exit(1) -path = sys.argv[1] -content = open(path).read() +path = Path(sys.argv[1]) +content = path.read_text() marker = "## nvidia.com/v1beta1" idx = content.find(marker) @@ -94,16 +95,17 @@ def project_standalone_dcd_schema(markdown: str) -> str: alpha_part = content[:idx] beta_part = content[idx:] -# Types whose names collide between v1alpha1 and v1beta1. -# Add to this list if future versions introduce additional same-named types. +# Match the package-qualified links in crd-ref-docs-config.yaml. duplicate_types = [ "DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRequestSpec", "DynamoGraphDeploymentRequestStatus", + "DynamoGraphDeploymentStatus", + "DynamoGraphDeploymentSpec", + "DynamoComponentDeploymentSharedSpec", ] for t in duplicate_types: - anchor = t.lower() # Rename section headings: #### TypeName → #### v1beta1 TypeName beta_part = re.sub( r"(####\s+)" + re.escape(t) + r"(\s*$)", @@ -111,8 +113,6 @@ def project_standalone_dcd_schema(markdown: str) -> str: beta_part, flags=re.MULTILINE, ) - # Update markdown links: (#anchor) → (#v1beta1-anchor) - beta_part = beta_part.replace(f"(#{anchor})", f"(#v1beta1-{anchor})") content = alpha_part + beta_part content = project_standalone_dcd_schema(content) @@ -125,5 +125,5 @@ def project_standalone_dcd_schema(markdown: str) -> str: anchor = t.lower() content = content.replace(f"[{t}](#{anchor})", t) -open(path, "w").write(content) +path.write_text(content) print(f"✅ Post-processed API reference in {path}") diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 71cbb0fdc622..7c6b3ae165a3 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -1019,7 +1019,7 @@ _Appears in:_ | `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | | `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | | `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | MaxItems: 9
MinItems: 1
| +| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | | @@ -1664,7 +1664,7 @@ users do not need to hand-wire them into the pod template. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | @@ -1722,7 +1722,7 @@ ComponentCheckpointStatus contains checkpoint information for a single component _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -1761,7 +1761,7 @@ ComponentReplicaStatus contains replica information for a single component. _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -1788,7 +1788,7 @@ The enclosing component type defines the allowed role names and cardinality. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -1812,7 +1812,7 @@ _Validation:_ - Enum: [frontend worker prefill decode planner epp lpx] _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | @@ -1858,7 +1858,7 @@ _Validation:_ - Enum: [initializing pending successful failed] _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | | --- | --- | @@ -1932,7 +1932,7 @@ operator's conversion webhook; see api/v1alpha1/*_conversion.go. | `spec` _[DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec)_ | spec defines the desired state for this Dynamo component deployment. | | | -#### DynamoComponentDeploymentSharedSpec +#### v1beta1 DynamoComponentDeploymentSharedSpec @@ -1952,7 +1952,7 @@ pod-level configuration directly in `podTemplate` without any _Appears in:_ -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2031,8 +2031,8 @@ operator's conversion webhook; see api/v1alpha1/*_conversion.go. | `apiVersion` _string_ | `nvidia.com/v1beta1` | | | | `kind` _string_ | `DynamoGraphDeployment` | | | | `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[DynamoGraphDeploymentSpec](#dynamographdeploymentspec)_ | spec defines the desired state for this graph deployment. | | | -| `status` _[DynamoGraphDeploymentStatus](#dynamographdeploymentstatus)_ | status reflects the current observed state of this graph deployment. | | | +| `spec` _[DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec)_ | spec defines the desired state for this graph deployment. | | | +| `status` _[DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus)_ | status reflects the current observed state of this graph deployment. | | | #### DynamoGraphDeploymentComponentRef @@ -2067,7 +2067,7 @@ v1beta1 releases. Component-level experimental features live under _Appears in:_ -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2085,7 +2085,7 @@ Both served DGD versions use this type, so conversion preserves its live fields _Appears in:_ - [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2240,7 +2240,7 @@ _Appears in:_ | `lastScaleTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | lastScaleTime is the last time the adapter scaled the target component. | | Optional: \{\}
| -#### DynamoGraphDeploymentSpec +#### v1beta1 DynamoGraphDeploymentSpec @@ -2258,7 +2258,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | annotations to propagate to all child resources (PCS, DCD, Deployments,
and pod templates). Component-level (`podTemplate`) values take precedence
on conflict. | | Optional: \{\}
| | `labels` _object (keys:string, values:string)_ | labels to propagate to all child resources. Same precedence rules as `annotations`. | | Optional: \{\}
| | `priorityClassName` _string_ | priorityClassName is the name of the PriorityClass to use for Grove PodCliqueSets.
Requires the Grove pathway. | | Optional: \{\}
| -| `components` _[DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) array_ | components are the components deployed as part of this graph. Each entry
carries its own stable logical `name`, and names must be unique within
the list. Component types are generally repeatable, except `type: epp`
which may appear at most once. | | MaxItems: 25
Optional: \{\}
| +| `components` _[DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) array_ | components are the components deployed as part of this graph. Each entry
carries its own stable logical `name`, and names must be unique within
the list. Component types are generally repeatable, except `type: epp`
which may appear at most once. | | MaxItems: 25
Optional: \{\}
| | `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | env is prepended to every component's environment. Component-specific
env entries with the same name take precedence and may reference values
from this list. | | Optional: \{\}
| | `backendFramework` _string_ | backendFramework specifies the backend framework (e.g. "sglang", "vllm", "trtllm"). | | Enum: [sglang vllm trtllm]
| | `restart` _[Restart](#restart)_ | restart specifies the restart policy for the graph deployment. | | Optional: \{\}
| @@ -2266,7 +2266,7 @@ _Appears in:_ | `experimental` _[DynamoGraphDeploymentExperimentalSpec](#dynamographdeploymentexperimentalspec)_ | experimental groups graph-level preview features whose API shape and
behavior may change in breaking ways between v1beta1 releases. | | Optional: \{\}
| -#### DynamoGraphDeploymentStatus +#### v1beta1 DynamoGraphDeploymentStatus @@ -2305,7 +2305,7 @@ explicitly by clearing `eppConfig` (and updating the image). _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | @@ -2329,7 +2329,7 @@ spec) once their API is considered stable. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | @@ -2350,7 +2350,7 @@ _Validation:_ - Enum: [override strategic] _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | @@ -2625,7 +2625,7 @@ _Appears in:_ | `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | | `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | | `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | MaxItems: 9
MinItems: 1
| +| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | | #### LPXConfig @@ -2638,7 +2638,7 @@ LPXConfig identifies the component's compiled model and runtime settings. _Appears in:_ - [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2707,7 +2707,7 @@ When specified, a headless service is created for endpoint discovery. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | @@ -2725,7 +2725,7 @@ MultinodeSpec configures a multinode component. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | @@ -2833,7 +2833,7 @@ current LPX placement. _Appears in:_ - [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2902,8 +2902,8 @@ All other providers, versions, targets, and fields are rejected. _Appears in:_ - [ComponentRoleSpec](#componentrolespec) -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2921,7 +2921,7 @@ Restart specifies the restart policy for a graph deployment. _Appears in:_ -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -2958,7 +2958,7 @@ RestartStatus contains the status of a graph-level restart. _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -3031,7 +3031,7 @@ RollingUpdateStatus tracks the progress of an operator-managed rolling update. _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -3073,7 +3073,7 @@ Omit the field to opt out. _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) @@ -3088,7 +3088,7 @@ SchedulingSpec configures LPX scheduling attempts. _Appears in:_ - [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -3122,7 +3122,7 @@ SpecTopologyConstraint defines deployment-level topology placement requirements. _Appears in:_ -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) +- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -3141,7 +3141,7 @@ The topology profile is inherited from the deployment-level _Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) +- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) - [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) | Field | Description | Default | Validation | diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 41e925ab7e30..ee44a7387e7a 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -1577,7 +1577,6 @@ See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/# See [LPXAttemptRequestStatus](#lpxattemptrequeststatus). -**Validation:** MaxItems: 9 MinItems: 1 @@ -2703,7 +2702,7 @@ See [DynamoComponentDeploymentSpec](#v1beta1-dynamocomponentdeploymentspec). - + DynamoComponentDeploymentSharedSpec is the shared configuration used by both standalone DCDs and by the components embedded in a DynamoGraphDeployment. @@ -3028,7 +3027,7 @@ Both served DGD versions use this type, so conversion preserves its live fields **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) +**Appears in:** [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus), [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) modelDownload contains the model download status for remote builds. @@ -3296,7 +3295,7 @@ See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/# - + DynamoGraphDeploymentSpec defines the desired state of a DynamoGraphDeployment. @@ -3366,7 +3365,7 @@ See [DynamoGraphDeploymentExperimentalSpec](#v1beta1-dynamographdeploymentexperi - + DynamoGraphDeploymentStatus defines the observed state of a DynamoGraphDeployment. Unchanged between v1alpha1 and v1beta1. @@ -3819,7 +3818,6 @@ See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/# See [LPXAttemptRequestStatus](#v1beta1-lpxattemptrequeststatus). -**Validation:** MaxItems: 9 MinItems: 1 @@ -3828,7 +3826,7 @@ LPXConfig identifies the component's compiled model and runtime settings. **Kind:** `type` -**Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) +**Appears in:** [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) buildId references the immutable model build. @@ -4303,7 +4301,7 @@ SchedulingSpec configures LPX scheduling attempts. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) +**Appears in:** [DynamoGraphDeploymentSpec](#dynamographdeploymentspec), [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. Omission means unlimited; the value is not a solver budget. diff --git a/docs/fern/scripts/tests/test_api_reference_regressions.py b/docs/fern/scripts/tests/test_api_reference_regressions.py index 75190e4ba005..8b00b667ccf6 100644 --- a/docs/fern/scripts/tests/test_api_reference_regressions.py +++ b/docs/fern/scripts/tests/test_api_reference_regressions.py @@ -494,6 +494,18 @@ def test_kubernetes_field_type_links_resolve( assert local_link_targets <= rendered_anchors +def test_kubernetes_embedded_server_backlinks_survive( + kubernetes_page: _KubernetesPage, +) -> None: + _, mdx = kubernetes_page + section = mdx.split('", 1)[0] + backlinks = section.split("**Appears in:** ", 1)[1].split("\n", 1)[0] + assert backlinks == ( + "[MetricsServer](#metricsserver), [ServerConfiguration](#serverconfiguration), " + "[WebhookServer](#webhookserver)" + ) + + def test_kubernetes_external_type_links_leave_the_type_attribute( kubernetes_page: _KubernetesPage, ) -> None: @@ -739,15 +751,23 @@ def test_kubernetes_type_anchors_are_globally_unique( assert len(all_anchors) == len(set(all_anchors)) -def test_kubernetes_type_references_stay_package_local( +def test_kubernetes_type_references_resolve_on_same_page( kubernetes_package_pairs: _KubernetesPackagePairs, ) -> None: _, packages = kubernetes_package_pairs + all_anchors = {type_.anchor for package in packages for type_ in package.types} for package in packages: package_anchors = {type_.anchor for type_ in package.types} - refs = list(package.resource_types) - refs.extend(ref for type_ in package.types for ref in type_.appears_in) - assert all(ref.anchor in package_anchors for ref in refs) + assert all(ref.anchor in package_anchors for ref in package.resource_types) + refs = [ref for type_ in package.types for ref in type_.appears_in] + field_anchors = { + match.group("anchor") + for type_ in package.types + for field in type_.fields + for match in kubernetes_api_discovery.LINK_RE.finditer(field.type) + } + assert all(ref.anchor in all_anchors for ref in refs) + assert field_anchors <= all_anchors def test_kubernetes_field_links_follow_package_remaps( diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index aa93ac016a6f..61972681cdfd 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -48,7 +48,7 @@ # (PodReference, PodSnapshotSource/Spec/Status, # PodSnapshotContentSource/Spec/Status, PodSnapshotReference), which are # owned by github.com/ai-dynamo/snapshot. - "nvidia.com/v1alpha1": 68, + "nvidia.com/v1alpha1": 72, "nvidia.com/v1beta1": 76, "operator.config.dynamo.nvidia.com/v1alpha1": 30, } @@ -71,6 +71,9 @@ "DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRequestSpec", "DynamoGraphDeploymentRequestStatus", + "DynamoGraphDeploymentStatus", + "DynamoGraphDeploymentSpec", + "DynamoComponentDeploymentSharedSpec", ) @@ -203,6 +206,24 @@ def test_resource_types_link_to_real_types_on_the_same_page( ) +def test_cross_package_status_links_resolve_to_v1beta1_types( + reference: kubernetes_api_discovery.KubernetesReference, +) -> None: + """v1alpha1 may intentionally reuse status types owned by v1beta1.""" + by_package = {package.name: package for package in reference.packages} + alpha = by_package["nvidia.com/v1alpha1"] + beta = by_package["nvidia.com/v1beta1"] + alpha_status = next( + type_ for type_ in alpha.types if type_.name == "DynamoGraphDeploymentStatus" + ) + lpx_field = next(field for field in alpha_status.fields if field.name == "lpx") + beta_lpx_status = next( + type_ for type_ in beta.types if type_.name == "DynamoGraphDeploymentLPXStatus" + ) + + assert f"](#{beta_lpx_status.anchor})" in lpx_field.type + + def test_v1beta1_shared_type_names_use_deduplicated_anchors( reference: kubernetes_api_discovery.KubernetesReference, ) -> None: @@ -416,15 +437,34 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( } +@pytest.mark.parametrize( + "name,parent", + ( + ("LPXConfig", "DynamoComponentDeploymentSharedSpec"), + ("SchedulingSpec", "DynamoGraphDeploymentSpec"), + ("DynamoGraphDeploymentLPXStatus", "DynamoGraphDeploymentStatus"), + ), +) def test_shared_lpx_type_links_across_api_versions( + source_text: str, reference: kubernetes_api_discovery.KubernetesReference, + name: str, + parent: str, ) -> None: rendered = kubernetes_api_rendering.render_mdx(reference) alpha = rendered.split("## nvidia.com/v1beta1", 1)[0] - for name in ("LPXConfig", "SchedulingSpec", "DynamoGraphDeploymentLPXStatus"): - assert f"See [{name}](#{name.lower()})." in alpha - lpx = rendered.split('", 1)[0] - assert lpx.count("[DynamoComponentDeploymentSharedSpec]") == 1 + assert f"See [{name}](#{name.lower()})." in alpha + expected = [f"[{parent}](#{prefix}{parent.lower()})" for prefix in ("", "v1beta1-")] + section = source_text.split(f"#### {name}\n", 1)[1].split("\n#### ", 1)[0] + backlinks = section.split("_Appears in:_\n", 1)[1].split("\n\n", 1)[0] + assert backlinks.splitlines() == [f"- {link}" for link in expected] + shared = next(type_ for type_ in _iter_types(reference) if type_.name == name) + assert [f"[{ref.name}](#{ref.anchor})" for ref in shared.appears_in] == expected + section = rendered.split(f'", 1 + )[0] + backlinks = section.split("**Appears in:** ", 1)[1].split("\n", 1)[0] + assert backlinks == ", ".join(expected) def test_check_mode_returns_zero_on_fresh_outputs(workspace: Path) -> None: From 4b3ff71fa99f14f146ffd376990ccb18e5491cb6 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:17:50 +0300 Subject: [PATCH 019/196] docs(operator): explain LPX finalizer handshake (#438) Signed-off-by: Andrew Paprotsky (cherry picked from commit f10a6a1160ac8b4ddf83c802f92d483d62fab190) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/attempt_deadline.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index b5cfc04e982c..be8fe0dbdbd7 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -412,6 +412,10 @@ func lpxRequestsByName(requests []lpxv1alpha1.LPUPipelineRequest) map[string]*lp } func (r *graphReconciler) releaseLPXAttemptRecordingFinalizer(ctx context.Context, request *lpxv1alpha1.LPUPipelineRequest) error { + // The scheduler finishes cleanup and removes scheduling.lpu.nvidia.com/plan-protection + // independently, never waiting for lpxAttemptRecordingFinalizer. Dynamo releases + // its finalizer last to retain the request identity and terminal status during cleanup. + // Unknown finalizers block release; their owners must resolve them, not Dynamo. if request.DeletionTimestamp.IsZero() || len(request.Finalizers) != 1 || request.Finalizers[0] != lpxAttemptRecordingFinalizer { return nil } From fc7d7f484e4fc5ac55652cc02810a82b2b0fc451 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:39:10 +0300 Subject: [PATCH 020/196] test(operator): cover LPX eviction ownership changes (#439) Signed-off-by: Andrew Paprotsky (cherry picked from commit ff3e0d7b0d1f71cdc891ba328c17fef79c99914a) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/eviction_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 794077be077f..7000cf0b44e6 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -310,7 +310,7 @@ func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { current := &corev1.Pod{} require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(object), current)) - current.Labels["changed"] = "true" + current.OwnerReferences[0].Kind = "ForeignPodClique" require.NoError(t, delegated.Update(ctx, current)) return delegated.Delete(ctx, object, opts...) }, From 5600d70a32e9457a409f57b96003bd79bcb21492 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:39:10 +0300 Subject: [PATCH 021/196] fix(operator): bound LPX model download checks (#440) Signed-off-by: Andrew Paprotsky (cherry picked from commit a54ce59f447d55d7c9b67d43bc0794e8acb255cb) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/modeldownload.go | 11 ++++------ .../controller/lpx/modeldownload_test.go | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index 4b0392744128..f6727eda8056 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -29,7 +29,7 @@ const ( modelDownloadPendingMessage string = "Waiting for model downloads to complete" modelDownloadRefreshInterval = 24 * time.Hour - modelDownloadRefreshTimeout = 30 * time.Second + modelDownloadCheckTimeout = 30 * time.Second ) type lpxModelRegistry interface { @@ -75,13 +75,10 @@ func (r *graphReconciler) reconcileModelDownloads( return true, nil } running := !forceCheck && current && meta.IsStatusConditionTrue(deployment.Status.Conditions, "Ready") - if running { - refreshCtx, cancel := context.WithTimeout(ctx, modelDownloadRefreshTimeout) - defer cancel() - ctx = refreshCtx - } + checkCtx, cancel := context.WithTimeout(ctx, modelDownloadCheckTimeout) + defer cancel() - downloaded, ready, err := ensureModelsDownloaded(ctx, dgd, r.modelRegistry) + downloaded, ready, err := ensureModelsDownloaded(checkCtx, dgd, r.modelRegistry) checkSucceeded := ready && err == nil // Retain the running child's last observation when refresh is incomplete. if !running || checkSucceeded { diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index a70f098bbf1e..0d97f6fdd640 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -220,6 +220,26 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { } } +func TestInitialLPXModelDownloadChecksHaveDeadline(t *testing.T) { + for _, forceCheck := range []bool{false, true} { + t.Run(fmt.Sprintf("forceCheck=%t", forceCheck), func(t *testing.T) { + dgd := newModelDownloadDGD(modelDownloadTestBuildID) + child := newLPXTestDeployment(t, dgd) + registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestBuildID: true}, nil) + lifecycle := &graphReconciler{modelRegistry: registry} + started := time.Now() + + ready, err := lifecycle.reconcileModelDownloads(t.Context(), child, dgd, forceCheck) + + require.NoError(t, err) + require.True(t, ready) + require.Len(t, registry.deadlines, 1) + require.False(t, registry.deadlines[0].IsZero()) + require.WithinDuration(t, started.Add(modelDownloadCheckTimeout), registry.deadlines[0], time.Second) + }) + } +} + func TestModelDownloadSpecChangeRemainsFailClosed(t *testing.T) { t.Log("Keep the source's download status independent of the unobserved child") source := newModelDownloadDGD(modelDownloadTestBuildID) From 58b2e0d8dea6e55776898ac6405d00afd2b1c9cf Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:39:11 +0300 Subject: [PATCH 022/196] fix(operator): clean up stale LPX runtime config (#441) Signed-off-by: Andrew Paprotsky (cherry picked from commit a6498294dccd9a88d86578e262ef4ad0322ef222) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 50 +++++++++++++++++++ .../controller/lpx/controller_test.go | 41 +++++++++++++-- .../internal/controller/lpx/lifecycle.go | 4 +- 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 9f5fda10a42e..3472530b579c 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -25,6 +25,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -305,6 +306,9 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a deployment.Status.Components[name] = component state.Reason, state.Message = readiness.Classification, readiness.Message if readiness.Ready && !changed { + if err := r.deleteStaleLPXConfigMaps(ctx, deployment, resources); err != nil { + return state, ctrl.Result{}, err + } state.State = v1beta1.DGDStateSuccessful } classification, err := r.reconcileSelectedLPX(ctx, deployment, selected) @@ -419,6 +423,52 @@ func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alp return r.Update(ctx, synced) } +func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, resources []client.Object) error { + // Retain every ConfigMap rendered for the current workload. + desiredNames := make(map[string]struct{}, len(resources)) + for _, resource := range resources { + if _, ok := resource.(*corev1.ConfigMap); ok { + desiredNames[resource.GetName()] = struct{}{} + } + } + + // Discover obsolete ConfigMaps rooted in this exact LPX child. + stale := make([]client.Object, 0) + if err := visitLifecycleObjectPages(ctx, r.apiReader, &corev1.ConfigMapList{}, func(object k8sruntime.Object) error { + configMap := object.(client.Object) + if metav1.IsControlledBy(configMap, deployment) { + if _, desired := desiredNames[configMap.GetName()]; !desired { + stale = append(stale, configMap) + } + } + return nil + }, client.InNamespace(deployment.Namespace)); err != nil { + return err + } + if len(stale) == 0 { + return nil + } + + // Revalidate publication authority before deleting the discovered objects. + if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { + return err + } + + // Preconditions prevent stale observations from deleting replacements. + for _, configMap := range stale { + if !configMap.GetDeletionTimestamp().IsZero() { + continue + } + uid, resourceVersion := configMap.GetUID(), configMap.GetResourceVersion() + if err := r.Delete(ctx, configMap, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ + UID: &uid, ResourceVersion: &resourceVersion, + }}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { // Use the same normalized identity for endpoint creation and retirement. component := lpx.ServingComponent(source) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index add8523cf782..1dad78df7aa4 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -260,20 +260,27 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T endpoint := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ Name: "obsolete-endpoint", Namespace: child.Namespace, UID: "old-endpoint", OwnerReferences: owner, }} + runtimeConfig := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "obsolete-runtime", Namespace: child.Namespace, UID: "old-runtime", OwnerReferences: owner, + }} ordinary := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ Name: "ordinary-model", Namespace: child.Namespace, UID: "ordinary-service", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, }} foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() + foreignRuntime := runtimeConfig.DeepCopy() foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" + foreignRuntime.UID = "foreign-runtime" foreignPCS.Name = dynamo.PCSNameForLPX(child) foreignEndpoint.Name = foreignPCS.Name + "-lpx" + foreignRuntime.Name = foreignPCS.Name + "-runtime" foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences + foreignRuntime.OwnerReferences = ordinary.OwnerReferences if scenario == "pending finalizer" { pcs.Finalizers = []string{"example.com/staged-cleanup"} } - r := newLPXTestReconciler(t, nil, child, source, pcs, endpoint, ordinary, foreignPCS, foreignEndpoint) - for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint} { + r := newLPXTestReconciler(t, nil, child, source, pcs, endpoint, runtimeConfig, ordinary, foreignPCS, foreignEndpoint, foreignRuntime) + for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint, foreignRuntime} { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) } @@ -341,11 +348,11 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") require.NoError(t, err) require.Nil(t, retiring) - require.Equal(t, 2, deletes, "cleanup must not repeat deletes after they are accepted") + require.Equal(t, 3, deletes, "cleanup must not repeat deletes after they are accepted") } t.Log("Ordinary DGD resources and foreign same-name objects remain byte-for-byte unchanged") - for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint} { + for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint, foreignRuntime} { before := object.DeepCopyObject().(client.Object) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) require.Equal(t, before, object) @@ -536,6 +543,32 @@ func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) require.Equal(t, updated.Data, unchanged.Data) } +func TestLPXDeletesOnlyStaleOwnedRuntimeConfigMaps(t *testing.T) { + t.Log("Create current, stale, and foreign runtime ConfigMaps") + source := newLPXTestSource(lpx.PipelineSingle, "test-build") + child := newLPXTestDeployment(t, source) + owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} + current := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-current", Namespace: child.Namespace, UID: "current", OwnerReferences: owner, + }} + stale := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-stale", Namespace: child.Namespace, UID: "stale", OwnerReferences: owner, + }} + foreign := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-foreign", Namespace: child.Namespace, UID: "foreign", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, + }} + r := newLPXTestReconciler(t, nil, child, source, current, stale, foreign) + + t.Log("Delete only the stale ConfigMap owned by the current LPX child") + require.NoError(t, r.deleteStaleLPXConfigMaps(t.Context(), child, []client.Object{current})) + + t.Log("Preserve the current and foreign ConfigMaps") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(current), &corev1.ConfigMap{})) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), client.ObjectKeyFromObject(stale), &corev1.ConfigMap{}))) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(foreign), &corev1.ConfigMap{})) +} + func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { const componentProvider = "component provider" for _, scenario := range []struct { diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index b1632b4f8e73..008694c4d7c6 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -400,9 +400,9 @@ func (r *graphReconciler) retireInvalidLPXWorkload( return r.retireLPXRequest(ctx, deployment, "", &requests[0], reason) } - // Source edits can rename both resources; discover only this exact child's old objects. + // Source edits can rename resources; discover only this exact child's old objects. owned := make([]client.Object, 0) - for _, list := range []client.ObjectList{&corev1.ServiceList{}, &grovev1alpha1.PodCliqueSetList{}} { + for _, list := range []client.ObjectList{&corev1.ServiceList{}, &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}} { if err := visitLifecycleObjectPages(ctx, r.apiReader, list, func(object k8sruntime.Object) error { resource := object.(client.Object) if metav1.IsControlledBy(resource, deployment) { From 85f51867fd8729a44337910b48bb34c0207516d9 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:32:44 +0300 Subject: [PATCH 023/196] fix(operator): resolve Grove watch ownership (#442) Signed-off-by: Andrew Paprotsky (cherry picked from commit 1d182279fb9364f856d61888f1315c323d9ed59f) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_watch_setup.go | 112 ++++++---- .../controller/dgd_grove_watch_setup_test.go | 206 ++++++++++-------- .../dynamographdeployment_controller_test.go | 45 +--- 3 files changed, 199 insertions(+), 164 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index 814ccba31409..c47ccf48d08c 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -20,18 +20,17 @@ package controller import ( "context" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -104,27 +103,16 @@ func (s *groveWatchSetup) mapPodCliqueToRequests( obj client.Object, ) []ctrl.Request { podClique, ok := obj.(*grovev1alpha1.PodClique) - // LPX owns its role observations; the DGD watches the aggregate child status. - if !ok || podClique == nil || dynamolpx.OwnsPodClique(ctx, s.reader, podClique) { + if !ok { return nil } - dgdName := podClique.GetLabels()[consts.KubeLabelDynamoGraphDeploymentName] - if dgdName == "" { - log.FromContext(ctx).V(1).Info( - "PodClique missing DGD label", - "podClique", podClique.Name, - "namespace", podClique.Namespace, - ) + pcs, found := s.resolveOwningPodCliqueSet(ctx, podClique) + if !found { return nil } - return []ctrl.Request{{ - NamespacedName: types.NamespacedName{ - Name: dgdName, - Namespace: podClique.Namespace, - }, - }} + return mapPodCliqueSetToDGDRequest(pcs) } // mapPodCliqueScalingGroupToRequests walks PCSG -> PCS -> DGD because the PCS @@ -138,46 +126,88 @@ func (s *groveWatchSetup) mapPodCliqueScalingGroupToRequests( return nil } + pcs, found := s.resolvePodCliqueSetOwner(ctx, pcsg) + if !found { + return nil + } + return mapPodCliqueSetToDGDRequest(pcs) +} + +func (s *groveWatchSetup) resolveOwningPodCliqueSet( + ctx context.Context, + podClique *grovev1alpha1.PodClique, +) (*grovev1alpha1.PodCliqueSet, bool) { + controllerRef := metav1.GetControllerOf(podClique) + if controllerRef == nil || controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() { + return nil, false + } + + switch controllerRef.Kind { + case "PodCliqueSet": + return s.getPodCliqueSet(ctx, podClique.Namespace, controllerRef) + case "PodCliqueScalingGroup": + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + if err := s.reader.Get(ctx, types.NamespacedName{ + Name: controllerRef.Name, + Namespace: podClique.Namespace, + }, pcsg); err != nil || pcsg.UID != controllerRef.UID { + return nil, false + } + return s.resolvePodCliqueSetOwner(ctx, pcsg) + default: + return nil, false + } +} + +func (s *groveWatchSetup) resolvePodCliqueSetOwner( + ctx context.Context, + pcsg *grovev1alpha1.PodCliqueScalingGroup, +) (*grovev1alpha1.PodCliqueSet, bool) { controllerRef := metav1.GetControllerOf(pcsg) if controllerRef == nil || - controllerRef.Kind != "PodCliqueSet" || - controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() { - log.FromContext(ctx).V(1).Info( - "PodCliqueScalingGroup missing PodCliqueSet controller ownerReference", - "podCliqueScalingGroup", pcsg.Name, - "namespace", pcsg.Namespace, - ) - return nil + controllerRef.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || + controllerRef.Kind != "PodCliqueSet" { + return nil, false } + return s.getPodCliqueSet(ctx, pcsg.Namespace, controllerRef) +} +func (s *groveWatchSetup) getPodCliqueSet( + ctx context.Context, + namespace string, + controllerRef *metav1.OwnerReference, +) (*grovev1alpha1.PodCliqueSet, bool) { pcs := &grovev1alpha1.PodCliqueSet{} if err := s.reader.Get(ctx, types.NamespacedName{ Name: controllerRef.Name, - Namespace: pcsg.Namespace, + Namespace: namespace, }, pcs); err != nil { - log.FromContext(ctx).V(1).Info( - "failed to look up PodCliqueSet for PCSG", - "podCliqueScalingGroup", pcsg.Name, - "pcsName", controllerRef.Name, - "error", err, - ) - return nil + return nil, false } + return pcs, pcs.UID == controllerRef.UID +} +func mapPodCliqueSetToDGDRequest(pcs *grovev1alpha1.PodCliqueSet) []ctrl.Request { pcsOwnerRef := metav1.GetControllerOf(pcs) - if pcsOwnerRef == nil || pcsOwnerRef.Kind != consts.ResourceTypeDynamoGraphDeployment { - log.FromContext(ctx).V(1).Info( - "PodCliqueSet missing DynamoGraphDeployment controller ownerReference", - "pcsName", pcs.Name, - "namespace", pcs.Namespace, - ) + if pcsOwnerRef == nil || + pcsOwnerRef.Name == "" || + pcsOwnerRef.UID == "" { + return nil + } + + // Accept any served DGD version from Dynamo's API group. + groupVersion, err := schema.ParseGroupVersion(pcsOwnerRef.APIVersion) + if err != nil || + groupVersion.Group != nvidiacomv1beta1.GroupVersion.Group || + groupVersion.Version == "" || + pcsOwnerRef.Kind != nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind { return nil } return []ctrl.Request{{ NamespacedName: types.NamespacedName{ Name: pcsOwnerRef.Name, - Namespace: pcsg.Namespace, + Namespace: pcs.Namespace, }, }} } diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go index ad3f29f2f37c..ae7adc47cb31 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup_test.go @@ -7,111 +7,141 @@ import ( "testing" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/event" ) -func TestGroveWatchSkipsOnlyVerifiedLPXOwners(t *testing.T) { - t.Log("Distinguish LPX-owned Grove roles from ordinary DGD readiness events") - const wrongAPIVersion = "unrelated/v1" - const sourceName = "source-graph" +func TestGroveWatchMapsOnlyDGDControllerOwnedPodCliques(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, grovev1alpha1.AddToScheme(scheme)) - child := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "independent-materialization", UID: "child-uid"}} - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: "engine-pcs", Namespace: "workloads", UID: "pcs-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(child, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, - }} - group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ - Name: "engine-group", Namespace: "workloads", UID: "group-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }} - clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ - Name: "role", Namespace: "workloads", - Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: sourceName}, - Annotations: map[string]string{ - dynamolpx.DeploymentNameAnnotation: child.Name, - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleConductor, - dynamolpx.WorkloadDigestAnnotation: "sha256:workload", - dynamo.LPXDeploymentUIDAnnotation: string(child.UID), - }, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, - }} - ready := clique.DeepCopy() - ready.Status.ReadyReplicas++ - require.True(t, podCliqueEventPredicates().Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: ready})) - t.Log("The LPX controller observes role readiness and the DGD observes aggregate child status") - parent := newGroveWatchSetup(fake.NewClientBuilder().WithScheme(scheme).WithObjects(pcs, group).Build()) - for _, role := range []string{lpxv1alpha1.PodRoleAgent, lpxv1alpha1.PodRoleConductor, lpxv1alpha1.PodRoleCyborgWorker} { - t.Run(role, func(t *testing.T) { - observed := ready.DeepCopy() - observed.Annotations[lpxv1alpha1.PodRoleAnnotation] = role - require.Empty(t, parent.mapPodCliqueToRequests(t.Context(), observed)) - }) + controllerRef := func(apiVersion, kind, name string, uid types.UID) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: apiVersion, + Kind: kind, + Name: name, + UID: uid, + Controller: ptr.To(true), + } + } + + newObjects := func() (*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) { + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: "engine-pcs", + Namespace: "workloads", + UID: "pcs-uid", + OwnerReferences: []metav1.OwnerReference{controllerRef( + nvidiacomv1beta1.GroupVersion.String(), "DynamoGraphDeployment", "graph", "dgd-uid", + )}, + }} + group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ + Name: "engine-group", + Namespace: "workloads", + UID: "group-uid", + OwnerReferences: []metav1.OwnerReference{controllerRef( + grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + )}, + }} + clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ + Name: "role", + Namespace: "workloads", + OwnerReferences: []metav1.OwnerReference{controllerRef( + grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueScalingGroup", group.Name, group.UID, + )}, + }} + return clique, group, pcs } - t.Log("Authorable LPX hints cannot suppress ordinary readiness, nor can incomplete cached ownership") - for _, test := range []struct { + want := []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "graph"}}} + tests := []struct { name string - mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) + mutate func(*grovev1alpha1.PodClique, *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodCliqueSet) []client.Object + want []ctrl.Request }{ - {"ordinary unmarked", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - c.Annotations = nil - }}, - {"ordinary direct owner", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - c.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(p, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - }}, - {"ordinary grouped owner", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - p.OwnerReferences[0].Kind = "DynamoGraphDeployment" - }}, - {"missing group", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - g.Name = "not-the-owner" - }}, - {"replaced group", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - g.UID = "new-group-uid" - }}, - {"wrong group API", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - c.OwnerReferences[0].APIVersion = wrongAPIVersion - }}, - {"missing PCS", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - p.Name = "not-the-owner" - }}, - {"replaced PCS", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - p.UID = "new-pcs-uid" - }}, - {"wrong PCS API", func(_ *grovev1alpha1.PodClique, g *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - g.OwnerReferences[0].APIVersion = wrongAPIVersion - }}, - {"wrong child API", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - p.OwnerReferences[0].APIVersion = wrongAPIVersion - }}, - {"different child", func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, p *grovev1alpha1.PodCliqueSet) { - p.OwnerReferences[0].Name = "other-graph" - }}, - {"stale child stamp", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - c.Annotations[dynamo.LPXDeploymentUIDAnnotation] = "old-child-uid" - }}, - {"missing child name stamp", func(c *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, _ *grovev1alpha1.PodCliqueSet) { - delete(c.Annotations, dynamolpx.DeploymentNameAnnotation) - }}, - } { + { + name: "direct PCS ownership", + mutate: func(clique *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + clique.OwnerReferences = []metav1.OwnerReference{controllerRef( + grovev1alpha1.SchemeGroupVersion.String(), "PodCliqueSet", pcs.Name, pcs.UID, + )} + return []client.Object{pcs} + }, + want: want, + }, + { + name: "PCSG-mediated ownership", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + return []client.Object{group, pcs} + }, + want: want, + }, + { + name: "LGD-owned PCS", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + pcs.OwnerReferences[0] = controllerRef( + nvidiacomv1alpha1.GroupVersion.String(), "LPXGraphDeployment", "graph", "lgd-uid", + ) + return []client.Object{group, pcs} + }, + }, + { + name: "wrong group owner UID", + mutate: func(clique *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + clique.OwnerReferences[0].UID = "old-group-uid" + return []client.Object{group, pcs} + }, + }, + { + name: "missing cached group", + mutate: func(_ *grovev1alpha1.PodClique, _ *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + return []client.Object{pcs} + }, + }, + { + name: "same-name foreign PCS", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + pcs.UID = "replacement-pcs-uid" + return []client.Object{group, pcs} + }, + }, + { + name: "legacy v1alpha1 DGD owner", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + pcs.OwnerReferences[0].APIVersion = nvidiacomv1alpha1.GroupVersion.String() + return []client.Object{group, pcs} + }, + want: want, + }, + { + name: "foreign PCS owner API group", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + pcs.OwnerReferences[0].APIVersion = "foreign.example/v1beta1" + return []client.Object{group, pcs} + }, + }, + { + name: "malformed PCS owner API version", + mutate: func(_ *grovev1alpha1.PodClique, group *grovev1alpha1.PodCliqueScalingGroup, pcs *grovev1alpha1.PodCliqueSet) []client.Object { + pcs.OwnerReferences[0].APIVersion = "nvidia.com/" + return []client.Object{group, pcs} + }, + }, + } + + for _, test := range tests { t.Run(test.name, func(t *testing.T) { - currentPCS, currentGroup := pcs.DeepCopy(), group.DeepCopy() - observed := ready.DeepCopy() - test.mutate(observed, currentGroup, currentPCS) - reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(currentPCS, currentGroup).Build() - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: observed.Namespace, Name: sourceName}}}, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), observed)) + clique, group, pcs := newObjects() + objects := test.mutate(clique, group, pcs) + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + require.Equal(t, test.want, newGroveWatchSetup(reader).mapPodCliqueToRequests(t.Context(), clique)) }) } } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 4598d1cebc89..336436e5508b 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -3529,39 +3529,6 @@ func TestDGDGroveTopologyConditionReconciler_Reconcile(t *testing.T) { } } -func TestGroveWatchSetup_MapPodCliqueToRequests(t *testing.T) { - setup := newGroveWatchSetup(nil) - - t.Run("labeled PodClique maps directly to its DGD", func(t *testing.T) { - requests := setup.mapPodCliqueToRequests( - context.Background(), - &grovev1alpha1.PodClique{ - ObjectMeta: metav1.ObjectMeta{ - Name: "graph-0-worker", - Namespace: "default", - Labels: map[string]string{ - commonconsts.KubeLabelDynamoGraphDeploymentName: "graph", - }, - }, - }, - ) - - require.Len(t, requests, 1) - assert.Equal(t, types.NamespacedName{Namespace: "default", Name: "graph"}, requests[0].NamespacedName) - }) - - t.Run("unlabeled or unrelated objects are ignored", func(t *testing.T) { - assert.Empty(t, setup.mapPodCliqueToRequests( - context.Background(), - &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{Name: "orphan", Namespace: "default"}}, - )) - assert.Empty(t, setup.mapPodCliqueToRequests( - context.Background(), - &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "not-a-podclique", Namespace: "default"}}, - )) - }) -} - func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { // Register Grove types with the scheme so fake client can handle them if err := grovev1alpha1.AddToScheme(scheme.Scheme); err != nil { @@ -3582,11 +3549,13 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "dynamo-recipe-0-worker", Namespace: "mwieczorek-dsv32-trtllm-agg", + UID: "pcsg-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet", Name: "dynamo-recipe", + UID: "pcs-uid", Controller: ptr.To(true), }, }, @@ -3596,14 +3565,16 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "dynamo-recipe", Namespace: "mwieczorek-dsv32-trtllm-agg", + UID: "pcs-uid", Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "dynamo-recipe", }, OwnerReferences: []metav1.OwnerReference{ { - APIVersion: v1alpha1.GroupVersion.String(), + APIVersion: v1beta1.GroupVersion.String(), Kind: "DynamoGraphDeployment", Name: "dynamo-recipe", + UID: "dgd-uid", Controller: ptr.To(true), }, }, @@ -3619,11 +3590,13 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "truncated-pcs-0-worker", Namespace: "default", + UID: "pcsg-uid", OwnerReferences: []metav1.OwnerReference{ { APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet", Name: "truncated-pcs", + UID: "pcs-uid", Controller: ptr.To(true), }, }, @@ -3633,14 +3606,16 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "truncated-pcs", Namespace: "default", + UID: "pcs-uid", Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "my-very-long-original-dgd-name", }, OwnerReferences: []metav1.OwnerReference{ { - APIVersion: v1alpha1.GroupVersion.String(), + APIVersion: v1beta1.GroupVersion.String(), Kind: "DynamoGraphDeployment", Name: "my-very-long-original-dgd-name", + UID: "dgd-uid", Controller: ptr.To(true), }, }, From a0e8c288ebf6cac1d91353ba0a0031bb41db89b4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:00:32 +0300 Subject: [PATCH 024/196] refactor(operator): keep provider rejection in program (#443) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7b90a943bbfe15546c829438064cc3fe5b28d063) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_component_program.go | 19 ++++++++++ .../controller/dgd_workload_program_test.go | 24 ++++++++++++ .../dynamographdeployment_controller.go | 7 ---- .../dynamographdeployment_controller_test.go | 38 +++++++++++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_component_program.go b/deploy/operator/internal/controller/dgd_component_program.go index 800701b85f84..354e432381eb 100644 --- a/deploy/operator/internal/controller/dgd_component_program.go +++ b/deploy/operator/internal/controller/dgd_component_program.go @@ -25,6 +25,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type componentProgram struct { @@ -69,6 +70,23 @@ func (p *componentProgram) Reconcile( ) (programResult workloadProgramResult, retErr error) { programResult = newWorkloadProgramResult(req.DGD) clearComponentGPUShapes(programResult.Status.Components) + + // Reject components that the component pathway cannot materialize. + for i := range req.DGD.Spec.Components { + component := &req.DGD.Spec.Components[i] + if !component.ManagedByExternalController() { + continue + } + err := fmt.Errorf( + "component %q of type %q requires the Grove workload provider", + component.ComponentName, + component.ComponentType, + ) + programResult.Fail(req.DGD.Generation, "LPXRejected", err) + return programResult, reconcile.TerminalError(err) + } + + // Classify failures from normal component reconciliation. defer func() { if retErr == nil { return @@ -79,6 +97,7 @@ func (p *componentProgram) Reconcile( } programResult.Fail(req.DGD.Generation, reason, retErr) }() + log.FromContext(ctx).Info( "Reconciling Dynamo components deployments", "hasMultinode", req.DGD.HasAnyMultinodeComponent(), diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index cc4327649f1c..543c0ec2def3 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -117,6 +117,30 @@ func TestSelectedGroveProgramDoesNotFallbackWhenUnavailable(t *testing.T) { assert.Contains(t, ready.Message, "Grove is disabled") } +func TestComponentProgramRejectsExternallyManagedComponents(t *testing.T) { + t.Log("Create a component-provider DGD containing an externally managed component") + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 4}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "serving", + ComponentType: nvidiacomv1beta1.ComponentTypeLPX, + }}, + }, + } + + t.Log("Reconcile the component program") + result, err := (&componentProgram{}).Reconcile(t.Context(), workloadProgramRequest{DGD: dgd}) + require.ErrorIs(t, err, reconcile.TerminalError(nil)) + + t.Log("Verify the program returns the public LPX rejection without a terminal-error prefix") + ready := meta.FindStatusCondition(result.Status.Conditions, "Ready") + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, "LPXRejected", ready.Reason) + assert.Equal(t, `component "serving" of type "lpx" requires the Grove workload provider`, ready.Message) +} + func TestNewWorkloadProgramResultCopiesStatus(t *testing.T) { dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ Status: nvidiacomv1beta1.DynamoGraphDeploymentStatus{ diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index a8b15675eca6..2c0ed0f82e56 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -184,13 +184,6 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, err } - // LPX workloads have their own Grove owner; component-provider rendering cannot realize them. - if dynamoDeployment.HasLPXComponent() && provider != workloadProviderGrove { - programResult := newWorkloadProgramResult(dynamoDeployment) - programResult.Fail(dynamoDeployment.Generation, "LPXRejected", fmt.Errorf("LPX requires the Grove workload provider")) - return ctrl.Result{}, r.persistWorkloadProgramResult(ctx, dynamoDeployment, programResult) - } - program, err := r.selectWorkloadProgram(provider) if err != nil { return ctrl.Result{}, err diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 336436e5508b..57ce547a2adb 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -57,6 +57,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) func newDynamoGraphDeploymentControllerTestScheme(t testing.TB) *runtime.Scheme { @@ -149,6 +150,43 @@ func TestDynamoGraphDeploymentReconcileLocksProviderBeforeRejectingStoredCheckpo require.Zero(t, stored.Status.ObservedGeneration) } +func TestDynamoGraphDeploymentReconcilePersistsComponentProgramLPXRejection(t *testing.T) { + t.Log("Create a finalized DGD durably assigned to the component provider with an LPX component") + dgd := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dgd", + Namespace: "default", + Generation: 4, + Annotations: map[string]string{ + commonconsts.KubeAnnotationWorkloadProvider: commonconsts.WorkloadProviderComponent, + }, + }, + Spec: v1beta1.DynamoGraphDeploymentSpec{ + Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "serving", + ComponentType: v1beta1.ComponentTypeLPX, + }}, + }, + } + controller_common.AddFinalizer(dgd) + reconciler := createTestDGDReconcilerWithStatus(dgd) + + t.Log("Reconcile through the outer controller") + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) + require.ErrorIs(t, err, reconcile.TerminalError(nil)) + + t.Log("Verify the outer controller persisted the complete program-owned failure status") + stored := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, reconciler.Client.Get(t.Context(), client.ObjectKeyFromObject(dgd), stored)) + require.Equal(t, commonconsts.WorkloadProviderComponent, stored.Annotations[commonconsts.KubeAnnotationWorkloadProvider]) + require.Equal(t, v1beta1.DGDStateFailed, stored.Status.State) + ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready") + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, "LPXRejected", ready.Reason) + require.Equal(t, `component "serving" of type "lpx" requires the Grove workload provider`, ready.Message) +} + func TestDynamoGraphDeploymentReconcileFinalizesDeletingStoredCheckpointIncompatibility(t *testing.T) { now := metav1.Now() dgd := &v1beta1.DynamoGraphDeployment{ From c69ed58c5b2a59a358e11bbf3503d7551cbe1a3f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:00:33 +0300 Subject: [PATCH 025/196] refactor(operator): isolate LPX restart observation (#444) Signed-off-by: Andrew Paprotsky (cherry picked from commit 646bcee4ad121fbe5a3940c9dfcc41d9ee29d20d) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_restart.go | 60 ++++++++++++++----- .../internal/controller/dgd_grove_program.go | 37 +++++++----- .../controller/dgd_lpx_envtest_test.go | 2 +- .../internal/controller/dgd_lpx_handoff.go | 20 ++++--- .../controller/dgd_lpx_handoff_test.go | 32 +++++----- .../controller/dgd_workload_program_test.go | 1 + .../dynamographdeployment_controller.go | 2 +- 7 files changed, 99 insertions(+), 55 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index 9111ec56deeb..b774cade5a52 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -14,38 +14,63 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// resolveRestartProgress composes child-owned LPX and ordinary Grove observations. -func (p *groveProgram) resolveRestartProgress( +type lpxRestartProgressResolver struct { + reader client.Reader +} + +func newLPXRestartProgressResolver(reader client.Reader) *lpxRestartProgressResolver { + return &lpxRestartProgressResolver{reader: reader} +} + +// Resolve returns LPX components whose current child has not completed the selected restart. +func (r *lpxRestartProgressResolver) Resolve( + ctx context.Context, + source *v1beta1.DynamoGraphDeployment, + inProgress []string, +) []string { + childComponents := r.observeRestart(ctx, source) + remaining := make([]string, 0, len(inProgress)) + for _, name := range inProgress { + if !childComponents[name].Ready { + remaining = append(remaining, name) + } + } + return remaining +} + +// resolveCompositeGroveRestartProgress composes child-owned LPX and ordinary Grove observations. +func resolveCompositeGroveRestartProgress( ctx context.Context, source *v1beta1.DynamoGraphDeployment, ordinaryDGD *v1beta1.DynamoGraphDeployment, inProgress []string, + ordinaryResolver *groveRestartProgressResolver, + lpxResolver *lpxRestartProgressResolver, ) []string { ordinary := make([]string, 0, len(inProgress)) + lpxComponents := make([]string, 0, len(inProgress)) pending := make(map[string]bool, len(inProgress)) - var childComponents map[string]v1beta1.ComponentReplicaStatus - childObserved := false for _, name := range inProgress { component := source.GetComponentByName(name) if component == nil { continue } - if !component.IsLPX() { + if component.IsLPX() { + lpxComponents = append(lpxComponents, name) + } else { ordinary = append(ordinary, name) - continue - } - if !childObserved { - childComponents = observeLPXRestart(ctx, p.lpx.Client, source) - childObserved = true } - if !childComponents[name].Ready { + } + + // Observe the shared LPX child before ordinary Grove restart progress. + if len(lpxComponents) > 0 { + for _, name := range lpxResolver.Resolve(ctx, source, lpxComponents) { pending[name] = true } } - if len(ordinary) > 0 { - for _, name := range p.restartProgress.Resolve(ctx, ordinaryDGD, ordinary) { + for _, name := range ordinaryResolver.Resolve(ctx, ordinaryDGD, ordinary) { pending[name] = true } } @@ -59,11 +84,14 @@ func (p *groveProgram) resolveRestartProgress( return remaining } -// observeLPXRestart returns component status from one current, ready restart observation. +// observeRestart returns component status from one current, ready restart observation. // A failed read or incomplete child leaves every requested member pending. -func observeLPXRestart(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment) map[string]v1beta1.ComponentReplicaStatus { +func (r *lpxRestartProgressResolver) observeRestart( + ctx context.Context, + source *v1beta1.DynamoGraphDeployment, +) map[string]v1beta1.ComponentReplicaStatus { child := &v1alpha1.LPXGraphDeployment{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || + if err := r.reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || dynamo.ValidateLPXSource(child, source) != nil || source.Spec.Restart == nil || child.Annotations[dynamo.LPXRestartAnnotation] != source.Spec.Restart.ID { diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 6f383506da60..d9a4d5aa613a 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -30,15 +30,16 @@ import ( ) type groveProgram struct { - sharedResources *dgdSharedResourcesReconciler - rollout *dgdWorkerRolloutReconciler - restart *dgdRestartReconciler - restartProgress *groveRestartProgressResolver - workloads *groveWorkloadsReconciler - scalingAdapters *dgdScalingAdaptersReconciler - topology *dgdGroveTopologyConditionReconciler - gate features.Gate - lpx *dgdLPXHandoff + sharedResources *dgdSharedResourcesReconciler + rollout *dgdWorkerRolloutReconciler + restart *dgdRestartReconciler + restartProgress *groveRestartProgressResolver + lpxRestartProgress *lpxRestartProgressResolver + workloads *groveWorkloadsReconciler + scalingAdapters *dgdScalingAdaptersReconciler + topology *dgdGroveTopologyConditionReconciler + gate features.Gate + lpx *dgdLPXHandoff } // newGroveProgram wires the Grove pathway at the DGD composition root. @@ -55,9 +56,10 @@ func (r *DynamoGraphDeploymentReconciler) newGroveProgram() *groveProgram { r.SSHKeyManager, r.RBACManager, ), - rollout: rollout, - restart: newDGDRestartReconciler(), - restartProgress: newGroveRestartProgressResolver(r.Client), + rollout: rollout, + restart: newDGDRestartReconciler(), + restartProgress: newGroveRestartProgressResolver(r.Client), + lpxRestartProgress: newLPXRestartProgressResolver(r.Client), workloads: newGroveWorkloadsReconciler( r.Client, r.Recorder, @@ -69,7 +71,7 @@ func (r *DynamoGraphDeploymentReconciler) newGroveProgram() *groveProgram { scalingAdapters: newDGDScalingAdaptersReconciler(r.Client, r.Recorder), topology: newDGDGroveTopologyConditionReconciler(r.Client), gate: r.RuntimeConfig.Gate, - lpx: &dgdLPXHandoff{Client: r.Client}, + lpx: &dgdLPXHandoff{client: r.Client}, } } @@ -131,7 +133,14 @@ func (p *groveProgram) Reconcile( req.DGD, &programResult.Status, func(ctx context.Context, source *nvidiacomv1beta1.DynamoGraphDeployment, inProgress []string) []string { - return p.resolveRestartProgress(ctx, source, ordinaryDGD, inProgress) + return resolveCompositeGroveRestartProgress( + ctx, + source, + ordinaryDGD, + inProgress, + p.restartProgress, + p.lpxRestartProgress, + ) }, ) recordRestartTransition(previousRestart, restart.Status, &programResult) diff --git a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go index 87d7bece97a2..967118cb0d57 100644 --- a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go @@ -38,7 +38,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.False(t, source.Status.Components["lpx"].Ready) t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") - handoff := &dgdLPXHandoff{Client: env.Client()} + handoff := &dgdLPXHandoff{client: env.Client()} child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.Equal(t, int64(1), child.Generation) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index e86bcafc2546..767cabd8b1ad 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -18,13 +18,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// dgdLPXHandoff owns only the generated child and its status projection. Build, -// Grove and scheduler lifecycles are exclusively the LPX controller's concern. -type dgdLPXHandoff struct{ client.Client } +// dgdLPXHandoff owns only convergence of the generated child resource family. +// Build, Grove and scheduler lifecycles are exclusively the LPX controller's concern. +type dgdLPXHandoff struct { + client client.Client +} func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGraphDeployment) (*v1alpha1.LPXGraphDeployment, error) { child := &v1alpha1.LPXGraphDeployment{} - err := r.Get(ctx, client.ObjectKeyFromObject(source), child) + err := r.client.Get(ctx, client.ObjectKeyFromObject(source), child) if err != nil && !apierrors.IsNotFound(err) { return nil, err } @@ -55,7 +57,7 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra child = &v1alpha1.LPXGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace}, } - if err := ctrl.SetControllerReference(source, child, r.Scheme()); err != nil { + if err := ctrl.SetControllerReference(source, child, r.client.Scheme()); err != nil { return nil, err } } @@ -63,9 +65,9 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXRestartAnnotation, restart) metav1.SetMetaDataAnnotation(&child.ObjectMeta, lpx.DGDGenerationAnnotation, strconv.FormatInt(source.Generation, 10)) if exists { - err = r.Update(ctx, child) + err = r.client.Update(ctx, child) } else { - err = r.Create(ctx, child) + err = r.client.Create(ctx, child) } return child, err } @@ -82,12 +84,12 @@ func (r *dgdLPXHandoff) deleteChild(ctx context.Context, child *v1alpha1.LPXGrap return nil } uid, version := child.UID, child.ResourceVersion - return client.IgnoreNotFound(r.Delete(ctx, child, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}})) + return client.IgnoreNotFound(r.client.Delete(ctx, child, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}})) } func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGraphDeployment) error { child := &v1alpha1.LPXGraphDeployment{} - if err := r.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil { + if err := r.client.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil { return client.IgnoreNotFound(err) } if !exactLPXSourceOwner(child, source) { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 020308b05c52..e9a82f8c8bce 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -57,7 +57,7 @@ func newLPXHandoffFixture(t *testing.T, fixture string) (*v1alpha1.LPXGraphDeplo WithRESTMapper(groveScaleRESTMapper()). WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}, &v1alpha1.LPXGraphDeployment{}). WithObjects(source).Build() - child, err := (&dgdLPXHandoff{Client: kube}).Reconcile(t.Context(), source) + child, err := (&dgdLPXHandoff{client: kube}).Reconcile(t.Context(), source) require.NoError(t, err) child.UID, child.Generation = "child-uid", 3 require.NoError(t, kube.Update(t.Context(), child)) @@ -99,7 +99,7 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { t.Log("Create the generated handoff from a real source DGD") source := newLPXHandoffSource(t, "node-local-v2-lpu-only") kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(source).Build() - handoff := &dgdLPXHandoff{Client: kube} + handoff := &dgdLPXHandoff{client: kube} child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.True(t, exactLPXSourceOwner(child, source)) @@ -120,7 +120,7 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { foreign.OwnerReferences = nil } foreignClient := fake.NewClientBuilder().WithScheme(kube.Scheme()).WithObjects(foreign).Build() - _, err := (&dgdLPXHandoff{Client: foreignClient}).Reconcile(t.Context(), source) + _, err := (&dgdLPXHandoff{client: foreignClient}).Reconcile(t.Context(), source) require.ErrorContains(t, err, "adoption is not supported", invalidIdentity) } @@ -303,16 +303,18 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { t.Log("Request an LPX-only restart without requiring an ordinary PCS") _, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") source.Spec.Restart = &v1beta1.Restart{ID: "restart-1", Strategy: &v1beta1.RestartStrategy{Type: strategy}} - handoff := &dgdLPXHandoff{Client: kube} + handoff := &dgdLPXHandoff{client: kube} unchanged, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.Empty(t, unchanged.Annotations[dynamo.LPXRestartAnnotation]) - resolveProgress := (&DynamoGraphDeploymentReconciler{ + program := (&DynamoGraphDeploymentReconciler{ Client: kube, RuntimeConfig: &commoncontroller.RuntimeConfig{}, - }).newGroveProgram().resolveRestartProgress + }).newGroveProgram() ordinaryDGD := projectOrdinaryGroveDeployment(source) progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { - return resolveProgress(ctx, source, ordinaryDGD, inProgress) + return resolveCompositeGroveRestartProgress( + ctx, source, ordinaryDGD, inProgress, program.restartProgress, program.lpxRestartProgress, + ) } restart := newDGDRestartReconciler().Resolve(t.Context(), source, &source.Status, progress) require.Equal(t, []string{"lpx"}, restart.Status.InProgress) @@ -377,7 +379,7 @@ func TestLPXHandoffOrdinaryScalingPreservesTheEntireAttempt(t *testing.T) { meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) require.NoError(t, kube.Status().Update(t.Context(), child)) beforeChild := child.DeepCopy() - handoff := &dgdLPXHandoff{Client: kube} + handoff := &dgdLPXHandoff{client: kube} t.Log("Reorder the pair and make five frontend-only edits without changing the child") source.Spec.Components[0], source.Spec.Components[1] = source.Spec.Components[1], source.Spec.Components[0] @@ -458,19 +460,21 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { return reader.Get(ctx, key, obj, opts...) }, }) - resolveProgress := (&DynamoGraphDeploymentReconciler{ + program := (&DynamoGraphDeploymentReconciler{ Client: observed, RuntimeConfig: &commoncontroller.RuntimeConfig{}, - }).newGroveProgram().resolveRestartProgress + }).newGroveProgram() ordinaryDGD := projectOrdinaryGroveDeployment(source) progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { - return resolveProgress(ctx, source, ordinaryDGD, inProgress) + return resolveCompositeGroveRestartProgress( + ctx, source, ordinaryDGD, inProgress, program.restartProgress, program.lpxRestartProgress, + ) } restarter := newDGDRestartReconciler() source.Status.Restart = restarter.Resolve(t.Context(), source, &source.Status, progress).Status require.NotEmpty(t, source.Status.Restart.InProgress) t.Log("Either selected member delivers one token to the same child") - handoff := &dgdLPXHandoff{Client: kube} + handoff := &dgdLPXHandoff{client: kube} child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.Equal(t, "pair-restart", child.Annotations[dynamo.LPXRestartAnnotation]) @@ -560,7 +564,7 @@ func TestLPXChildFinalizationAndDeselectionWaitForCleanup(t *testing.T) { child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") child.Finalizers = []string{"test.example/child-cleanup"} require.NoError(t, kube.Update(t.Context(), child)) - handoff := &dgdLPXHandoff{Client: kube} + handoff := &dgdLPXHandoff{client: kube} source.Spec.Components = nil deleting, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) @@ -609,7 +613,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { }, }) require.NoError(t, kube.Update(t.Context(), source)) - child, err := (&dgdLPXHandoff{Client: kube}).Reconcile(t.Context(), source) + child, err := (&dgdLPXHandoff{client: kube}).Reconcile(t.Context(), source) require.NoError(t, err) child.Status.ObservedGeneration = child.Generation child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "ModelDownloadPending"}} diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index 543c0ec2def3..b1425ff2f542 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -89,6 +89,7 @@ func TestDGDWorkloadProgramSelection(t *testing.T) { assert.NotNil(t, grove.rollout) assert.NotNil(t, grove.restart) assert.NotNil(t, grove.restartProgress) + assert.NotNil(t, grove.lpxRestartProgress) assert.NotNil(t, grove.workloads) assert.NotNil(t, grove.scalingAdapters) assert.NotNil(t, grove.topology) diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 2c0ed0f82e56..3ff87eb87efa 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -236,7 +236,7 @@ func (r *DynamoGraphDeploymentReconciler) persistWorkloadProgramResult( func (r *DynamoGraphDeploymentReconciler) FinalizeResource(ctx context.Context, dynamoDeployment *nvidiacomv1beta1.DynamoGraphDeployment) error { // Wait for the LPX child's cleanup before deleting graph-owned checkpoints. - if err := (&dgdLPXHandoff{Client: r.Client}).Finalize(ctx, dynamoDeployment); err != nil { + if err := (&dgdLPXHandoff{client: r.Client}).Finalize(ctx, dynamoDeployment); err != nil { return err } From db52c6f7f648806cbb0a46347fc16888a9c6557f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:32:49 +0300 Subject: [PATCH 026/196] refactor(operator): model absent ordinary Grove workload (#445) Signed-off-by: Andrew Paprotsky (cherry picked from commit ac5b298d9a91b6f99a2e9ef5000c81456704ed93) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_workload_renderer.go | 8 ++++ .../dgd_grove_workloads_reconciler.go | 3 +- .../dgd_grove_workloads_reconciler_test.go | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index 90075338ff30..087ec4e4996a 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -95,6 +95,14 @@ func (r *groveWorkloadRenderer) Render( existingPodCliqueSet = nil } + // Represent an empty ordinary projection as no desired PodCliqueSet. + if len(ordinary.Spec.Components) == 0 { + return &grovePodCliqueSetRender{ + existing: existingPodCliqueSet, + renderDeployment: ordinary, + }, nil + } + workerHashSuffixNeeded := shouldRenderGroveWorkerHashSuffix(ordinary, existingPodCliqueSet, workerGenerationChanged) renderDeployment, err := groveRenderDeployment(ordinary, source, existingPodCliqueSet, workerHashSuffixNeeded) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 09f95481a684..ebc590cd3301 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -95,7 +95,8 @@ func (r *groveWorkloadsReconciler) Reconcile( logger.Error(err, "failed to generate the Grove GangSet") return ReconcileResult{}, fmt.Errorf("failed to generate the Grove GangSet: %w", err) } - if len(renderedPodCliqueSet.desired.Spec.Template.Cliques) == 0 { + // Retire the exact owned observation when the desired ordinary workload is absent. + if renderedPodCliqueSet.desired == nil { if existing := renderedPodCliqueSet.existing; existing != nil { if !metav1.IsControlledBy(existing, source) { return ReconcileResult{}, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 3ccf0e612bf6..6584a3c8ad4f 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -136,6 +136,49 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { assert.Equal(t, *wantSpec, dgd.Spec) } +func TestGroveWorkloadsReconcilerUsesStableReadinessWithoutOrdinaryPodCliqueSet(t *testing.T) { + t.Log("Project an LPX-only graph with an obsolete ordinary PodCliqueSet") + const dgdName = "graph" + source := newLPXHandoffSource(t, "node-local-v2-lpu-only") + source.Name, source.Namespace, source.UID = dgdName, corev1.NamespaceDefault, "dgd-uid" + ordinary := projectOrdinaryGroveDeployment(source) + owned := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), + Namespace: source.Namespace, + UID: "pcs-uid", + ResourceVersion: "1", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + }} + kubeClient := fake.NewClientBuilder(). + WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). + WithRESTMapper(groveScaleRESTMapper()). + WithObjects(source, owned). + Build() + reconciler := &DynamoGraphDeploymentReconciler{ + Client: kubeClient, + Config: &configv1alpha1.OperatorConfiguration{}, + Recorder: events.NewFakeRecorder(10), + RuntimeConfig: &commoncontroller.RuntimeConfig{}, + DockerSecretRetriever: &mockDockerSecretRetriever{GetSecretsFunc: func(string, string) ([]string, error) { + return nil, nil + }}, + } + + t.Log("Retire the obsolete workload under the compatible public condition reason") + result, err := reconciler.newGroveProgram().workloads.Reconcile(t.Context(), source, ordinary, nil, nil) + require.NoError(t, err) + require.Equal(t, nvidiacomv1beta1.DGDStatePending, result.State) + require.Equal(t, Reason("RemovingEmptyPodCliqueSet"), result.Reason) + + t.Log("Report stable-resource readiness after observing the PodCliqueSet absent") + result, err = reconciler.newGroveProgram().workloads.Reconcile(t.Context(), source, ordinary, nil, nil) + require.NoError(t, err) + require.Equal(t, nvidiacomv1beta1.DGDStateSuccessful, result.State) + pcsList := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, kubeClient.List(t.Context(), pcsList)) + require.Empty(t, pcsList.Items) +} + func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFails(t *testing.T) { tests := []struct { name string From 2b17babe1b42b83fa0344618d26e2131bdb42725 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:50:26 +0300 Subject: [PATCH 027/196] fix(helm): require Grove for LPX (#446) Signed-off-by: Andrew Paprotsky (cherry picked from commit c29f1bdbb34e1e9d4364f9383ed46afcddc63ece) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/Makefile | 2 +- .../platform/components/operator/.helmignore | 2 + .../operator/templates/validate-values.yaml | 19 +++++++ .../operator/tests/lpx_config_test.yaml | 55 +++++++++++++++++++ .../platform/tests/lpx_config_test.yaml | 25 ++++++++- .../tests/validate_removed_values_test.yaml | 16 ------ 6 files changed, 101 insertions(+), 18 deletions(-) create mode 100644 deploy/helm/charts/platform/components/operator/tests/lpx_config_test.yaml diff --git a/deploy/helm/charts/platform/Makefile b/deploy/helm/charts/platform/Makefile index 3452fe3260ec..ba928fe4afcd 100644 --- a/deploy/helm/charts/platform/Makefile +++ b/deploy/helm/charts/platform/Makefile @@ -84,7 +84,7 @@ verify-operator-chart-crds: ## Verify operator CRDs are managed from the image, .PHONY: test test: helm-unittest-install charts-prep verify-operator-chart-crds ## Run helm-unittest chart tests - @helm unittest . + @helm unittest . components/operator .PHONY: lint lint: charts-prep verify-operator-chart-crds ## Run helm lint diff --git a/deploy/helm/charts/platform/components/operator/.helmignore b/deploy/helm/charts/platform/components/operator/.helmignore index 0e8a0eb36f4c..fea6ca5f742d 100644 --- a/deploy/helm/charts/platform/components/operator/.helmignore +++ b/deploy/helm/charts/platform/components/operator/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# test files +tests/ diff --git a/deploy/helm/charts/platform/components/operator/templates/validate-values.yaml b/deploy/helm/charts/platform/components/operator/templates/validate-values.yaml index 1f551cdfc3e4..031c36b55d2d 100644 --- a/deploy/helm/charts/platform/components/operator/templates/validate-values.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/validate-values.yaml @@ -19,6 +19,25 @@ Each check detects an old value path and fails with a migration message. */}} +{{/* + LPX requires Grove integration. Only a parent chart can install Grove; + a standalone operator chart must explicitly enable an external Grove. +*/}} + +{{- if and .Values.lpx.enabled (not (and + .Values.global + .Values.global.grove + (or + .Values.global.grove.enabled + (and (not .Chart.IsRoot) .Values.global.grove.install) + ) +)) }} + {{- fail ` +CONFIGURATION ERROR: LPX requires Grove integration to be enabled. +Set global.grove.enabled=true to use an externally installed Grove operator. +` }} +{{- end }} + {{/* ---- NATS toggle (moved to global.nats.install) ---- */}} {{- if and (hasKey .Values "nats") (hasKey .Values.nats "enabled") }} diff --git a/deploy/helm/charts/platform/components/operator/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/components/operator/tests/lpx_config_test.yaml new file mode 100644 index 000000000000..0a95a95fa85e --- /dev/null +++ b/deploy/helm/charts/platform/components/operator/tests/lpx_config_test.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: standalone LPX and Grove configuration +templates: + - templates/operator-config.yaml + - templates/validate-values.yaml +tests: + - it: leaves LPX disabled when Grove is not configured + template: templates/operator-config.yaml + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpx:\n enabled: false$' + + - it: rejects LPX when global values are absent + template: templates/validate-values.yaml + set: + lpx.enabled: true + asserts: + - failedTemplate: + errorPattern: "LPX requires Grove integration to be enabled" + + - it: rejects LPX when Grove integration is explicitly disabled + template: templates/validate-values.yaml + set: + lpx.enabled: true + global.grove.enabled: false + global.grove.install: false + asserts: + - failedTemplate: + errorPattern: "LPX requires Grove integration to be enabled" + + - it: enables LPX with externally installed Grove + template: templates/operator-config.yaml + set: + lpx.enabled: true + global.grove.enabled: true + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^lpx:\n enabled: true$' + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^ grove:\n enabled: true$' + + - it: rejects install-only Grove settings without a parent chart + template: templates/validate-values.yaml + set: + lpx.enabled: true + global.grove.enabled: false + global.grove.install: true + asserts: + - failedTemplate: + errorPattern: "LPX requires Grove integration to be enabled" diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index 1bbaf7af5d34..2886c4dc30e8 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -3,10 +3,28 @@ suite: LPX operator configuration and RBAC templates: + - templates/validate-values.yaml - charts/dynamo-operator/templates/operator-config.yaml - charts/dynamo-operator/templates/manager-rbac.yaml - charts/dynamo-operator/templates/validate-values.yaml tests: + - it: preserves the bundled Grove compatibility error + template: templates/validate-values.yaml + set: + global.grove.install: true + dynamo-operator.lpx.enabled: true + asserts: + - failedTemplate: + errorPattern: "bundled Grove v0.1.0-alpha.13 does not support" + + - it: rejects LPX without Grove integration + template: charts/dynamo-operator/templates/validate-values.yaml + set: + dynamo-operator.lpx.enabled: true + asserts: + - failedTemplate: + errorPattern: "LPX requires Grove integration to be enabled" + - it: keeps LPX disabled by default template: charts/dynamo-operator/templates/operator-config.yaml asserts: @@ -14,18 +32,23 @@ tests: path: data["config.yaml"] pattern: '(?m)^lpx:\n enabled: false$' - - it: renders only Dynamo operator LPX configuration when enabled + - it: enables LPX and externally installed Grove together template: charts/dynamo-operator/templates/operator-config.yaml set: + global.grove.enabled: true dynamo-operator.lpx.enabled: true asserts: - matchRegex: path: data["config.yaml"] pattern: '(?m)^lpx:\n enabled: true$' + - matchRegex: + path: data["config.yaml"] + pattern: '(?m)^ grove:\n enabled: true$' - it: renders canonical LPX model registry configuration template: charts/dynamo-operator/templates/operator-config.yaml set: + global.grove.enabled: true dynamo-operator.lpx.enabled: true dynamo-operator.lpx.modelRegistryURL: gs://models/canonical asserts: diff --git a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml index b7ccc00058bc..51662fcc3895 100644 --- a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml +++ b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml @@ -97,22 +97,6 @@ tests: asserts: - notFailedTemplate: {} - - it: should reject bundled Grove with LPX - set: - global.grove.install: true - dynamo-operator.lpx.enabled: true - asserts: - - failedTemplate: - errorPattern: "bundled Grove v0.1.0-alpha.13 does not support" - - - it: should allow externally installed scheduler-scoped Grove with LPX - set: - global.grove.install: false - global.grove.enabled: true - dynamo-operator.lpx.enabled: true - asserts: - - notFailedTemplate: {} - # --- Default values should render cleanly --- - it: should render successfully with default values From b95844706f91cb38c21938996a97076cc3c128a1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:50:27 +0300 Subject: [PATCH 028/196] fix(operator): validate LPX ConfigMap size (#447) Signed-off-by: Andrew Paprotsky (cherry picked from commit 9497019a4b338e2beedf187cb181b15c847ab41a) Signed-off-by: Andrew Paprotsky --- .../dynamo/lpx/hybrid_cyborg_bindings.go | 14 +++------ .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 8 +++++ .../internal/dynamo/lpx/lpu_configmap.go | 21 ++++++++++++- .../internal/dynamo/lpx/lpu_configmap_test.go | 31 +++++++++++++++++++ 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 1e16c71ac266..693a48757d67 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -14,7 +14,6 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -143,13 +142,8 @@ func (w *SelectedWorkload) RenderCyborgConfigMap( offset += partition.effectiveNodeCount() } - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - }, - Immutable: ptr.To(true), - Data: map[string]string{"tokenizer_dir": tokenizerDir, "lpu_servers": strings.Join(servers, "\n")}, - } - configMap.Name = boundedAuxiliaryName(root, fmt.Sprintf("-decode-%.16s", LPUConfigMapHash(configMap))) - return configMap, nil + return renderRuntimeConfigMap(namespace, root+"-decode", map[string]string{ + "tokenizer_dir": tokenizerDir, + "lpu_servers": strings.Join(servers, "\n"), + }) } diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 8a8181eaeb8c..10f2d30702d6 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -9,10 +9,12 @@ import ( "encoding/json" "path/filepath" "strconv" + "strings" "testing" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" ) func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { @@ -70,6 +72,12 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { require.Equal(t, filepath.Join(runtimePath, "metadata/tokenizer"), configMap.Data["tokenizer_dir"]) require.NotEqual(t, initial.Name, configMap.Name) + t.Log("Reject oversized rendered Cyborg configuration before publication") + projection.configuredBuild.RuntimeTokenizerPath = strings.Repeat("x", corev1.MaxSecretSize) + _, err = workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + require.ErrorContains(t, err, "rendered LPX ConfigMap") + require.ErrorContains(t, err, "maximum is 1048576") + t.Log("Render a Cap'n Proto build without tokenizer metadata") projection.configuredBuild.RuntimeTokenizerPath = "" projection.configuredBuild.Path = "relative/build" diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 18523ff154d7..89fdaa35176f 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -97,6 +97,11 @@ func renderLPUConfigMap( } data["model_config.toml"] = modelTOML.String() data["datacenter.toml"] = datacenterTOML.String() + return renderRuntimeConfigMap(namespace, materializationName+"-lpu", data) +} + +func renderRuntimeConfigMap(namespace, namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { + // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -104,7 +109,21 @@ func renderLPUConfigMap( Immutable: ptr.To(true), Data: data, } - configMap.Name = LPUConfigMapName(materializationName, LPUConfigMapHash(configMap)) + configMap.Name = boundedAuxiliaryName(namePrefix, fmt.Sprintf("-%.16s", LPUConfigMapHash(configMap))) + + // Reject oversized configuration before any caller can publish it. + totalSize := 0 + for _, value := range data { + totalSize += len(value) + } + if totalSize > corev1.MaxSecretSize { + return nil, fmt.Errorf( + "rendered LPX ConfigMap %q data is %d bytes; maximum is %d", + configMap.Name, + totalSize, + corev1.MaxSecretSize, + ) + } return configMap, nil } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 86a833e5d810..3b92a586b198 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -11,9 +11,27 @@ import ( "testing" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation" ) +func TestRenderRuntimeConfigMapSizeLimit(t *testing.T) { + t.Log("Accept exactly 1 MiB of UTF-8 bytes across data values, excluding keys") + data := map[string]string{ + "text": strings.Repeat("é", corev1.MaxSecretSize/2-1), + "suffix": "é", + } + _, err := renderRuntimeConfigMap("test-namespace", "test-lpu", data) + require.NoError(t, err) + + t.Log("Reject one extra byte with a named error and no usable ConfigMap") + data["suffix"] += "x" + configMap, err := renderRuntimeConfigMap("test-namespace", "test-lpu", data) + require.Nil(t, configMap) + require.ErrorContains(t, err, `rendered LPX ConfigMap "test-lpu-`) + require.ErrorContains(t, err, "data is 1048577 bytes; maximum is 1048576") +} + func TestLPXAuxiliaryNamesAreBounded(t *testing.T) { t.Log("Use a source name already at the Kubernetes length limit") root := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) @@ -98,6 +116,19 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { require.True(t, *configMap.Immutable) require.Equal(t, LPUConfigMapName("test-dgd", LPUConfigMapHash(configMap)), configMap.Name) + t.Log("Reject an LPX setting that makes the rendered ConfigMap exceed the API limit") + projection.configuredBuild.runtimeSettings["oversized"] = strings.Repeat("x", corev1.MaxSecretSize) + _, err = renderLPUConfigMap( + "test-namespace", + "test-dgd", + "/models", + []*ModelProjection{projection}, + nil, + ) + require.ErrorContains(t, err, "rendered LPX ConfigMap") + require.ErrorContains(t, err, "maximum is 1048576") + delete(projection.configuredBuild.runtimeSettings, "oversized") + t.Log("Render the preserved runtime with an invalid snapshot build reference") projection.configuredBuild.Path = "relative-build" _, err = renderLPUConfigMap( From 162fe39301dbff04797852cdad81c6dcbe201d8d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:30:52 +0300 Subject: [PATCH 029/196] fix(operator): use fixed LPX SSH volume name (#448) Signed-off-by: Andrew Paprotsky (cherry picked from commit db335a18b4bbdd8a18c18b8bb3c4f318f1e408df) --- .../internal/dynamo/lpx/pod_roles_test.go | 30 ++++++++ .../internal/dynamo/lpx/runtime_agent.go | 25 +++---- .../internal/dynamo/lpx/runtime_ssh.go | 33 ++++++--- .../internal/dynamo/lpx/runtime_ssh_test.go | 68 +++++++++++++++++++ .../node-local-v3-hx-lpu-only.yaml | 8 +-- .../node-local-v3-hx-specdecode.yaml | 16 ++--- 6 files changed, 144 insertions(+), 36 deletions(-) create mode 100644 deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index b177cc4dd4cc..1a9bb07cd333 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -366,6 +366,36 @@ func TestConfigureNodeLocalXTConductorSSHInitUsesMainImage(t *testing.T) { require.Equal(t, corev1.PullIfNotPresent, podSpec.InitContainers[0].ImagePullPolicy) } +func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { + const sshSecretName = "mpi.ssh" + + t.Log("Configure each XT runtime role with a valid dotted SSH Secret name") + base := corev1.PodSpec{Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}} + configureAgentScheduling(&base, BuildFamilyXT) + direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() + require.NoError(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", sshSecretName)) + require.NoError(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, sshSecretName)) + require.NoError(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", sshSecretName)) + + for _, test := range []struct { + name string + pod *corev1.PodSpec + mounts []corev1.VolumeMount + }{ + {"direct hybrid agent", direct, direct.Containers[0].VolumeMounts}, + {"node-local agent", agent, agent.Containers[0].VolumeMounts}, + {"node-local conductor", conductor, conductor.InitContainers[0].VolumeMounts}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Keep the configured Secret separate from the fixed volume and mount name") + sshVolume := testVolumeByName(t, test.pod.Volumes, "ssh-secret") + require.Equal(t, sshSecretName, sshVolume.Secret.SecretName) + require.NotContains(t, testVolumeNames(test.pod.Volumes), sshSecretName) + require.Contains(t, test.mounts, corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}) + }) + } +} + func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T) { t.Log("Define custom entrypoint and probe preservation cases") customStartup := testExecProbe("custom-startup") diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index b8c0be7346b4..fa42cb050f55 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -19,7 +19,6 @@ import ( const ( runtimeTemporaryStorageVolumeName = "tmp" runtimeTemporaryStorageMountPath = "/tmp" - runtimeSSHVolumeName = "agent-v2-ssh" lp30InitContainerName = "prepare-lp30" lp30InitPath = "/bin/lp30-agent-v2-init" ) @@ -141,7 +140,7 @@ func configureDirectHybridAgentRuntime( ) addLPUHostDeviceVolumeMounts(agent) - agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount(sshSecretName)) + agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount()) retargetMainContainerReferences(agentPodSpec, agent) agent.Env = append(agent.Env, @@ -167,7 +166,9 @@ func configureDirectHybridAgentRuntime( RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), } - addSSHVolumeWithDefaultMode(agentPodSpec, sshSecretName, sshSecretName, 0644) + if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { + return err + } applyLPUHostDeviceVolumes(agentPodSpec, false) agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ Name: "hugepages", @@ -207,12 +208,7 @@ func configureNodeLocalConductorRuntime( conductor.Args = append(conductor.Args, nodeLocalHXAgentEnvironmentArgs...) } retargetMainContainerReferences(conductorPodSpec, conductor) - sshVolumeName := runtimeSSHVolumeName - if isXT { - sshVolumeName = sshSecretName - } - addConductorSSHKey(conductorPodSpec, conductor, sshSecretName, sshVolumeName) - return nil + return addConductorSSHKey(conductorPodSpec, conductor, sshSecretName) } // configureNodeLocalAgentRuntime consumes a nonnil Agent PodSpec. @@ -239,11 +235,6 @@ func configureNodeLocalAgentRuntime( return fmt.Errorf("node-local LPU runtime requires an image-only main container without command") } preserveAgentEntrypoint := isXT && (len(agent.Command) != 0 || len(agent.Args) != 0) - sshVolumeName := runtimeSSHVolumeName - if isXT { - sshVolumeName = sshSecretName - } - // Materialize the LPU role as the privileged SSH target that Nova launches. setNodeLocalPodIPEnv(agent, isXT) agent.Name = lpuAgentContainerName @@ -253,11 +244,13 @@ func configureNodeLocalAgentRuntime( agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) applyLPUHostDeviceVolumes(agentPodSpec, !isXT) - addSSHVolumeWithDefaultMode(agentPodSpec, sshSecretName, sshVolumeName, 0644) + if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { + return err + } // Expose node-local devices and hugetlbfs through host /dev without an unaccounted HugePages volume. addLPUHostDeviceVolumeMounts(agent) - agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount(sshVolumeName)) + agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount()) if isXT { applyLPUWorkerContainerBase(agent, preserveAgentEntrypoint) agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index 37140327dd3f..c435cba1d37e 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -6,11 +6,14 @@ package lpx import ( + "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" ) const ( + runtimeSSHVolumeName = "ssh-secret" runtimeSSHSecretMountPath = "/ssh-pk" runtimeSSHSecretPrivateKeyPath = runtimeSSHSecretMountPath + "/private.key" conductorSSHKeyDir = runtimeTemporaryStorageMountPath + "/dynamo-lpu-ssh" @@ -21,9 +24,18 @@ const ( conductorSSHPrivateKeyPath + "; chmod 600 " + conductorSSHPrivateKeyPath ) -func addSSHVolumeWithDefaultMode(podSpec *corev1.PodSpec, secretName, volumeName string, defaultMode int32) { +func addSSHVolume(podSpec *corev1.PodSpec, secretName string, defaultMode int32) error { + // Preserve authored storage and key projections instead of replacing their contents. + for _, volume := range podSpec.Volumes { + if volume.Name == runtimeSSHVolumeName && (volume.Secret == nil || + volume.Secret.SecretName != secretName || len(volume.Secret.Items) != 0) { + return fmt.Errorf("selected LPX podTemplate volume %q is reserved for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) + } + } + + // Apply the runtime's key permissions to its dedicated Secret volume. sshVolume := corev1.Volume{ - Name: volumeName, + Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: secretName, @@ -32,19 +44,24 @@ func addSSHVolumeWithDefaultMode(podSpec *corev1.PodSpec, secretName, volumeName }, } podSpec.Volumes = setVolumeByName(podSpec.Volumes, sshVolume) + return nil } -func sshVolumeMount(volumeName string) corev1.VolumeMount { +func sshVolumeMount() corev1.VolumeMount { return corev1.VolumeMount{ - Name: volumeName, + Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true, } } -func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName, volumeName string) { +func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName string) error { // Mount the source Secret and the writable destination used by OpenMPI. - addSSHVolumeWithDefaultMode(podSpec, secretName, volumeName, 0600) + if err := addSSHVolume(podSpec, secretName, 0600); err != nil { + return err + } + + // Keep the copied private key in a Pod-local writable volume. podSpec.Volumes = setVolumeByName(podSpec.Volumes, corev1.Volume{ Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{ @@ -64,10 +81,10 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), }, - VolumeMounts: []corev1.VolumeMount{conductorSSHKeyVolumeMount()}, + VolumeMounts: []corev1.VolumeMount{conductorSSHKeyVolumeMount(), sshVolumeMount()}, } - initContainer.VolumeMounts = setVolumeMount(initContainer.VolumeMounts, sshVolumeMount(volumeName)) podSpec.InitContainers = setContainerByName(podSpec.InitContainers, initContainer) + return nil } func conductorSSHKeyVolumeMount() corev1.VolumeMount { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go new file mode 100644 index 000000000000..e0e7493ef248 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +func TestAddSSHVolumePreservesAuthoredStorage(t *testing.T) { + for _, test := range []struct { + name string + source corev1.VolumeSource + wantErr bool + }{ + {"matching Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "mpi.ssh"}}, false}, + {"persistent data", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}, true}, + {"different Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}, true}, + {"restricted keys", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: "mpi.ssh", Items: []corev1.KeyToPath{{Key: "public.key", Path: "public.key"}}, + }}, true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Bind the authored storage to the runtime SSH volume name") + pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: test.source}}} + before := pod.DeepCopy() + + t.Log("Install the MPI key only when the existing Secret binding is compatible") + err := addSSHVolume(&pod, "mpi.ssh", 0644) + if test.wantErr { + require.ErrorContains(t, err, `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"`) + require.Equal(t, *before, pod) + return + } + require.NoError(t, err) + require.Len(t, pod.Volumes, 1) + require.Equal(t, &corev1.SecretVolumeSource{SecretName: "mpi.ssh", DefaultMode: ptr.To(int32(0644))}, pod.Volumes[0].Secret) + }) + } +} + +func TestXTRuntimesRejectSSHVolumeCollision(t *testing.T) { + t.Log("Start each XT role with an authored volume using the fixed SSH name") + base := corev1.PodSpec{ + Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, + Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + } + configureAgentScheduling(&base, BuildFamilyXT) + direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() + + t.Log("Propagate the collision through every XT runtime configuration path") + const wantError = `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"` + require.ErrorContains(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", "mpi.ssh"), wantError) + require.ErrorContains(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "mpi.ssh"), wantError) + require.ErrorContains(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", "mpi.ssh"), wantError) + + t.Log("Preserve the authored volume in every rejected runtime") + for _, pod := range []*corev1.PodSpec{direct, agent, conductor} { + require.Equal(t, base.Volumes[0].VolumeSource, testVolumeByName(t, pod.Volumes, "ssh-secret")) + } +} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 49d14c08cb96..c97b7f142938 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -263,7 +263,7 @@ spec: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true restartPolicy: Always schedulerName: default-scheduler @@ -284,7 +284,7 @@ spec: name: config - emptyDir: {} name: tmp - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret @@ -459,7 +459,7 @@ spec: - mountPath: /dev name: host-dev - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true dnsPolicy: ClusterFirstWithHostNet hostIPC: true @@ -519,7 +519,7 @@ spec: path: /sys type: Directory name: host-sys - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 420 secretName: ssh-secret diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 440239bc863b..1a571d96eddd 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -319,7 +319,7 @@ spec: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true restartPolicy: Always schedulerName: default-scheduler @@ -340,7 +340,7 @@ spec: name: config - emptyDir: {} name: tmp - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret @@ -519,7 +519,7 @@ spec: - mountPath: /dev name: host-dev - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true dnsPolicy: ClusterFirstWithHostNet hostIPC: true @@ -579,7 +579,7 @@ spec: path: /sys type: Directory name: host-sys - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 420 secretName: ssh-secret @@ -752,7 +752,7 @@ spec: - mountPath: /dev name: host-dev - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true dnsPolicy: ClusterFirstWithHostNet hostIPC: true @@ -812,7 +812,7 @@ spec: path: /sys type: Directory name: host-sys - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 420 secretName: ssh-secret @@ -985,7 +985,7 @@ spec: - mountPath: /dev name: host-dev - mountPath: /ssh-pk - name: agent-v2-ssh + name: ssh-secret readOnly: true dnsPolicy: ClusterFirstWithHostNet hostIPC: true @@ -1045,7 +1045,7 @@ spec: path: /sys type: Directory name: host-sys - - name: agent-v2-ssh + - name: ssh-secret secret: defaultMode: 420 secretName: ssh-secret From 113a2ae2303fa1729ab38838a5b607bcfdb4282d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:29 +0300 Subject: [PATCH 030/196] fix(operator): report LPX component runtime namespaces (#450) Signed-off-by: Andrew Paprotsky (cherry picked from commit 21548b8a755c6e200b922fe1d05e4dcfc48d6839) --- deploy/operator/internal/dynamo/grove_lpx.go | 2 +- .../internal/dynamo/grove_lpx_test.go | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 84e57a896ada..3d2c6efc85a4 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -69,7 +69,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source for _, member := range lpx.Components(source) { if member.ComponentName != component.ComponentName { statuses[member.ComponentName] = v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodClique, RuntimeNamespace: status.RuntimeNamespace, + ComponentKind: v1beta1.ComponentKindPodClique, RuntimeNamespace: source.GetDynamoNamespaceForComponent(member), } } } diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go index cac4eba15c0b..550eac707933 100644 --- a/deploy/operator/internal/dynamo/grove_lpx_test.go +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -6,6 +6,8 @@ package dynamo import ( "testing" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -18,6 +20,35 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T) { + draft := v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "draft", + ComponentType: v1beta1.ComponentTypeLPX, + GlobalDynamoNamespace: true, + LPX: &v1beta1.LPXConfig{BuildID: "draft-build"}, + Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent}}, + } + serving := v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "serving", + ComponentType: v1beta1.ComponentTypeLPX, + LPX: &v1beta1.LPXConfig{BuildID: "serving-build"}, + Roles: []v1beta1.ComponentRoleSpec{ + {Name: v1beta1.ComponentRoleLPXConductor}, + {Name: v1beta1.ComponentRoleLPXAgent}, + }, + } + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "graph", Namespace: "tenant"}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{draft, serving}}, + } + + readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, &v1alpha1.LPXGraphDeployment{}, nil) + require.NoError(t, err) + require.NotEqual(t, source.GetDynamoNamespaceForComponent(&draft), source.GetDynamoNamespaceForComponent(&serving)) + require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) + require.Equal(t, source.GetDynamoNamespaceForComponent(&serving), readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) +} + func TestResolveLPXGPUShape(t *testing.T) { for _, test := range []struct { name string From 0b2460dc305cdb4a563578ddb8a223d64e1fba92 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:29 +0300 Subject: [PATCH 031/196] fix(operator): bound LPX metadata snapshot RPCs (#455) Signed-off-by: Andrew Paprotsky (cherry picked from commit 4f18ea709973184252746469a2f0890c4a675506) --- .../internal/dynamo/lpx/build_snapshot.go | 6 ++++ .../dynamo/lpx/build_snapshot_gcs_test.go | 35 +++++++++++++++++-- .../internal/dynamo/lpx/model_fixture_test.go | 6 +++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot.go b/deploy/operator/internal/dynamo/lpx/build_snapshot.go index 6a4b5b5b2533..84cf45dd40fa 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot.go @@ -18,6 +18,7 @@ import ( "strconv" "strings" "syscall" + "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -38,6 +39,7 @@ const ( // metadata budget instead of permitting an unbounded sum of streamed // chunks or local files. maxBuildSnapshotMetadataBytes = modelExpressMaxMessageSize + buildSnapshotTimeout = 30 * time.Second ) // BuildSnapshot is a provider-fenced copy of the exact manifest consumed by @@ -51,6 +53,10 @@ type BuildSnapshot struct { // AcquireBuildSnapshot fences the required manifest-v2 compiler metadata with inventories // and duplicate reads. The receiver must be non-nil and is not mutated. func (r *ModelRegistry) AcquireBuildSnapshot(ctx context.Context, id string) (*BuildSnapshot, error) { + // Bound all metadata RPCs for one snapshot acquisition. + ctx, cancel := context.WithTimeout(ctx, buildSnapshotTimeout) + defer cancel() + refURL, err := r.BuildURL(id) if err != nil { return nil, err diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go index 0256ee3eecf8..84f25828f909 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go @@ -6,8 +6,10 @@ package lpx import ( + "context" "errors" "testing" + "time" modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" "github.com/stretchr/testify/require" @@ -89,8 +91,17 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { t.Log("Serve a valid revision-2 compiler manifest through Model Express") payload := manifestV2Payload(t) - for _, ref := range []string{"model/build", "gs://bucket/registry/model/build"} { - t.Run(ref, func(t *testing.T) { + shortCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + for _, testCase := range []struct { + ref string + ctx context.Context + }{ + {ref: "model/build", ctx: t.Context()}, + {ref: "gs://bucket/registry/model/build", ctx: t.Context()}, + {ref: "model/build", ctx: shortCtx}, + } { + t.Run(testCase.ref, func(t *testing.T) { t.Log("Create an independent client and registry for this reference form") client := &fakeModelServiceClient{ fileStreams: []*fakeModelFileStream{ @@ -103,7 +114,8 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { require.NoError(t, err) t.Log("Normalize the manifest and retain the current registry locator") - build, err := normalizeRegistryFixtureBuild(t.Context(), registry, ref) + started := time.Now() + build, err := normalizeRegistryFixtureBuild(testCase.ctx, registry, testCase.ref) require.NoError(t, err) require.Equal(t, "gs://bucket/registry/model/build", build.Path) require.Equal(t, 8, build.BatchSize) @@ -114,6 +126,23 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { for _, request := range client.filesRequests { require.Equal(t, []string{gbuildManifestV2CapnpFile}, request.GetFileSelector().GetPaths()) } + + t.Log("Share a finite deadline across all metadata RPCs and release only their contexts") + require.Len(t, client.metadataContexts, 4) + deadline, ok := client.metadataContexts[0].Deadline() + require.True(t, ok) + if parentDeadline, hasDeadline := testCase.ctx.Deadline(); hasDeadline { + require.Equal(t, parentDeadline, deadline) + } else { + require.WithinRange(t, deadline, started.Add(30*time.Second), time.Now().Add(30*time.Second)) + } + for _, rpcCtx := range client.metadataContexts { + rpcDeadline, hasDeadline := rpcCtx.Deadline() + require.True(t, hasDeadline) + require.Equal(t, deadline, rpcDeadline) + require.ErrorIs(t, rpcCtx.Err(), context.Canceled) + } + require.NoError(t, testCase.ctx.Err()) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go index 7f226e08969d..2855820be855 100644 --- a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go @@ -45,6 +45,8 @@ type fakeModelServiceClient struct { listRequests []*modelpb.ModelFilesRequest list *modelpb.ModelFileList listErr error + + metadataContexts []context.Context } func (c *fakeModelServiceClient) EnsureModelDownloaded( @@ -65,6 +67,7 @@ func (c *fakeModelServiceClient) StreamModelFiles( _ ...grpc.CallOption, ) (grpc.ServerStreamingClient[modelpb.FileChunk], error) { c.filesRequests = append(c.filesRequests, request) + c.metadataContexts = append(c.metadataContexts, ctx) c.filesContext = ctx if c.filesErr != nil { return nil, c.filesErr @@ -75,11 +78,12 @@ func (c *fakeModelServiceClient) StreamModelFiles( } func (c *fakeModelServiceClient) ListModelFiles( - _ context.Context, + ctx context.Context, request *modelpb.ModelFilesRequest, _ ...grpc.CallOption, ) (*modelpb.ModelFileList, error) { c.listRequests = append(c.listRequests, request) + c.metadataContexts = append(c.metadataContexts, ctx) if c.listErr != nil { return nil, c.listErr } From 7d5c415528ca2d06d0b17adc71056b611945dcca Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:30 +0300 Subject: [PATCH 032/196] fix(operator): preserve direct LPX agent pod security (#457) Signed-off-by: Andrew Paprotsky (cherry picked from commit cd0e203c314e437b72895bbe0856e19336fec255) --- .../internal/dynamo/lpx/pod_roles_test.go | 34 ++++++++++++++++--- .../internal/dynamo/lpx/runtime_agent.go | 11 ++++-- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 3 ++ .../from_dgd_yaml/node-local-v2-hybrid.yaml | 3 ++ .../node-local-v3-hx-hybrid.yaml | 3 ++ 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 1a9bb07cd333..6b6e363f8d8c 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -396,13 +396,14 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { } } -func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T) { - t.Log("Define custom entrypoint and probe preservation cases") +func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { + t.Log("Define custom entrypoint, probe and Pod security preservation cases") customStartup := testExecProbe("custom-startup") customReadiness := testExecProbe("custom-ready") tests := []struct { name string container corev1.Container + podSecurity *corev1.PodSecurityContext wantCommand []string wantArgs []string wantStartup *corev1.Probe @@ -417,8 +418,18 @@ func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T wantReadiness: lpuV2ReadinessProbe(true), }, { - name: "command", - container: corev1.Container{Name: commonconsts.MainContainerName, Command: []string{"custom-entrypoint"}}, + name: "command", + container: corev1.Container{Name: commonconsts.MainContainerName, Command: []string{"custom-entrypoint"}}, + podSecurity: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(1000)), + RunAsGroup: ptr.To(int64(1000)), + RunAsNonRoot: ptr.To(true), + FSGroup: ptr.To(int64(1000)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + SupplementalGroups: []int64{2000}, + Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, wantCommand: []string{"custom-entrypoint"}, wantStartup: lpuV2StartupProbe(true), wantReadiness: lpuV2ReadinessProbe(true), @@ -441,7 +452,10 @@ func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Log("Configure the direct LPU agent runtime from a user-owned entrypoint") - podSpec := corev1.PodSpec{Containers: []corev1.Container{*test.container.DeepCopy()}} + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{*test.container.DeepCopy()}, + SecurityContext: test.podSecurity.DeepCopy(), + } configureAgentScheduling(&podSpec, BuildFamilyXT) err := configureDirectHybridAgentRuntime(&podSpec, "graph-lpu", "ssh-secret") require.NoError(t, err) @@ -452,6 +466,16 @@ func TestConfigureDirectHybridAgentRuntimePreservesCustomEntrypoint(t *testing.T require.Equal(t, test.wantArgs, agent.Args) require.Equal(t, test.wantStartup, agent.StartupProbe) require.Equal(t, test.wantReadiness, agent.ReadinessProbe) + + t.Log("Preserve orthogonal Pod security settings while using the root runtime identity") + wantSecurity := test.podSecurity.DeepCopy() + if wantSecurity == nil { + wantSecurity = &corev1.PodSecurityContext{} + } + wantSecurity.RunAsUser = ptr.To(int64(0)) + wantSecurity.RunAsGroup = ptr.To(int64(0)) + wantSecurity.RunAsNonRoot = ptr.To(false) + require.Equal(t, wantSecurity, podSpec.SecurityContext) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index fa42cb050f55..760a213de10d 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -162,10 +162,15 @@ func configureDirectHybridAgentRuntime( setNodeLocalPodIPEnv(agent, true) addRuntimeTemporaryStorage(agentPodSpec, agent, false) updateWorkerPodSpec(agentPodSpec) - agentPodSpec.SecurityContext = &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(0)), - RunAsGroup: ptr.To(int64(0)), + + // Keep volume permissions and other Pod settings while forcing the Agent's root identity. + if agentPodSpec.SecurityContext == nil { + agentPodSpec.SecurityContext = &corev1.PodSecurityContext{} } + agentPodSpec.SecurityContext.RunAsUser = ptr.To(int64(0)) + agentPodSpec.SecurityContext.RunAsGroup = ptr.To(int64(0)) + agentPodSpec.SecurityContext.RunAsNonRoot = ptr.To(false) + if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { return err } diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 6d295d912150..e12bfe489032 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -591,7 +591,10 @@ spec: restartPolicy: Always schedulerName: lpx-scheduler securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch runAsGroup: 0 + runAsNonRoot: false runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 2dcf9f9e1033..8737b165090e 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -380,7 +380,10 @@ spec: restartPolicy: Always schedulerName: lpx-scheduler securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch runAsGroup: 0 + runAsNonRoot: false runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index b3b0882190ad..79e024cbf2fb 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -534,7 +534,10 @@ spec: restartPolicy: Always schedulerName: lpx-scheduler securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch runAsGroup: 0 + runAsNonRoot: false runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: From 9260de61b6594504386a6940cbd407b185e2e113 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:30 +0300 Subject: [PATCH 033/196] fix(operator): reject LPX init container name collisions (#458) Signed-off-by: Andrew Paprotsky (cherry picked from commit 85d1929dd663de99a2fbd2c0dcba6d276dbe1b53) --- .../internal/dynamo/lpx/pod_mutation.go | 6 --- .../internal/dynamo/lpx/pod_roles_test.go | 47 +++++++++++++++++++ .../internal/dynamo/lpx/runtime_agent.go | 13 +++-- .../internal/dynamo/lpx/runtime_ssh.go | 8 +++- .../dynamo/lpx/selection_validation.go | 2 +- 5 files changed, 65 insertions(+), 11 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/pod_mutation.go b/deploy/operator/internal/dynamo/lpx/pod_mutation.go index 583538fef9f9..004730f089f5 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_mutation.go +++ b/deploy/operator/internal/dynamo/lpx/pod_mutation.go @@ -32,12 +32,6 @@ func setVolumeMount(volumeMounts []corev1.VolumeMount, volumeMount corev1.Volume }) } -func setContainerByName(containers []corev1.Container, container corev1.Container) []corev1.Container { - return replaceConflicts(containers, container, func(existing corev1.Container) bool { - return existing.Name == container.Name - }) -} - func replaceConflicts[T any](items []T, replacement T, conflicts func(T) bool) []T { // Preserve the first conflict's position while removing later conflicts, or append when none exists. first := slices.IndexFunc(items, conflicts) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 6b6e363f8d8c..8eb9bc771ea5 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -480,6 +480,53 @@ func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { } } +func TestLPXInitContainerNames(t *testing.T) { + t.Log("Reserve a generated name only in the Pod receiving that init container") + for _, test := range []struct { + name string + add func(*corev1.PodSpec, *corev1.Container) error + }{ + {name: "prepare-lp30", add: addLP30InitContainer}, + {name: "prepare-ssh-key", add: func(spec *corev1.PodSpec, main *corev1.Container) error { + return addConductorSSHKey(spec, main, "ssh-secret") + }}, + } { + for _, name := range []string{"prepare-lp30", "prepare-ssh-key"} { + for _, list := range []string{"containers", "initContainers"} { + t.Run(test.name+"/"+name+"/"+list, func(t *testing.T) { + t.Log("Author an independent container with an image and command to preserve") + spec := corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "runtime"}}} + authored := corev1.Container{Name: name, Image: "custom", Command: []string{"/custom-init"}} + if list == "containers" { + spec.Containers = append(spec.Containers, authored) + } else { + spec.InitContainers = []corev1.Container{authored} + } + before := spec.DeepCopy() + + t.Log("Reject only the actual generated name, before any mutation") + err := test.add(&spec, &spec.Containers[0]) + if name == test.name { + require.ErrorContains(t, err, list) + require.ErrorContains(t, err, name) + require.Equal(t, *before, spec) + return + } + + t.Log("Preserve a name belonging to the other role and add the intended init") + require.NoError(t, err) + require.Equal(t, before.Containers, spec.Containers) + require.Len(t, spec.InitContainers, len(before.InitContainers)+1) + for i, container := range before.InitContainers { + require.Equal(t, container, spec.InitContainers[i]) + } + require.Equal(t, test.name, spec.InitContainers[len(before.InitContainers)].Name) + }) + } + } + } +} + func TestConfigureNodeLocalLPURuntimeRolesRejectsUnsupportedShape(t *testing.T) { t.Log("Define unsupported node-local PodSpec shapes") tests := []struct { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 760a213de10d..dfd8027c0349 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -12,6 +12,7 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ) @@ -267,7 +268,7 @@ func configureNodeLocalAgentRuntime( }}, }) } else { - addLP30InitContainer(agentPodSpec, agent) + return addLP30InitContainer(agentPodSpec, agent) } return nil @@ -325,7 +326,12 @@ func configureNodeLocalAgentWorkerContainer( } } -func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) { +func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) error { + // Reserve this name only when the selected runtime actually adds the init container. + if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), lp30InitContainerName).ToAggregate(); err != nil { + return err + } + initContainer := corev1.Container{ Name: lp30InitContainerName, Image: agent.Image, @@ -345,7 +351,8 @@ func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) { }, } addLPUHostDeviceVolumeMounts(&initContainer) - podSpec.InitContainers = setContainerByName(podSpec.InitContainers, initContainer) + podSpec.InitContainers = append(podSpec.InitContainers, initContainer) + return nil } func setNodeLocalPodIPEnv(container *corev1.Container, isXT bool) { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index c435cba1d37e..62e9401c10ba 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -9,6 +9,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ) @@ -56,6 +57,11 @@ func sshVolumeMount() corev1.VolumeMount { } func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName string) error { + // Check both container lists before replacing any authored configuration. + if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), conductorSSHKeyInitContainerName).ToAggregate(); err != nil { + return err + } + // Mount the source Secret and the writable destination used by OpenMPI. if err := addSSHVolume(podSpec, secretName, 0600); err != nil { return err @@ -83,7 +89,7 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se }, VolumeMounts: []corev1.VolumeMount{conductorSSHKeyVolumeMount(), sshVolumeMount()}, } - podSpec.InitContainers = setContainerByName(podSpec.InitContainers, initContainer) + podSpec.InitContainers = append(podSpec.InitContainers, initContainer) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index 790715bc2c90..1ce092898239 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -168,7 +168,7 @@ func ValidateAgentContainerNames(dgd *dynamov1beta1.DynamoGraphDeployment) field // validateRolePodSpecContainerNames checks both lists that share the Pod's name space. // spec and fldPath must be non-nil. func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path, reservedName string) field.ErrorList { - // Regular and init containers must not collide with the renamed main container. + // Both lists must avoid renamed main and generated init-container names. allErrs := field.ErrorList{} for _, group := range []struct { name string From fd499e4efc9de3fb4e97a3cc1114801a527b70de Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:30 +0300 Subject: [PATCH 034/196] test(operator): cover ordinary Grove PCS identity (#459) Signed-off-by: Andrew Paprotsky (cherry picked from commit 66bfbc91d3863e4aa209de87d4061d0060d9cb02) --- .../controller/dgd_lpx_handoff_test.go | 105 +++++++++++++++--- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index e9a82f8c8bce..af823ee75d24 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "testing" "time" @@ -19,6 +20,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" "github.com/stretchr/testify/require" @@ -77,22 +79,97 @@ func projectLPXChildStatus( } func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { - t.Log("Project the exact DGD-owned component subset from a hybrid graph") - source := newLPXHandoffSource(t, "node-local-v2-hybrid") - source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "prefill", - ComponentType: v1beta1.ComponentTypePrefill, - }) - before := source.DeepCopy() - ordinary := projectOrdinaryGroveDeployment(source) + for _, test := range []struct { + name string + longNames bool + lpxOnly bool + }{ + {name: "mixed"}, + {name: "mixed long names", longNames: true}, + {name: "LPX only", lpxOnly: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Project the exact DGD-owned subset without changing the source") + source := newLPXHandoffSource(t, "node-local-v2-hybrid") + lpxComponent := &source.Spec.Components[0] + if test.longNames { + source.Name = strings.Repeat("graph-", 10) + "a" + lpxComponent.ComponentName = strings.Repeat("l", 30) + } + lpxName := lpxComponent.ComponentName + if !test.lpxOnly { + source.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "test-topology", PackDomain: "rack"} + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "ordinary-frontend", ComponentType: v1beta1.ComponentTypeFrontend, + Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main", Image: "frontend"}}, + }}, + }) + } + before := source.DeepCopy() + ordinary := projectOrdinaryGroveDeployment(source) + require.Nil(t, ordinary.GetComponentByName(lpxName)) + require.Len(t, ordinary.Spec.Components, len(source.Spec.Components)-1) + + t.Log("Seed only the projected PCS identity, including the truncated-name case") + pcsName := dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components) + if test.longNames { + require.NotEqual(t, source.Name, pcsName) + require.NotEqual(t, dynamo.PCSNameForDGD(source.Name, source.Spec.Components), pcsName) + } + pcs := &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{Name: pcsName, Namespace: source.Namespace, Generation: 1}, + Status: grovev1alpha1.PodCliqueSetStatus{ObservedGeneration: ptr.To(int64(1)), Conditions: []metav1.Condition{{ + Type: groveconstants.ConditionTopologyLevelsUnavailable, Status: metav1.ConditionTrue, + Reason: groveconstants.ConditionReasonClusterTopologyNotFound, Message: "missing ordinary topology", + }}}, + } + clique := &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(ordinary, "ordinary-frontend"), Namespace: source.Namespace, Generation: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, + } + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(pcs, clique).Build() - require.Equal(t, before, source) - require.Nil(t, ordinary.GetComponentByName("lpx")) - require.Len(t, ordinary.Spec.Components, len(source.Spec.Components)-1) + t.Log("Render and observe restarts against that same ordinary PCS") + renderer := newGroveWorkloadRenderer(kube, &configv1alpha1.OperatorConfiguration{}, &commoncontroller.RuntimeConfig{}, nil) + rendered, err := renderer.Render(t.Context(), source, ordinary, nil, nil, false) + require.NoError(t, err) + require.NotNil(t, rendered.existing) + require.Equal(t, pcsName, rendered.existing.Name) + if test.lpxOnly { + require.Nil(t, rendered.desired) + } else { + require.Equal(t, pcsName, rendered.desired.Name) + } + remaining := resolveCompositeGroveRestartProgress(t.Context(), source, ordinary, + []string{lpxName, "ordinary-frontend"}, newGroveRestartProgressResolver(kube), newLPXRestartProgressResolver(kube)) + require.Equal(t, []string{lpxName}, remaining) + + t.Log("Observe the graph-level topology through the same ordinary PCS") + result := newWorkloadProgramResult(source) + newDGDGroveTopologyConditionReconciler(kube).Reconcile(t.Context(), ordinary, &result) + condition := meta.FindStatusCondition(result.Status.Conditions, v1beta1.ConditionTypeTopologyLevelsAvailable) + if test.lpxOnly { + require.Nil(t, condition) + require.Empty(t, result.Events) + } else { + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, v1beta1.ConditionReasonTopologyDefinitionNotFound, condition.Reason) + require.Equal(t, "missing ordinary topology", condition.Message) + } - t.Log("Keep the projected value independently mutable by later Grove preparation") - ordinary.Spec.Components[0].ComponentName = "ordinary-copy" - require.Equal(t, before, source) + t.Log("Keep the projected copy independently mutable after all observations") + require.Equal(t, before, source) + ordinary.Annotations["projection-only"] = "independent-copy" + if len(ordinary.Spec.Components) > 0 { + ordinary.Spec.Components[0].ComponentName = "ordinary-copy" + } + require.Equal(t, before, source) + }) + } } func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { From 2e24946efcc2b402e8792c1598913d3e804eb57c Mon Sep 17 00:00:00 2001 From: kathleen french Date: Fri, 11 Sep 2026 11:00:53 -0400 Subject: [PATCH 035/196] fix(operator): refresh LPX APIs and eviction identity (#454) * fix(operator): refresh LPX APIs and eviction identity Signed-off-by: kathleen french * docs(operator): remove vendored LPX API readme Signed-off-by: kathleen french * fix(operator): authenticate eviction runtime config Signed-off-by: kathleen french --------- Signed-off-by: kathleen french (cherry picked from commit 412e83593e1d3ea9893294944f1da1ab4f084113) Signed-off-by: Andrew Paprotsky --- deploy/operator/Makefile | 6 +- deploy/operator/api/scripts/lpx-crd-schema.yq | 21 +- .../nvidia.com_dynamographdeployments.yaml | 12419 +++++++++++++++- .../internal/controller/lpx/eviction.go | 63 +- .../internal/controller/lpx/eviction_test.go | 231 +- .../internal/dynamo/lpx/lpu_partition.go | 65 + .../internal/dynamo/lpx/lpu_partition_test.go | 120 + .../thirdparty/lpxscheduler/v1alpha1/doc.go | 4 + .../thirdparty/lpxscheduler/v1alpha1/types.go | 243 +- .../webhook/validation/shared_v1beta1.go | 5 +- 10 files changed, 13010 insertions(+), 167 deletions(-) create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_partition.go create mode 100644 deploy/operator/internal/dynamo/lpx/lpu_partition_test.go create mode 100644 deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index ac8d8db5c160..ba818bc8f16c 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -282,10 +282,8 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C $(YQ) eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \ fi; \ done - # Compact LPX schemas last: subsequent schema edits could change an aliased role. - for file in config/crd/bases/nvidia.com_dynamographdeployments.yaml config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml; do \ - $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 $$file || exit 1; \ - done + # Remove DGD-only LPX fields and rules from the standalone DCD schema. + $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml .PHONY: generate generate: controller-gen generate-pydantic ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/deploy/operator/api/scripts/lpx-crd-schema.yq b/deploy/operator/api/scripts/lpx-crd-schema.yq index f57a06163060..af0f5da3474d 100644 --- a/deploy/operator/api/scripts/lpx-crd-schema.yq +++ b/deploy/operator/api/scripts/lpx-crd-schema.yq @@ -7,25 +7,6 @@ with(select(.metadata.name == "dynamocomponentdeployments.nvidia.com"); (.spec.versions[].schema.openAPIV3Schema.properties.spec) |= ( del(.properties.lpx) | (select(has("x-kubernetes-validations"))."x-kubernetes-validations") |= - map(select(.rule | test("\\b(self|oldSelf)\\.(lpx|roles)\\b") | not)) + map(select(.rule | test("\\b(self|oldSelf)\\.lpx\\b") | not)) ) -) | - -# Keep native validation and merge semantics; omit only LPX template descriptions. -# Ordinary PodTemplate and alpha extraPodSpec documentation remains unchanged. -del(.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | .. | select(has("description")) | .description) | - -# Compact only LPX templates, after every other schema transformation has finished. -# YAML requires the anchor before every alias: traversal order is not document order. -with(select([.. | select(has("lpx") and has("roles"))] | length > 0); - ([.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - {"path": (path | to_json), "line": line, "column": column}] | sort_by(.line, .column))[0].path as $first | - [.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) == $first) | to_json][0] as $schema | - (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) == $first)) anchor = "lpx-pod-template" | - - # Different role-specific constraints must never be replaced by an alias. - (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) != $first) | select((to_json) == $schema)) alias = "lpx-pod-template" ) diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index cc5dd6298568..fe552efe235a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11880,9 +11880,16 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: &lpx-pod-template + podTemplate: + description: |- + PodTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. LPX components support role-specific Pod templates. properties: metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata properties: annotations: additionalProperties: @@ -11902,27 +11909,64 @@ spec: type: string type: object spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. format: int64 type: integer affinity: + description: If specified, the pod's scheduling constraints properties: nodeAffinity: + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: + description: A list of node selector requirements by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11934,13 +11978,27 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: + description: A list of node selector requirements by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11954,6 +12012,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -11963,18 +12022,43 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: + description: A list of node selector requirements by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11986,13 +12070,27 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: + description: A list of node selector requirements by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -12013,22 +12111,51 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12042,29 +12169,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12078,20 +12246,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -12101,18 +12287,49 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12126,29 +12343,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12162,15 +12420,30 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12179,22 +12452,51 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12208,29 +12510,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12244,20 +12587,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -12267,18 +12628,49 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12292,29 +12684,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12328,15 +12761,30 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12346,62 +12794,138 @@ spec: type: object type: object automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. type: boolean containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. items: + description: A single application container that you want to run within a pod. properties: args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -12410,29 +12934,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -12447,25 +12986,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -12473,31 +13039,67 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12506,58 +13108,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12566,75 +13214,130 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12643,62 +13346,129 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -12709,39 +13479,71 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12750,51 +13552,99 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -12803,13 +13653,34 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -12825,6 +13696,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -12833,20 +13707,69 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -12861,111 +13784,261 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12974,61 +14047,134 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -13039,21 +14185,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -13064,6 +14253,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -13073,87 +14267,203 @@ spec: - name x-kubernetes-list-type: map dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. properties: nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. items: type: string type: array x-kubernetes-list-type: atomic options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. items: + description: PodDNSConfigOption defines DNS resolver options of a pod. properties: name: + description: |- + Name is this DNS resolver option's name. + Required. type: string value: + description: Value is this DNS resolver option's value. type: string type: object type: array x-kubernetes-list-type: atomic searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. items: type: string type: array x-kubernetes-list-type: atomic type: object dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. type: boolean ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. properties: args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -13162,29 +14472,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -13199,25 +14524,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13225,31 +14577,63 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: Lifecycle is not allowed for ephemeral containers. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13258,58 +14642,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13318,75 +14748,126 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13395,62 +14876,121 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. type: string ports: + description: Ports are not allowed for ephemeral containers. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -13461,39 +15001,67 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13502,51 +15070,97 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: Resources resize policy for the container. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -13555,13 +15169,33 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -13577,6 +15211,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -13585,20 +15222,49 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + You cannot set this field on ephemeral containers. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. You cannot set this field on + ephemeral containers. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -13613,111 +15279,253 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13726,63 +15534,143 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. type: string terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -13793,21 +15681,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -13818,6 +15749,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -13827,14 +15763,22 @@ spec: - name x-kubernetes-list-type: map hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. properties: hostnames: + description: Hostnames for the above IP address. items: type: string type: array x-kubernetes-list-type: atomic ip: + description: IP address of the host file entry. type: string required: - ip @@ -13844,22 +15788,70 @@ spec: - ip x-kubernetes-list-type: map hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. type: boolean hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + When using HostNetwork you should specify ports so the scheduler is aware. + When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, + and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. + Default to false. type: boolean hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. type: boolean hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. type: boolean hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. type: string hostnameOverride: + description: |- + HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. + This field only specifies the pod's hostname and does not affect its DNS records. + When this field is set to a non-empty string: + - It takes precedence over the values set in `hostname` and `subdomain`. + - The Pod's hostname will be set to this value. + - `setHostnameAsFQDN` must be nil or set to false. + - `hostNetwork` must be set to false. + + This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. + Requires the HostnameOverride feature gate to be enabled. type: string imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -13868,60 +15860,144 @@ spec: - name x-kubernetes-list-type: map initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ items: + description: A single application container that you want to run within a pod. properties: args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -13930,29 +16006,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -13967,25 +16058,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13993,31 +16111,67 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14026,58 +16180,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14086,75 +16286,130 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14163,62 +16418,129 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -14229,39 +16551,71 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14270,51 +16624,99 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -14323,13 +16725,34 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -14345,6 +16768,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -14353,20 +16779,69 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -14381,111 +16856,261 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14494,61 +17119,134 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -14559,21 +17257,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -14584,6 +17325,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -14593,15 +17339,63 @@ spec: - name x-kubernetes-list-type: map nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename type: string nodeSelector: additionalProperties: type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.resources + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup properties: name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null type: string required: - name @@ -14613,18 +17407,50 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md type: object preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. type: string priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. format: int32 type: integer priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. type: string readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates items: + description: PodReadinessGate contains the reference to a pod condition properties: conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. type: string required: - conditionType @@ -14632,13 +17458,73 @@ spec: type: array x-kubernetes-list-type: atomic resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is a stable field but requires that the + DynamicResourceAllocation feature gate is enabled. + + This field is immutable. items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + + When the DRAWorkloadResourceClaims feature gate is enabled and this Pod + belongs to a PodGroup, a PodResourceClaim is matched to a + PodGroupResourceClaim if all of their fields are equal (Name, + ResourceClaimName, and ResourceClaimTemplateName). A matched claim references + a single ResourceClaim shared across all Pods in the PodGroup, reserved for + the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual + Pods. properties: name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. type: string resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. type: string resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + When the DRAWorkloadResourceClaims feature gate is enabled and the pod + belongs to a PodGroup that defines a PodGroupResourceClaim with the same + Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the + ResourceClaim generated for the PodGroup. All pods in the group that + define an equivalent PodResourceClaim matching the + PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same + generated ResourceClaim. ResourceClaims generated for a PodGroup are + owned by the PodGroup and their lifecycles are tied to the PodGroup + instead of any individual pod. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. type: string required: - name @@ -14648,13 +17534,40 @@ spec: - name x-kubernetes-list-type: map resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -14670,6 +17583,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -14678,18 +17594,47 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy type: string runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class type: string schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. type: string schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. properties: name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. type: string required: - name @@ -14699,70 +17644,217 @@ spec: - name x-kubernetes-list-type: map schedulingGroup: + description: |- + SchedulingGroup provides a reference to the immediate scheduling runtime + grouping object that this Pod belongs to. + This field is used by the scheduler to identify the group and apply the + correct group scheduling policies. The association with a group also + impacts other lifecycle aspects of a Pod that are relevant in a wider context + of scheduling like preemption, resource attachment, etc. If not specified, + the Pod is treated as a single unit in all of these aspects. + The group object referenced by this field may not exist at the time the + Pod is created. + This field is immutable, but a group object with the same name may be + recreated with different policies. Doing this during pod scheduling + may result in the placement not conforming to the expected policies. properties: podGroupName: + description: |- + PodGroupName specifies the name of the standalone PodGroup object + that represents the runtime instance of this group. + Must be a DNS subdomain. type: string type: object securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. properties: appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. type: string runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. type: string seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. type: string sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. items: + description: Sysctl defines a kernel parameter to be set properties: name: + description: Name of a property to set type: string value: + description: Value of a property to set type: string required: - name @@ -14771,60 +17863,156 @@ spec: type: array x-kubernetes-list-type: atomic windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. type: string serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ type: string setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. type: boolean shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. type: boolean subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. type: string terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. format: int64 type: integer tolerations: + description: If specified, the pod's tolerations. items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array x-kubernetes-list-type: atomic topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -14838,27 +18026,125 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string type: array x-kubernetes-list-type: atomic maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. format: int32 type: integer nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. type: string nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. type: string topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. type: string whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. type: string required: - maxSkew @@ -14871,111 +18157,244 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore properties: fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). format: int32 type: integer readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: boolean volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string required: - volumeID type: object azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. properties: cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' type: string diskName: + description: diskName is the Name of the data disk in the blob storage type: string diskURI: + description: diskURI is the URI of data disk in the blob storage type: string fsType: default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. properties: readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key type: string shareName: + description: shareName is the azure share Name type: string required: - secretName - shareName type: object cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. properties: monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: boolean secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string required: - monitors type: object cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: boolean secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string required: - volumeID type: object configMap: + description: configMap represents a configMap that should populate this volume properties: defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -14985,67 +18404,129 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. properties: driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. type: string fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. type: string nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: Items is a list of downward API volume file items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource @@ -15058,21 +18539,87 @@ spec: x-kubernetes-list-type: atomic type: object emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir properties: medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string sizeLimit: anyOf: - type: integer - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. properties: volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. properties: metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. properties: annotations: additionalProperties: @@ -15092,19 +18639,42 @@ spec: type: string type: object spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. properties: accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 items: type: string type: array x-kubernetes-list-type: atomic dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. type: string kind: + description: Kind is the type of resource being referenced type: string name: + description: Name is the name of resource being referenced type: string required: - kind @@ -15112,20 +18682,60 @@ spec: type: object x-kubernetes-map-type: atomic dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. type: string kind: + description: Kind is the type of resource being referenced type: string name: + description: Name is the name of resource being referenced type: string namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. type: string required: - kind - name type: object resources: + description: |- + resources represents the minimum resources the volume should have. + Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources properties: limits: additionalProperties: @@ -15134,6 +18744,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -15142,18 +18755,37 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object selector: + description: selector is a label query over volumes to consider for binding. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -15167,16 +18799,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ type: string volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. type: string volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object required: @@ -15184,41 +18838,79 @@ spec: type: object type: object fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string lun: + description: 'lun is Optional: FC target lun number' format: int32 type: integer readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' items: type: string type: array x-kubernetes-list-type: atomic wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. items: type: string type: array x-kubernetes-list-type: atomic type: object flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. properties: driver: + description: driver is the name of the driver to use for this volume. type: string fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string + description: 'options is Optional: this field holds extra command options if any.' type: object readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -15226,98 +18918,230 @@ spec: - driver type: object flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. properties: datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated type: string datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk properties: fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk format: int32 type: integer pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: boolean required: - pdName type: object gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. properties: directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. type: string repository: + description: repository is the URL type: string revision: + description: revision is the commit hash for the specified revision. type: string required: - repository type: object glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. properties: endpoints: + description: endpoints is the endpoint name that details Glusterfs topology. type: string path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: string readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: boolean required: - endpoints - path type: object hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath properties: path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string required: - path type: object image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. properties: pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. type: string reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string type: object iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication type: boolean fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi type: string initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. type: string iqn: + description: iqn is the target iSCSI Qualified Name. type: string iscsiInterface: default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). type: string lun: + description: lun represents iSCSI Target Lun number. format: int32 type: integer portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). items: type: string type: array x-kubernetes-list-type: atomic readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. type: boolean secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). type: string required: - iqn @@ -15325,68 +19149,161 @@ spec: - targetPortal type: object name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs properties: path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: boolean server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string required: - path - server type: object persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. type: boolean required: - claimName type: object photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: + description: pdID is the ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver. properties: fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean volumeID: + description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: + description: projected items for all in one resources secrets, configmaps, and downward API properties: defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. properties: clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. properties: labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -15400,31 +19317,72 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. type: string optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. type: boolean path: + description: Relative path from the volume root to write the bundle. type: string signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. type: string required: - path type: object configMap: + description: configMap information about the configMap data to project properties: items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15434,42 +19392,69 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic downwardAPI: + description: downwardAPI information about the downwardAPI data to project properties: items: + description: Items is a list of DownwardAPIVolume file items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource @@ -15482,39 +19467,161 @@ spec: x-kubernetes-list-type: atomic type: object podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. properties: certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. type: string credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. type: string keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. type: string keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". type: string maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. format: int32 type: integer signerName: + description: Kubelet's generated CSRs will be addressed to this signer. type: string userAnnotations: additionalProperties: type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. type: object required: - keyType - signerName type: object secret: + description: secret information about the secret data to project properties: items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15524,19 +19631,42 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional field specify whether the Secret or its key must be defined type: boolean type: object x-kubernetes-map-type: atomic serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project properties: audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. type: string expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. format: int64 type: integer path: + description: |- + path is the path relative to the mount point of the file to project the + token into. type: string required: - path @@ -15546,84 +19676,176 @@ spec: x-kubernetes-list-type: atomic type: object quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. properties: group: + description: |- + group to map volume access to + Default is no group type: string readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. type: boolean registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes type: string tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: + description: |- + user to map volume access to + Defaults to serivceaccount user type: string volume: + description: volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd type: string image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string keyring: default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic pool: default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: boolean secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string required: - image - monitors type: object scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. properties: fsType: default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". type: string gateway: + description: gateway is the host address of the ScaleIO API Gateway. type: string protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false type: boolean storageMode: default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. type: string storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. type: string system: + description: system is the name of the storage system as configured in ScaleIO. type: string volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. type: string required: - gateway @@ -15631,19 +19853,52 @@ spec: - system type: object secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret properties: defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15652,37 +19907,81 @@ spec: type: array x-kubernetes-list-type: atomic optional: + description: optional field specify whether the Secret or its keys must be defined type: boolean secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret type: string type: object storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. type: string volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. properties: fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. type: string volumePath: + description: volumePath is the path that identifies vSphere volume vmdk type: string required: - volumePath @@ -25160,7 +29459,8123 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: *lpx-pod-template + podTemplate: + description: |- + podTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. LPX components support role-specific Pod templates. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + You cannot set this field on ephemeral containers. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. You cannot set this field on + ephemeral containers. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + When using HostNetwork you should specify ports so the scheduler is aware. + When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, + and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + hostnameOverride: + description: |- + HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. + This field only specifies the pod's hostname and does not affect its DNS records. + When this field is set to a non-empty string: + - It takes precedence over the values set in `hostname` and `subdomain`. + - The Pod's hostname will be set to this value. + - `setHostnameAsFQDN` must be nil or set to false. + - `hostNetwork` must be set to false. + + This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. + Requires the HostnameOverride feature gate to be enabled. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.resources + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is a stable field but requires that the + DynamicResourceAllocation feature gate is enabled. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + + When the DRAWorkloadResourceClaims feature gate is enabled and this Pod + belongs to a PodGroup, a PodResourceClaim is matched to a + PodGroupResourceClaim if all of their fields are equal (Name, + ResourceClaimName, and ResourceClaimTemplateName). A matched claim references + a single ResourceClaim shared across all Pods in the PodGroup, reserved for + the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual + Pods. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + When the DRAWorkloadResourceClaims feature gate is enabled and the pod + belongs to a PodGroup that defines a PodGroupResourceClaim with the same + Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the + ResourceClaim generated for the PodGroup. All pods in the group that + define an equivalent PodResourceClaim matching the + PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same + generated ResourceClaim. ResourceClaims generated for a PodGroup are + owned by the PodGroup and their lifecycles are tied to the PodGroup + instead of any individual pod. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + schedulingGroup: + description: |- + SchedulingGroup provides a reference to the immediate scheduling runtime + grouping object that this Pod belongs to. + This field is used by the scheduler to identify the group and apply the + correct group scheduling policies. The association with a group also + impacts other lifecycle aspects of a Pod that are relevant in a wider context + of scheduling like preemption, resource attachment, etc. If not specified, + the Pod is treated as a single unit in all of these aspects. + The group object referenced by this field may not exist at the time the + Pod is created. + This field is immutable, but a group object with the same name may be + recreated with different policies. Doing this during pod scheduling + may result in the placement not conforming to the expected policies. + properties: + podGroupName: + description: |- + PodGroupName specifies the name of the standalone PodGroup object + that represents the runtime instance of this group. + Must be a DNS subdomain. + type: string + type: object + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + properties: + endpoints: + description: endpoints is the endpoint name that details Glusterfs topology. + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. + type: object + required: + - keyType + - signerName + type: object + secret: + description: secret information about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object providerOverride: description: |- providerOverride configures the provider workload unit generated for this diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index 005f1cb029be..eeae5fcc5bd5 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -10,6 +10,7 @@ import ( "fmt" "slices" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" @@ -70,9 +71,39 @@ func setupLPUEviction(mgr ctrl.Manager) error { Watches(&corev1.Pod{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(r.evictionPredicate()), ). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.evictionRequestsForConfigMap)). Complete(r) } +func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context, object client.Object) []ctrl.Request { + // Ignore ConfigMaps outside the hybrid runtime contract before listing Pods. + config, ok := object.(*corev1.ConfigMap) + if !ok || config.Data["partition_models"] == "" { + return nil + } + + // A late runtime table observation must wake disrupted Agents using that exact table. + var pods corev1.PodList + if err := r.List(ctx, &pods, client.InNamespace(config.GetNamespace()), client.MatchingLabels{ + commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, + }); err != nil { + log.FromContext(ctx).Error(err, "list LPU eviction triggers for runtime ConfigMap") + return nil + } + + // Unrelated workloads and other immutable runtime revisions do not need another reconcile. + var requests []ctrl.Request + for i := range pods.Items { + pod := &pods.Items[i] + root := pod.Labels[grovecommon.LabelPartOfKey] + hash := pod.Annotations[commonconsts.AnnotationExtraResourcesHash] + if r.isTrigger(pod) && root != "" && hash != "" && dynamolpx.LPUConfigMapName(root, hash) == config.GetName() { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) + } + } + return requests +} + func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { owner := metav1.GetControllerOf(trigger) if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || @@ -98,18 +129,43 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor switch mode { case lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly: case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: - triggerRow, err := lpxv1alpha1.ParsePodLogicalRow(trigger.Annotations) + // The Pod's content-addressed runtime table survives native template updates. + root := trigger.Labels[grovecommon.LabelPartOfKey] + hash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] + if root == "" || hash == "" { + return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap identity", trigger.Namespace, trigger.Name) + } + var config corev1.ConfigMap + key := client.ObjectKey{Namespace: trigger.Namespace, Name: dynamolpx.LPUConfigMapName(root, hash)} + if err := r.Get(ctx, key, &config); err != nil { + return nil, fmt.Errorf("read LPU-GPU eviction runtime ConfigMap: %w", err) + } + if config.Immutable == nil || !*config.Immutable { + return nil, fmt.Errorf("LPU-GPU eviction runtime ConfigMap %s is not immutable", key) + } + + // Bind the table to the verified deployment and the Pod's full content hash. + configOwner := metav1.GetControllerOf(&config) + if configOwner == nil || configOwner.APIVersion != nvidiacomv1alpha1.GroupVersion.String() || + configOwner.Kind != nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind || + configOwner.Name != clique.Annotations[dynamolpx.DeploymentNameAnnotation] || + string(configOwner.UID) != clique.Annotations[dynamolpx.DeploymentUIDAnnotation] || + dynamolpx.LPUConfigMapHash(&config) != hash { + return nil, fmt.Errorf("cannot authenticate LPU-GPU eviction runtime ConfigMap %s", key) + } + triggerPartition, err := dynamolpx.LPUAgentRuntimePartition(trigger, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction trigger pod %s/%s: %w", trigger.Namespace, trigger.Name, err) } + // Select the full runtime partition before authorizing any sibling deletion. selected := pods[:0] for _, pod := range pods { - row, err := lpxv1alpha1.ParsePodLogicalRow(pod.Annotations) + partition, err := dynamolpx.LPUAgentRuntimePartition(&pod, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction candidate pod %s/%s: %w", pod.Namespace, pod.Name, err) } - if row.ModelPartitionID == triggerRow.ModelPartitionID { + if partition == triggerPartition { selected = append(selected, pod) } } @@ -151,6 +207,7 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev owner.APIVersion != candidateOwner.APIVersion || owner.Kind != candidateOwner.Kind || owner.Name != candidateOwner.Name || owner.UID != candidateOwner.UID || pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || + pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] || model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model }), nil } diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 7000cf0b44e6..f66da81d939d 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -30,16 +30,18 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" ) const ( - lpuEvictionTestNamespace = "test-ns" - lpuEvictionTestDGD = "test-dgd" - lpuEvictionTestLGD = "test-lgd" - lpuEvictionTestPCS = "materialization-12345678" - lpuEvictionTestComponent = "lpx-worker" - lpuEvictionTestPCSG = "lpx-worker" - lpuEvictionTestModel = "test-model" + lpuEvictionTestNamespace = "test-ns" + lpuEvictionTestDGD = "test-dgd" + lpuEvictionTestLGD = "test-lgd" + lpuEvictionTestPCS = "materialization-12345678" + lpuEvictionTestComponent = "lpx-worker" + lpuEvictionTestPCSG = "lpx-worker" + lpuEvictionTestModel = "test-model" + lpuEvictionTestConfigHash = "bf665efcf39a0c79fa97ea2c05664ca7235dd525aced22f47da0633d729d6171" ) func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { @@ -70,10 +72,21 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, }} + config := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + }, + Immutable: ptr.To(true), + Data: map[string]string{ + "nodes_per_partition": "2\n1", "partition_node_offsets": "0\n2", + "partition_models": lpuEvictionTestModel + "\n" + lpuEvictionTestModel, + }, + } cb := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&corev1.Pod{}). - WithObjects(deployment, pcs, group, clique) + WithObjects(deployment, pcs, group, clique, config) for _, o := range objs { cb = cb.WithObjects(o) } @@ -98,10 +111,9 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, }, Annotations: map[string]string{ - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - lpxv1alpha1.PodPartitionIDAnnotation: "0", - lpxv1alpha1.PodRankInPartitionAnnotation: podIndex, - dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + commonconsts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, }, }, Spec: corev1.PodSpec{SchedulerName: dynamolpx.SchedulerName}, @@ -110,9 +122,6 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting if podIndex != "" { pod.Labels[grovecommon.LabelPodCliquePodIndex] = podIndex } - if podIndex == "2" { - pod.Annotations[lpxv1alpha1.PodPartitionIDAnnotation] = "1" - } pod.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel if conditionReason != "" { pod.Status.Conditions = []corev1.PodCondition{{ @@ -172,13 +181,15 @@ func lpuEvictionPodNames(pods []corev1.Pod) []string { } func TestLPUEviction_PodsForTrigger(t *testing.T) { - t.Run("lpu-gpu pod without partition returns error", func(t *testing.T) { + t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { + t.Log("Observe a hybrid Agent without its canonical Grove index") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - delete(trigger.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") r, _ := newLPUEvictionReconciler(trigger, sibling) + t.Log("Reject the incomplete identity before selecting any deletion") pods, err := r.podsForTrigger(context.Background(), trigger) require.Error(t, err) @@ -322,36 +333,166 @@ func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { } func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { - t.Log("Observe live neighbors alongside a terminating Agent without scheduler partition metadata") - trigger := newLPUEvictionPod("stage-0-partition-3", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - samePartition := newLPUEvictionPod("stage-1-partition-3", "0", "1", false, "") - otherPartition := newLPUEvictionPod("stage-0-partition-4", "0", "2", false, "") - otherModel := newLPUEvictionPod("stage-0-draft-partition-3", "0", "0", false, "") - otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" - noPartition := newLPUOnlyConductor() - otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") - delete(terminating.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) - delete(terminating.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) - - r, c := newLPUEvictionReconciler( - trigger, - samePartition, - otherPartition, - otherModel, - noPartition, - otherReplica, - terminating, - ) - - t.Log("Evict the live neighbor in the trigger's partition") - result := reconcileLPUEviction(t, r, "stage-0-partition-3") - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"conductor-0", "replica-1-partition-3", "stage-0-draft-partition-3", "stage-0-partition-4"}, remainingLPUEvictionPods(t, c)) + for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid} { + t.Run(string(mode), func(t *testing.T) { + t.Log("Observe live neighbors alongside a terminating Agent without scheduler partition metadata") + trigger := newLPUEvictionPod("stage-0-partition-3", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + samePartition := newLPUEvictionPod("stage-1-partition-3", "0", "1", false, "") + otherPartition := newLPUEvictionPod("stage-0-partition-4", "0", "2", false, "") + otherModel := newLPUEvictionPod("stage-0-draft-partition-3", "0", "0", false, "") + otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" + noPartition := newLPUOnlyConductor() + otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") + + t.Log("Consume the real shared-cache Pod projection without deprecated row annotations") + for _, pod := range []*corev1.Pod{trigger, samePartition, otherPartition, otherModel, noPartition, otherReplica, terminating} { + podcache.Project(pod) + } - t.Log("Reconcile successfully after the partition's live neighbors are gone") + r, c := newLPUEvictionReconciler( + trigger, + samePartition, + otherPartition, + otherModel, + noPartition, + otherReplica, + terminating, + ) + + t.Log("Evict the live neighbor in the trigger's partition") + result := reconcileLPUEviction(t, r, "stage-0-partition-3") + assert.Equal(t, ctrl.Result{}, result) + assert.Equal(t, []string{"conductor-0", "replica-1-partition-3", "stage-0-draft-partition-3", "stage-0-partition-4"}, remainingLPUEvictionPods(t, c)) + + t.Log("Reconcile successfully after the partition's live neighbors are gone") + reconcileLPUEviction(t, r, trigger.Name) + }) + } +} + +func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { + t.Log("Define missing runtime dependencies and malformed candidate identities") + tests := []struct { + name string + candidateIndex string + missingHash bool + missingModel bool + missingConfig bool + mutableConfig bool + missingOwner bool + foreignOwner bool + alteredData bool + }{ + {name: "missing candidate index", candidateIndex: ""}, + {name: "invalid candidate index", candidateIndex: "bad"}, + {name: "out of range candidate", candidateIndex: "3"}, + {name: "missing config identity", candidateIndex: "2", missingHash: true}, + {name: "missing model", candidateIndex: "2", missingModel: true}, + {name: "config not observed", candidateIndex: "2", missingConfig: true}, + {name: "mutable config", candidateIndex: "2", mutableConfig: true}, + {name: "unowned config", candidateIndex: "2", missingOwner: true}, + {name: "foreign config owner", candidateIndex: "2", foreignOwner: true}, + {name: "replaced config with altered ranges", candidateIndex: "2", alteredData: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Observe a valid sibling before another candidate whose identity cannot authorize deletion") + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + if test.missingHash { + delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) + } + if test.missingModel { + delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) + } + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + candidate := newLPUEvictionPod("agent-2", "0", test.candidateIndex, false, "") + r, c := newLPUEvictionReconciler(trigger, sibling, candidate) + config := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + require.NoError(t, c.Get(t.Context(), key, config)) + require.Equal(t, lpuEvictionTestConfigHash, dynamolpx.LPUConfigMapHash(config)) + if test.missingConfig { + require.NoError(t, c.Delete(t.Context(), config)) + } + if test.mutableConfig { + config.Immutable = ptr.To(false) + require.NoError(t, c.Update(t.Context(), config)) + } + if test.missingOwner || test.foreignOwner || test.alteredData { + require.NoError(t, c.Delete(t.Context(), config)) + config.ResourceVersion = "" + config.UID = "replacement-config-uid" + if test.missingOwner { + config.OwnerReferences = nil + } + if test.foreignOwner { + config.OwnerReferences[0].UID = "foreign-lgd-uid" + } + if test.alteredData { + config.Data["nodes_per_partition"] = "3\n1" + config.Data["partition_node_offsets"] = "0\n3" + } + require.NoError(t, c.Create(t.Context(), config)) + } + + t.Log("Return an error without partially deleting the cohort") + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) + require.Error(t, err) + require.Equal(t, []string{sibling.Name, candidate.Name}, remainingLPUEvictionPods(t, c)) + }) + } +} + +func TestLPUEvictionConfigMapObservationWakesTrigger(t *testing.T) { + t.Log("Observe a hybrid disruption before its immutable runtime table reaches the cache") + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, c := newLPUEvictionReconciler(trigger, sibling) + config := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + require.NoError(t, c.Get(t.Context(), key, config)) + require.NoError(t, c.Delete(t.Context(), config)) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) + require.Error(t, err) + require.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) + + t.Log("Wake only the disrupted Pod when its exact runtime table becomes visible") + config.ResourceVersion = "" + require.NoError(t, c.Create(t.Context(), config)) + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(trigger)}}, r.evictionRequestsForConfigMap(t.Context(), config)) + unrelated := config.DeepCopy() + unrelated.Name = "different-runtime-config" + require.Empty(t, r.evictionRequestsForConfigMap(t.Context(), unrelated)) + + t.Log("Converge from the new cached state through the original deletion owner") reconcileLPUEviction(t, r, trigger.Name) + require.Empty(t, remainingLPUEvictionPods(t, c)) +} + +func TestLPUEvictionLPUOnlyNeedsNoRuntimePartitionTable(t *testing.T) { + for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly} { + t.Run(string(mode), func(t *testing.T) { + t.Log("Observe an LPU-only disruption without hybrid runtime partition metadata") + trigger := newLPUEvictionPod("agent-0", "0", "", true, podDisruptionReasonTaintManagerDeletion) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + sibling := newLPUEvictionPod("agent-1", "0", "", false, "") + delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) + delete(sibling.Annotations, commonconsts.AnnotationExtraResourcesHash) + r, c := newLPUEvictionReconciler(trigger, sibling) + config := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), + }} + require.NoError(t, c.Delete(t.Context(), config)) + + t.Log("Keep the existing model-cohort eviction behavior independent of partition lookup") + reconcileLPUEviction(t, r, trigger.Name) + require.Empty(t, remainingLPUEvictionPods(t, c)) + }) + } } func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition.go b/deploy/operator/internal/dynamo/lpx/lpu_partition.go new file mode 100644 index 000000000000..3ed835ef97c0 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_partition.go @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "strconv" + "strings" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + corev1 "k8s.io/api/core/v1" +) + +// LPUAgentRuntimePartition maps a hybrid Agent's Grove index to its runtime +// ConfigMap row. Both inputs must be non-nil and config must be the immutable +// table stamped on the Pod's template. Inputs are not mutated. Runtime rows, +// rather than scheduler partitions, preserve XT's collapsed PropSync cohorts. +func LPUAgentRuntimePartition(pod *corev1.Pod, config *corev1.ConfigMap) (int, error) { + // Consume only canonical Grove identity, never deprecated partition/rank annotations. + rawIndex := pod.Labels[grovecommon.LabelPodCliquePodIndex] + index, err := strconv.ParseUint(rawIndex, 10, 32) + if err != nil || strconv.FormatUint(index, 10) != rawIndex { + return 0, fmt.Errorf("pod %s/%s has invalid Grove Pod index %q", pod.Namespace, pod.Name, rawIndex) + } + model := pod.Annotations[lpxv1alpha1.PodModelAnnotation] + if model == "" { + return 0, fmt.Errorf("pod %s/%s has no LPU model", pod.Namespace, pod.Name) + } + + // Read the same three columns used by the hybrid Agent's partition metadata command. + counts := strings.Split(config.Data["nodes_per_partition"], "\n") + offsets := strings.Split(config.Data["partition_node_offsets"], "\n") + models := strings.Split(config.Data["partition_models"], "\n") + if len(counts) != len(offsets) || len(counts) != len(models) { + return 0, fmt.Errorf("ConfigMap %s/%s has inconsistent runtime partition columns", config.Namespace, config.Name) + } + + // Require one unambiguous runtime range for this model and Grove index. + matched := -1 + for row, rowModel := range models { + if rowModel != model { + continue + } + count, countErr := strconv.ParseUint(counts[row], 10, 32) + offset, offsetErr := strconv.ParseUint(offsets[row], 10, 32) + if countErr != nil || offsetErr != nil || count == 0 { + return 0, fmt.Errorf("ConfigMap %s/%s has invalid runtime partition row %d", config.Namespace, config.Name, row) + } + if index < offset || index-offset >= count { + continue + } + if matched >= 0 { + return 0, fmt.Errorf("Grove Pod index %d matches multiple runtime partitions for model %q", index, model) + } + matched = row + } + if matched < 0 { + return 0, fmt.Errorf("Grove Pod index %d matches no runtime partition for model %q", index, model) + } + return matched, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go b/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go new file mode 100644 index 000000000000..0f27f415fb4b --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "testing" + + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLPUAgentRuntimePartition(t *testing.T) { + t.Log("Cover model-local Grove indexes and incomplete or ambiguous runtime mappings") + tests := []struct { + name string + index string + model string + data map[string]string + want int + wantError string + }{ + {name: "second rank", index: "1", model: "model", want: 0}, + {name: "next partition", index: "2", model: "model", want: 1}, + {name: "other model resets indexes", index: "1", model: "other", want: 2}, + {name: "missing index", model: "model", wantError: "invalid Grove Pod index"}, + {name: "noncanonical index", index: "01", model: "model", wantError: "invalid Grove Pod index"}, + {name: "negative index", index: "-1", model: "model", wantError: "invalid Grove Pod index"}, + {name: "overflow index", index: "4294967296", model: "model", wantError: "invalid Grove Pod index"}, + {name: "missing model", index: "0", wantError: "has no LPU model"}, + {name: "absent model", index: "0", model: "absent", wantError: "matches no runtime partition"}, + {name: "out of range", index: "3", model: "model", wantError: "matches no runtime partition"}, + {name: "missing column", index: "0", model: "model", data: map[string]string{"partition_node_offsets": ""}, wantError: "inconsistent runtime partition columns"}, + {name: "bad count", index: "0", model: "model", data: map[string]string{"nodes_per_partition": "bad\n1\n3"}, wantError: "invalid runtime partition row"}, + {name: "zero count", index: "0", model: "model", data: map[string]string{"nodes_per_partition": "0\n1\n3"}, wantError: "invalid runtime partition row"}, + {name: "bad offset", index: "0", model: "model", data: map[string]string{"partition_node_offsets": "-1\n2\n0"}, wantError: "invalid runtime partition row"}, + {name: "overlapping ranges", index: "1", model: "model", data: map[string]string{"partition_node_offsets": "0\n1\n0"}, wantError: "matches multiple runtime partitions"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Build a runtime table with variable partition sizes and model-local offsets") + config := &corev1.ConfigMap{Data: map[string]string{ + "nodes_per_partition": "2\n1\n3", "partition_node_offsets": "0\n2\n0", + "partition_models": "model\nmodel\nother", + }} + for key, value := range test.data { + config.Data[key] = value + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{grovecommon.LabelPodCliquePodIndex: test.index}, + Annotations: map[string]string{ + lpxv1alpha1.PodModelAnnotation: test.model, + lpxv1alpha1.PodPartitionIDAnnotation: "stale", lpxv1alpha1.PodRankInPartitionAnnotation: "not-authority", + }, + }} + originalPod, originalConfig := pod.DeepCopy(), config.DeepCopy() + + t.Log("Resolve the canonical row without mutating producer-owned data") + got, err := LPUAgentRuntimePartition(pod, config) + require.Equal(t, originalPod, pod) + require.Equal(t, originalConfig, config) + if test.wantError != "" { + require.ErrorContains(t, err, test.wantError) + return + } + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} + +func TestLPUAgentRuntimePartitionUsesRenderedHybridTables(t *testing.T) { + for _, test := range []struct { + name string + family BuildFamily + collapsed bool + }{ + {name: "XT", family: BuildFamilyXT}, + {name: "HX", family: BuildFamilyHX}, + {name: "XT collapsed PropSync chain", family: BuildFamilyXT, collapsed: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Render a hybrid runtime table with a two-Agent partition followed by one Agent") + partitions := []BuildPartition{ + {Topology: Topology{ChipCount: 16}, HXExtent: []int64{1, 1, 1, 2}}, + {Topology: Topology{ChipCount: 8}, HXExtent: []int64{1, 1, 1, 1}}, + } + projection := &ModelProjection{ + model: "model", pipeline: PipelineLPX, partitions: partitions, + configuredBuild: Build{Family: test.family, Partitions: partitions}, + } + if test.collapsed { + projection.configuredBuild.Partitions = []BuildPartition{{Topology: Topology{ChipCount: 24}}} + } + config := &corev1.ConfigMap{Data: resolvedPartitionData([]*ModelProjection{projection})} + + t.Log("Resolve every rendered Grove index to the same runtime grouping") + for _, row := range []struct { + index string + want int + }{{"0", 0}, {"1", 0}, {"2", 1}} { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{grovecommon.LabelPodCliquePodIndex: row.index}, + Annotations: map[string]string{lpxv1alpha1.PodModelAnnotation: "model"}, + }} + got, err := LPUAgentRuntimePartition(pod, config) + require.NoError(t, err) + if test.collapsed { + row.want = 0 + } + require.Equal(t, row.want, got) + } + }) + } +} diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go new file mode 100644 index 000000000000..151bb242f863 --- /dev/null +++ b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go @@ -0,0 +1,4 @@ +// Code generated by lpx-scheduler. DO NOT EDIT. + +// Package v1alpha1 contains the published LPX scheduling API types. +package v1alpha1 diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go index fa4c6d817a31..1a84d61a1d0e 100644 --- a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go +++ b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go @@ -7,7 +7,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "strconv" "unicode/utf8" ) @@ -26,15 +25,9 @@ const ( PodRoleAgent = "agent" PodRoleConductor = "conductor" PodRoleCyborgWorker = "cyborg-worker" - MaxModelAnnotationBytes = 261982 + MaxModelAnnotationBytes = 261054 ) -type PodLogicalRow struct { - Model string - ModelPartitionID uint32 - RankInPartition uint16 -} - func ParsePodRole(value string) (string, error) { switch value { case PodRoleAgent, PodRoleConductor, PodRoleCyborgWorker: @@ -62,80 +55,6 @@ func ValidateModelAnnotation(value string) error { return nil } -func ParseModelPartitionID(value string) (uint32, error) { - parsed, err := parseCanonicalUnsigned(PodPartitionIDAnnotation, value, 32) - return uint32(parsed), err -} - -func ParseRankInPartition(value string) (uint16, error) { - parsed, err := parseCanonicalUnsigned(PodRankInPartitionAnnotation, value, 16) - return uint16(parsed), err -} - -func ParsePodLogicalRow(annotations map[string]string) (PodLogicalRow, error) { - role, err := requiredPodAnnotation(annotations, PodRoleAnnotation) - if err != nil { - return PodLogicalRow{}, err - } - if _, err := ParsePodRole(role); err != nil { - return PodLogicalRow{}, err - } - if role != PodRoleAgent { - return PodLogicalRow{}, fmt.Errorf("logical-row parsing requires role=agent, got %q", role) - } - model, err := requiredPodAnnotation(annotations, PodModelAnnotation) - if err != nil { - return PodLogicalRow{}, err - } - if err := ValidateModelAnnotation(model); err != nil { - return PodLogicalRow{}, err - } - partition, err := requiredPodAnnotation(annotations, PodPartitionIDAnnotation) - if err != nil { - return PodLogicalRow{}, err - } - modelPartitionID, err := ParseModelPartitionID(partition) - if err != nil { - return PodLogicalRow{}, err - } - rank, err := requiredPodAnnotation(annotations, PodRankInPartitionAnnotation) - if err != nil { - return PodLogicalRow{}, err - } - rankInPartition, err := ParseRankInPartition(rank) - if err != nil { - return PodLogicalRow{}, err - } - return PodLogicalRow{Model: model, ModelPartitionID: modelPartitionID, RankInPartition: rankInPartition}, nil -} - -func requiredPodAnnotation(annotations map[string]string, key string) (string, error) { - value, ok := annotations[key] - if !ok { - return "", fmt.Errorf("required Pod annotation %q is missing", key) - } - return value, nil -} - -func parseCanonicalUnsigned(key, value string, bitSize int) (uint64, error) { - if value == "" { - return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) - } - for index := 0; index < len(value); index++ { - if value[index] < '0' || value[index] > '9' { - return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) - } - } - parsed, err := strconv.ParseUint(value, 10, bitSize) - if err != nil { - return 0, fmt.Errorf("Pod annotation %q value %q exceeds its public integer range", key, value) - } - if strconv.FormatUint(parsed, 10) != value { - return 0, fmt.Errorf("Pod annotation %q value %q is not canonical unsigned decimal", key, value) - } - return parsed, nil -} - func isAnnotationBoundaryWhitespace(value rune) bool { switch { case value >= 0x0009 && value <= 0x000d: @@ -233,6 +152,7 @@ const ( BindingTargetPhaseRecorded BindingTargetPhase = "recorded" BindingTargetPhaseClaimsAuthorized BindingTargetPhase = "claimsAuthorized" BindingTargetPhaseBindingAttempted BindingTargetPhase = "bindingAttempted" + BindingTargetPhaseRetiring BindingTargetPhase = "retiring" ) type ClaimLifecyclePhase string @@ -641,6 +561,8 @@ func (in *GroveLineage) DeepCopy() *GroveLineage { return out } +type GroveSlotDigest string + type HxLogicalConnection struct { FromLogicalDevice int64 `json:"fromLogicalDevice"` ToLogicalDevice int64 `json:"toLogicalDevice"` @@ -784,13 +706,14 @@ func (in *LPUPipelineRequestSpec) DeepCopy() *LPUPipelineRequestSpec { } type LPUPipelineRequestStatus struct { - CommitFence *CommitFence `json:"commitFence,omitempty"` - Committed *Committed `json:"committed,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - Diagnostics []StatusDiagnostic `json:"diagnostics,omitempty"` - LastPlanRevision int64 `json:"lastPlanRevision"` - ObservedGeneration *int64 `json:"observedGeneration,omitempty"` - Phase RequestPhase `json:"phase"` + CommitFence *CommitFence `json:"commitFence,omitempty"` + Committed *Committed `json:"committed,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + Diagnostics []StatusDiagnostic `json:"diagnostics,omitempty"` + LastAbortedPlanReplacement *PlanReplacementAbortFence `json:"lastAbortedPlanReplacement,omitempty"` + LastPlanRevision int64 `json:"lastPlanRevision"` + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + Phase RequestPhase `json:"phase"` } func (in *LPUPipelineRequestStatus) DeepCopyInto(out *LPUPipelineRequestStatus) { @@ -813,6 +736,10 @@ func (in *LPUPipelineRequestStatus) DeepCopyInto(out *LPUPipelineRequestStatus) in.Diagnostics[i].DeepCopyInto(&out.Diagnostics[i]) } } + if in.LastAbortedPlanReplacement != nil { + value := *in.LastAbortedPlanReplacement + out.LastAbortedPlanReplacement = &value + } if in.ObservedGeneration != nil { value := *in.ObservedGeneration out.ObservedGeneration = &value @@ -902,6 +829,7 @@ func (in *NodeLocalEndpoint) DeepCopy() *NodeLocalEndpoint { type NodeLocalExecution struct { PartitionSelections []NodeLocalPartitionExecution `json:"partitionSelections"` + PlanReplacement *NodeLocalPlanReplacement `json:"planReplacement,omitempty"` } func (in *NodeLocalExecution) DeepCopyInto(out *NodeLocalExecution) { @@ -912,6 +840,9 @@ func (in *NodeLocalExecution) DeepCopyInto(out *NodeLocalExecution) { in.PartitionSelections[i].DeepCopyInto(&out.PartitionSelections[i]) } } + if in.PlanReplacement != nil { + out.PlanReplacement = in.PlanReplacement.DeepCopy() + } } func (in *NodeLocalExecution) DeepCopy() *NodeLocalExecution { @@ -996,6 +927,7 @@ type NodeLocalPlacement struct { EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` InventoryRevision string `json:"inventoryRevision"` Model string `json:"model"` + NativeRepairBase *string `json:"nativeRepairBase,omitempty"` Objective PlanObjective `json:"objective"` PartitionSelections []NodeLocalPartitionSelection `json:"partitionSelections"` PodGangRef ObjectReference `json:"podGangRef"` @@ -1017,6 +949,10 @@ func (in *NodeLocalPlacement) DeepCopyInto(out *NodeLocalPlacement) { out.EffectiveNodeRequests[key] = value } } + if in.NativeRepairBase != nil { + value := *in.NativeRepairBase + out.NativeRepairBase = &value + } in.Objective.DeepCopyInto(&out.Objective) if in.PartitionSelections != nil { out.PartitionSelections = make([]NodeLocalPartitionSelection, len(in.PartitionSelections)) @@ -1036,6 +972,48 @@ func (in *NodeLocalPlacement) DeepCopy() *NodeLocalPlacement { return out } +type NodeLocalPlanReplacement struct { + BasePlanDigest PlanDigest `json:"basePlanDigest"` + BasePlanRevision int64 `json:"basePlanRevision"` + RealizeMembers []RealizeMember `json:"realizeMembers"` +} + +func (in *NodeLocalPlanReplacement) DeepCopyInto(out *NodeLocalPlanReplacement) { + *out = *in + if in.RealizeMembers != nil { + out.RealizeMembers = make([]RealizeMember, len(in.RealizeMembers)) + for i := range in.RealizeMembers { + in.RealizeMembers[i].DeepCopyInto(&out.RealizeMembers[i]) + } + } +} + +func (in *NodeLocalPlanReplacement) DeepCopy() *NodeLocalPlanReplacement { + if in == nil { + return nil + } + out := new(NodeLocalPlanReplacement) + in.DeepCopyInto(out) + return out +} + +type NodeLocalRepairAuthorization struct { + GroveSlotDigest GroveSlotDigest `json:"groveSlotDigest"` +} + +func (in *NodeLocalRepairAuthorization) DeepCopyInto(out *NodeLocalRepairAuthorization) { + *out = *in +} + +func (in *NodeLocalRepairAuthorization) DeepCopy() *NodeLocalRepairAuthorization { + if in == nil { + return nil + } + out := new(NodeLocalRepairAuthorization) + in.DeepCopyInto(out) + return out +} + type NodeLocalRequest struct { Model string `json:"model"` PartitionMappings []NodeLocalPartitionMapping `json:"partitionMappings"` @@ -1061,9 +1039,10 @@ func (in *NodeLocalRequest) DeepCopy() *NodeLocalRequest { } type NodeLocalRowExecution struct { - Current *CurrentBinding `json:"current,omitempty"` - RankInPartition int64 `json:"rankInPartition"` - Target *BindingTarget `json:"target,omitempty"` + Current *CurrentBinding `json:"current,omitempty"` + RankInPartition int64 `json:"rankInPartition"` + Repair *NodeLocalRepairAuthorization `json:"repair,omitempty"` + Target *BindingTarget `json:"target,omitempty"` } func (in *NodeLocalRowExecution) DeepCopyInto(out *NodeLocalRowExecution) { @@ -1071,6 +1050,9 @@ func (in *NodeLocalRowExecution) DeepCopyInto(out *NodeLocalRowExecution) { if in.Current != nil { out.Current = in.Current.DeepCopy() } + if in.Repair != nil { + out.Repair = in.Repair.DeepCopy() + } if in.Target != nil { out.Target = in.Target.DeepCopy() } @@ -1147,6 +1129,28 @@ func (in *ObjectReference) DeepCopy() *ObjectReference { return out } +type OldOnlyMember struct { + Current CurrentBinding `json:"current"` + OldEndpoint NodeLocalEndpoint `json:"oldEndpoint"` + OldNodeRef NodeReference `json:"oldNodeRef"` +} + +func (in *OldOnlyMember) DeepCopyInto(out *OldOnlyMember) { + *out = *in + in.Current.DeepCopyInto(&out.Current) + in.OldEndpoint.DeepCopyInto(&out.OldEndpoint) + in.OldNodeRef.DeepCopyInto(&out.OldNodeRef) +} + +func (in *OldOnlyMember) DeepCopy() *OldOnlyMember { + if in == nil { + return nil + } + out := new(OldOnlyMember) + in.DeepCopyInto(out) + return out +} + type PartitionClaimRequest struct { ClaimName string `json:"claimName"` PartitionID string `json:"partitionId"` @@ -1263,6 +1267,8 @@ func (in *PlanPlacement) DeepCopy() *PlanPlacement { return out } +type PlanReplacementAbortFence string + type PropSyncConnectorKind string const ( @@ -1324,6 +1330,57 @@ func (in *PropSyncConnectorRequirement) DeepCopy() *PropSyncConnectorRequirement return out } +type RealizeMember struct { + GroveSlotDigest GroveSlotDigest `json:"groveSlotDigest"` + NextBindingRevision int64 `json:"nextBindingRevision"` + PartitionID string `json:"partitionId"` + Predecessor RealizeMemberPredecessor `json:"predecessor"` + RankInPartition int64 `json:"rankInPartition"` +} + +func (in *RealizeMember) DeepCopyInto(out *RealizeMember) { + *out = *in + in.Predecessor.DeepCopyInto(&out.Predecessor) +} + +func (in *RealizeMember) DeepCopy() *RealizeMember { + if in == nil { + return nil + } + out := new(RealizeMember) + in.DeepCopyInto(out) + return out +} + +type RealizeMemberPredecessor struct { + OldOnly *OldOnlyMember `json:"oldOnly,omitempty"` + Phase RealizeMemberPredecessorPhase `json:"phase"` +} + +func (in *RealizeMemberPredecessor) DeepCopyInto(out *RealizeMemberPredecessor) { + *out = *in + if in.OldOnly != nil { + out.OldOnly = in.OldOnly.DeepCopy() + } +} + +func (in *RealizeMemberPredecessor) DeepCopy() *RealizeMemberPredecessor { + if in == nil { + return nil + } + out := new(RealizeMemberPredecessor) + in.DeepCopyInto(out) + return out +} + +type RealizeMemberPredecessorPhase string + +const ( + RealizeMemberPredecessorPhaseProtected RealizeMemberPredecessorPhase = "protected" + RealizeMemberPredecessorPhaseAuthorized RealizeMemberPredecessorPhase = "authorized" + RealizeMemberPredecessorPhaseRetired RealizeMemberPredecessorPhase = "retired" +) + type ReleaseJournal struct { Reason ReleaseReason `json:"reason"` RequestedAtGeneration int64 `json:"requestedAtGeneration"` @@ -1496,6 +1553,8 @@ type RepairPolicyMode string const ( RepairPolicyModeColdReplacement RepairPolicyMode = "coldReplacement" + RepairPolicyModeSamePlacement RepairPolicyMode = "samePlacement" + RepairPolicyModeRepair RepairPolicyMode = "repair" ) type RequestIntentDigest string diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index ac4f7843cc59..63452d81b084 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -814,7 +814,7 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( "cannot change node topology between single-node and multi-node after creation", )) } - } else if !newComponent.IsLPX() && !oldComponent.IsLPX() { + } else { if !hasUnsupportedMultinode(newComponent) && newComponent.Multinode != nil && oldComponent.Multinode != nil && newComponent.Multinode.NodeCount != oldComponent.Multinode.NodeCount { @@ -939,6 +939,9 @@ func validateComponentRolesUpdate( oldComponent *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, fldPath *field.Path, ) field.ErrorList { + if newComponent.IsLPX() || oldComponent.IsLPX() { + return nil + } // Permit representation-only implicit/explicit migrations, but require // role-specific configuration changes to happen in a subsequent update. if (newComponent.Roles == nil) != (oldComponent.Roles == nil) { From 11c373ba4c7ee09de83f4fd463bb5c66818e2737 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:25:01 +0300 Subject: [PATCH 036/196] refactor(operator): keep PCS absence in workload convergence (#461) Signed-off-by: Andrew Paprotsky (cherry picked from commit ddbd8c09e77f141729aeb72193c002563bd80c1a) Signed-off-by: Andrew Paprotsky --- .../dgd_grove_workloads_reconciler.go | 40 ++++++++----- .../dgd_grove_workloads_reconciler_test.go | 57 +++++++++++++++++++ 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index ebc590cd3301..1f22e95a8fb8 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -95,18 +95,14 @@ func (r *groveWorkloadsReconciler) Reconcile( logger.Error(err, "failed to generate the Grove GangSet") return ReconcileResult{}, fmt.Errorf("failed to generate the Grove GangSet: %w", err) } - // Retire the exact owned observation when the desired ordinary workload is absent. + + // Converge the ordinary PCS before rollout or readiness observation. + syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) + if err != nil { + return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) + } if renderedPodCliqueSet.desired == nil { - if existing := renderedPodCliqueSet.existing; existing != nil { - if !metav1.IsControlledBy(existing, source) { - return ReconcileResult{}, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) - } - if existing.DeletionTimestamp.IsZero() { - uid, version := existing.UID, existing.ResourceVersion - if err := r.syncer.Delete(ctx, existing, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); client.IgnoreNotFound(err) != nil { - return ReconcileResult{}, err - } - } + if syncedPodCliqueSet != nil { return ReconcileResult{State: nvidiacomv1beta1.DGDStatePending, Reason: "RemovingEmptyPodCliqueSet", Message: "Waiting for the removed ordinary workload"}, nil } stableResources, err := r.stableResources.Reconcile(ctx, source, renderedPodCliqueSet.renderDeployment) @@ -115,10 +111,6 @@ func (r *groveWorkloadsReconciler) Reconcile( } return checkResourcesReadiness(stableResources), nil } - syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) - if err != nil { - return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) - } // Defer commit if the PCS was written this reconcile; the informer hasn't caught up yet. if workerHashTransition.needsCommit() && !pcsWasWritten { // Pre-existing legacy PCS without a suffix is the only case where unstamped is valid. @@ -167,11 +159,29 @@ func (r *groveWorkloadsReconciler) Reconcile( return result, nil } +// reconcilePodCliqueSet returns the current PCS and whether it was created or updated. +// A nil desired PCS retires the observation, retaining it until deletion is observed. func (r *groveWorkloadsReconciler) reconcilePodCliqueSet( ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment, rendered *grovePodCliqueSetRender, ) (*grovev1alpha1.PodCliqueSet, bool, error) { + // Retire only the exact owned observation when no ordinary workload is desired. + if rendered.desired == nil { + if existing := rendered.existing; existing != nil { + if !metav1.IsControlledBy(existing, dgd) { + return nil, false, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) + } + if existing.DeletionTimestamp.IsZero() { + uid, version := existing.UID, existing.ResourceVersion + if err := r.syncer.Delete(ctx, existing, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); client.IgnoreNotFound(err) != nil { + return nil, false, err + } + } + } + return rendered.existing, false, nil + } + // Compose opaque provider fields only at the serialization boundary. desiredProviderObject, err := provideroverride.ComposeGroveOverrides(dgd, rendered.desired) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 6584a3c8ad4f..013fffea51f3 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -179,6 +179,63 @@ func TestGroveWorkloadsReconcilerUsesStableReadinessWithoutOrdinaryPodCliqueSet( require.Empty(t, pcsList.Items) } +func TestReconcilePodCliqueSetAbsent(t *testing.T) { + const deleting = "deleting" + for _, state := range []string{"absent", "owned", deleting, "foreign", "stale"} { + t.Run(state, func(t *testing.T) { + t.Log("Observe an ordinary PCS that is no longer desired") + source := &nvidiacomv1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: "graph", Namespace: "default", UID: "dgd-uid", + }} + observed := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: "graph", Namespace: source.Namespace, UID: "pcs-uid", ResourceVersion: "1", + Finalizers: []string{"hold-deletion"}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, + }} + if state == deleting { + observed.DeletionTimestamp = ptr.To(metav1.Now()) + } + if state == "foreign" { + observed.OwnerReferences[0].UID = "another-dgd" + } + builder := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)) + if state == "absent" { + observed = nil + } else { + builder.WithObjects(observed) + } + kube := builder.Build() + if state == "stale" { + observed.ResourceVersion = "0" + } + before := observed.DeepCopy() + reconciler := &groveWorkloadsReconciler{syncer: newDGDResourceSyncer(kube, nil)} + + t.Log("Converge absence in the PCS reconciler without changing the observation") + pcs, _, err := reconciler.reconcilePodCliqueSet(t.Context(), source, &grovePodCliqueSetRender{existing: observed}) + require.Equal(t, before, observed) + if state == "foreign" { + require.ErrorContains(t, err, "foreign") + } else if state == "stale" { + require.True(t, apierrors.IsConflict(err), "expected deletion precondition conflict, got %v", err) + } else { + require.NoError(t, err) + require.Equal(t, observed, pcs) + } + + t.Log("Only an owned current observation may start deletion") + live := &grovev1alpha1.PodCliqueSet{} + err = kube.Get(t.Context(), client.ObjectKeyFromObject(source), live) + if state == "absent" { + require.True(t, apierrors.IsNotFound(err)) + } else { + require.NoError(t, err) + require.Equal(t, state == "owned" || state == deleting, !live.DeletionTimestamp.IsZero()) + } + }) + } +} + func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFails(t *testing.T) { tests := []struct { name string From f63c48eae0a205018b8ea6ca74b4a539498e99d1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:25:02 +0300 Subject: [PATCH 037/196] fix(operator): preserve Cyborg partition zero (#462) Signed-off-by: Andrew Paprotsky (cherry picked from commit 52859a71822e7136f9d4dd64126f54af43a91572) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 12 ++-- .../dynamo/lpx/workload_projection_v2.go | 19 ++----- .../dynamo/lpx/workload_projection_v2_test.go | 57 ++++++++++--------- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index f7b5e4968385..8fe2cd112c41 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1155,10 +1155,12 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NoError(t, err) require.Empty(t, plan.ConductorTemplate) require.Len(t, plan.Agents, 1) + require.Equal(t, 6, plan.Agents[0].Replicas) request := projection.RequestSpec(dgd.Namespace, "test", nil) - require.Len(t, request.Partitions, 2) - require.Equal(t, int64(1), request.Partitions[0].CompilerPartitionID) - require.Equal(t, int64(2), request.Partitions[1].CompilerPartitionID) + require.Len(t, request.Partitions, 3) + for index, partition := range request.Partitions { + require.Equal(t, int64(index), partition.CompilerPartitionID) + } controllerConfig := &configv1alpha1.OperatorConfiguration{ Infrastructure: configv1alpha1.InfrastructureConfiguration{ETCDAddress: "etcd", NATSAddress: "nats"}, @@ -1316,9 +1318,9 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, decodeConfigName, tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) require.Equal(t, "tokenizer_dir", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Key) - t.Log("Verify the preserved runtime and LPX projection use the same filtered physical partition view") + t.Log("Keep partition zero as the collapsed runtime root for all three physical partitions") lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, lpuConfigName) - require.Equal(t, "1", lpuConfig.Data["partition_ids"]) + require.Equal(t, "0", lpuConfig.Data["partition_ids"]) require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", lpuConfig.Data["gas_dir"]) t.Log("Verify generated Cyborg and LPU config resource order and contents") diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index 33194e2175e6..1549fb42d20f 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -56,17 +56,13 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu if err := normalizeLegacyNovaSettings(modelSettings); err != nil { return nil, fmt.Errorf("resolving configured V2 build: %w", err) } - configured.runtimeSettings = modelSettings - } - allocationMetadata := json.RawMessage(`{}`) - - // Hybrid embeddings always come from Cyborg, after validating authored settings. - if !usesResolvedRuntime { + // Validate the Nova setting without removing Cyborg's required partition zero. if _, err := boolSetting(modelSettings, "cpu_embeddings"); err != nil { return nil, fmt.Errorf("model settings.%w", err) } - configured.omitStandaloneEmbeddingPartition() + configured.runtimeSettings = modelSettings } + allocationMetadata := json.RawMessage(`{}`) // Bind the V2 workload and runtime contract into projection identity before partition validation. transcripts := newModelProjectionTranscripts(intent, v2ProjectionVersion) @@ -105,16 +101,9 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu } // Preserve physical scheduler partitions while collapsing selected chains only in LPU runtime state. if intent.Pipeline == PipelineLPX && len(configured.SelectedPropSyncChains) != 0 { - omittedSourceZero := len(partitions) < len(connectorBuild.Partitions) runtimeChainByRoot := make(map[int][]int, len(connectorBuild.SelectedPropSyncChains)) for _, chain := range connectorBuild.SelectedPropSyncChains { - runtimeChain := chain - if omittedSourceZero && runtimeChain[0] == 0 { - runtimeChain = runtimeChain[1:] - } - if len(runtimeChain) >= 2 { - runtimeChainByRoot[runtimeChain[0]] = runtimeChain - } + runtimeChainByRoot[chain[0]] = chain } collapsed := make([]BuildPartition, 0, len(partitions)) for partitionIndex := 0; partitionIndex < len(partitions); { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index 94dc3630267b..bb3a7511bc23 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -162,7 +162,7 @@ func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { require.Equal(t, int64(0), onLPU.RequestSpec("test", "agents", nil).Partitions[0].CompilerPartitionID) } -func TestProjectModelV2StrictHybridForcesCPUEmbeddingsAfterValidation(t *testing.T) { +func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { t.Log("Prepare a hybrid V2 build with a standalone embedding partition") normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) build := normalized.build @@ -173,21 +173,21 @@ func TestProjectModelV2StrictHybridForcesCPUEmbeddingsAfterValidation(t *testing build.SupportsCPUEmbeddings = true build.SelectedPropSyncChains = nil - t.Log("Force strict-hybrid CPU embeddings after accepting an explicit false setting") - projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineLPX, - Models: []string{"default"}, - BuildSnapshot: normalized, - ModelSettings: json.RawMessage(`{"cpu_embeddings":false}`), - }) - require.NoError(t, err) - projection := projectionBatch[0] - spec := projection.RequestSpec("test", "agents", nil) - require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) - require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) + t.Log("Keep Cyborg's partition zero regardless of the Nova CPU-embedding setting") + for _, settings := range []string{"", `{"cpu_embeddings":false}`, `{"cpu_embeddings":true}`} { + projection := projectTestBuild(t, normalized, PipelineLPX, settings) + spec := projection.RequestSpec("test", "agents", nil) + require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) + require.Len(t, spec.Partitions, 2) + require.Equal(t, int64(0), spec.Partitions[0].CompilerPartitionID) + require.Equal(t, 4, projection.agentReplicas) + data := resolvedPartitionData([]*ModelProjection{projection}) + require.Equal(t, "0\n1", data["partition_ids"]) + require.Equal(t, "0\n2", data["partition_node_offsets"]) + } - t.Log("Reject a malformed setting before applying the strict-hybrid override") - _, err = appendModelProjections(nil, ModelProjectionInput{ + t.Log("Still reject a malformed setting") + _, err := appendModelProjections(nil, ModelProjectionInput{ Pipeline: PipelineLPX, Models: []string{"default"}, BuildSnapshot: normalized, @@ -287,13 +287,13 @@ func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing. require.NotSame(t, connectors[0].Requirement.MaxInterPartitionOffset, connectors[1].Requirement.MaxInterPartitionOffset) } -func TestProjectModelV2CollapsesSelectedChainAfterOmittingEmbeddingPrefix(t *testing.T) { +func TestProjectModelV2CollapsesSelectedChainIncludingPartitionZero(t *testing.T) { t.Parallel() normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) build := normalized.build build.CompilationMode = BuildCompilationModeHybrid - t.Log("Model the normalized adjacent 0/1/2 chain whose standalone embedding root is omitted") + t.Log("Model a selected chain starting at Cyborg's standalone embedding partition") topology, err := build.Partitions[0].Topology.withChipCount(8) require.NoError(t, err) partitions := make([]BuildPartition, 3) @@ -308,21 +308,22 @@ func TestProjectModelV2CollapsesSelectedChainAfterOmittingEmbeddingPrefix(t *tes build.StandaloneTokenEmbeddings = true build.SupportsCPUEmbeddings = true - t.Log("Project after omitting the standalone embedding prefix") + t.Log("Project without applying Nova's CPU-embedding partition omission") projection := projectTestBuild(t, normalized, PipelineLPX, "") - t.Log("Keep physical scheduler partitions 1/2 while collapsing their Agent runtime projection") + t.Log("Keep all physical scheduler partitions while collapsing their Agent runtime projection") spec := projection.RequestSpec("test", "agents", nil) - require.Len(t, spec.Partitions, 2) - require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) - require.Equal(t, int64(2), spec.Partitions[1].CompilerPartitionID) - require.Len(t, spec.PropSyncConnectors, 1) - require.Equal(t, 2, projection.agentReplicas) + require.Len(t, spec.Partitions, 3) + for index, partition := range spec.Partitions { + require.Equal(t, int64(index), partition.CompilerPartitionID) + } + require.Len(t, spec.PropSyncConnectors, 2) + require.Equal(t, 3, projection.agentReplicas) data := resolvedPartitionData([]*ModelProjection{projection}) - require.Equal(t, "1", data["partition_ids"]) - require.Equal(t, "2", data["nodes_per_partition"]) - require.Equal(t, "part-1", data["partition_paths"]) - require.Equal(t, 16, projection.configuredBuild.Partitions[0].Topology.ChipCount) + require.Equal(t, "0", data["partition_ids"]) + require.Equal(t, "3", data["nodes_per_partition"]) + require.Equal(t, "part-0", data["partition_paths"]) + require.Equal(t, 24, projection.configuredBuild.Partitions[0].Topology.ChipCount) require.Empty(t, projection.configuredBuild.SelectedPropSyncChains) require.Equal(t, [][]int{{0, 1, 2}}, build.SelectedPropSyncChains) } From b6223ddaf658e137f2b463ba96931ad57e966af3 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:25:02 +0300 Subject: [PATCH 038/196] test(operator): preserve projected Cyborg endpoints (#464) Signed-off-by: Andrew Paprotsky (cherry picked from commit b2ea63973249f1df646940cb9dd312e6b56c50bb) Signed-off-by: Andrew Paprotsky --- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 10f2d30702d6..110121609599 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -86,3 +86,55 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Verify missing tokenizer metadata is rejected") require.EqualError(t, err, "capnp manifest build is missing model.tokenizer.path") } + +func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { + t.Parallel() + + t.Log("Cover the legacy launcher's 14 endpoints and selected PropSync chain roots") + for _, test := range []struct { + name string + partitions int + chains [][]uint32 + wantOffsets []int + }{ + {name: "legacy 14 endpoints", partitions: 14, wantOffsets: []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26}}, + {name: "chain followed by endpoint", partitions: 4, chains: [][]uint32{{1, 2}}, wantOffsets: []int{0, 2, 6}}, + {name: "two chains followed by endpoint", partitions: 5, chains: [][]uint32{{0, 1}, {2, 3}}, wantOffsets: []int{0, 4, 8}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + t.Log("Project a hybrid compiler inventory with two-node physical partitions") + fixture := newV2CompilerFixture() + partition := fixture.partitions[0] + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + fixture.numLPUNodes = uint32(2 * test.partitions) + fixture.selectedPropSyncChains = test.chains + fixture.partitions = make([]testV3CapnpPartition, test.partitions) + for index := range fixture.partitions { + fixture.partitions[index] = partition + fixture.partitions[index].id = uint32(index) + } + snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) + projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineLPX, `{"prop_sync":false}`) + projection.stage = testRenderComponentName + + t.Log("Keep every physical scheduler partition and Agent, including chain followers") + require.Len(t, projection.RequestSpec("test", "agents", nil).Partitions, test.partitions) + require.Equal(t, 2*test.partitions, projection.agentReplicas) + + t.Log("Render only projected endpoints without compressing their physical Agent offsets") + workload := &SelectedWorkload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1} + plan, err := workload.PlanNodeLocalMaterialization("test-dgd") + require.NoError(t, err) + configMap, err := workload.RenderCyborgConfigMap("test", "test-dgd", renderTestPodSpec()) + require.NoError(t, err) + prefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" + servers := make([]string, len(test.wantOffsets)) + for index, offset := range test.wantOffsets { + servers[index] = prefix + strconv.Itoa(offset) + } + require.Equal(t, strings.Join(servers, "\n"), configMap.Data["lpu_servers"]) + }) + } +} From c4bf30eb6dc3d29621c2f64aee38385be419deee Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:25:02 +0300 Subject: [PATCH 039/196] docs(operator): correct LPX replica limits (#466) Signed-off-by: Andrew Paprotsky (cherry picked from commit 40a8f9679fa6b1ff48545e029752361a981f5511) Signed-off-by: Andrew Paprotsky --- .../reference/kubernetes-api/dynamo-component-deployment.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 9d67a1bedd87..387f48a411e4 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -67,7 +67,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Desired number of complete component instances. For a single-node component, each instance is one pod. For a multinode component, each instance contains all of its roles. Minimum `0`. When `scalingAdapter` is set, this field is owned by the [DynamoGraphDeploymentScalingAdapter](full-api-reference.mdx#dynamographdeploymentscalingadapter) and should not be modified directly. - For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid engines support `1–9` fixed replicas; LPU-only requires `1`. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. + For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid and LPU-only engines allow up to `2496` replicas within the 512 KiB scheduling-status budget; generated Pod hostname limits may lower that maximum. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. From d692870c037766e64cdc897769dd69e1103fa18e Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:48:10 +0300 Subject: [PATCH 040/196] fix(operator): fence PCSG readiness by controller UID (#467) Signed-off-by: Andrew Paprotsky (cherry picked from commit 662a258b884c58fef5568719f6e8317fd4062297) Signed-off-by: Andrew Paprotsky --- .../dynamographdeployment_controller_test.go | 15 ++++ deploy/operator/internal/dynamo/grove.go | 24 ++++-- deploy/operator/internal/dynamo/grove_test.go | 84 ++++++++++++++++++- 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 57ce547a2adb..ac8b785e76ed 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -779,6 +779,10 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(replica), commonconsts.KubeLabelDynamoComponent: component, }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup", + Name: pcsg, UID: types.UID(pcsg), Controller: ptr.To(true), + }}, }, Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, Status: grovev1alpha1.PodCliqueStatus{ @@ -1186,10 +1190,21 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { pcs := &grovev1alpha1.PodCliqueSet{} g.Expect(fakeKubeClient.Get(ctx, client.ObjectKey{Name: "test-dgd", Namespace: "default"}, pcs)).To(gomega.Succeed()) + pcs.UID = "current-pcs" pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) pcs.Status.CurrentGenerationHash = ptr.To("current") g.Expect(fakeKubeClient.Update(ctx, pcs)).To(gomega.Succeed()) + t.Log("Record the current PCS controller identity on Grove's scaling groups") + for _, object := range tt.existingGroveResources { + if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { + g.Expect(fakeKubeClient.Get(ctx, client.ObjectKeyFromObject(group), group)).To(gomega.Succeed()) + group.UID = types.UID(group.Name) + group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + g.Expect(fakeKubeClient.Update(ctx, group)).To(gomega.Succeed()) + } + } + result, err = reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index 99f0e700a54d..4ead8af4b3df 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -152,17 +152,20 @@ func observeComponentPCSGReadiness( ) (groveComponentReadiness, error) { logger := log.FromContext(ctx) pcsg := &grovev1alpha1.PodCliqueScalingGroup{} - if err := reader.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, pcsg); err != nil { - if errors.IsNotFound(err) { - status := v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{resourceName}, - } - return groveComponentReadiness{status: status}.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } + err := reader.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, pcsg) + if err != nil && !errors.IsNotFound(err) { return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) } + // A cached group from a previous PCS instance cannot contribute readiness or status. + if errors.IsNotFound(err) || pcs == nil || !metav1.IsControlledBy(pcsg, pcs) { + status := v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{resourceName}, + } + return groveComponentReadiness{status: status}.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil + } + // Reuse the native observation while retaining the component's requested capacity for cutover. desiredReplicas := ptr.Deref(dgdReplicas, pcsg.Spec.Replicas) componentReadiness := pcsgReadiness(pcsg, logger) @@ -206,6 +209,11 @@ func observeComponentPCSGReadiness( podCliquesByReplica := make(map[int][]grovev1alpha1.PodClique) for _, clique := range podCliqueList.Items { + // Matching labels do not establish ownership of a recreated group's children. + if !metav1.IsControlledBy(&clique, pcsg) { + continue + } + index, err := strconv.Atoi(clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex]) if err != nil { logger.V(1).Info("Failed to read PodClique metadata", "error", err, "resourceName", clique.Name) diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index 0533507bfa63..ab063b8056a6 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -759,6 +760,10 @@ func TestEvaluateGroveReadiness(t *testing.T) { grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(groupReplica), commonconsts.KubeLabelDynamoComponent: component, }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup", + Name: pcsg, UID: types.UID(pcsg), Controller: ptr.To(true), + }}, }, Spec: grovev1alpha1.PodCliqueSpec{Replicas: replicas}, Status: grovev1alpha1.PodCliqueStatus{ @@ -1132,6 +1137,14 @@ func TestEvaluateGroveReadiness(t *testing.T) { } } if observedPCS != nil { + t.Log("Give each scaling group the current PCS controller identity") + for _, object := range tt.existingGroveResources { + if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { + group.UID = types.UID(group.Name) + group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(observedPCS, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + } + } + if betaDGD.Status.Components == nil { betaDGD.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) } @@ -1632,6 +1645,8 @@ func TestEvaluateGroveReadinessPublishesAcceptedNamespaceForZeroReplicaWorkers(t CurrentPodCliqueSetGenerationHash: ptr.To(acceptedRevision), }, } + podCliqueSet.UID = "current-pcs" + child.SetOwnerReferences([]metav1.OwnerReference{*metav1.NewControllerRef(podCliqueSet, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}) } else { child = &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{ @@ -1952,6 +1967,10 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { name string pcsg *grovev1alpha1.PodCliqueScalingGroup dgdReplicas *int32 + pcsgReplicas *int32 + oldPCS bool + oldPCSG bool + foreignClique bool wantReady bool wantClassification string wantReasonContains string @@ -1974,6 +1993,42 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { }), wantReady: true, }, + { + name: "ready scaling group belongs to the previous same-name PCS", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, + ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }), + oldPCS: true, + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + }, + { + name: "zero-replica scaling group belongs to the previous same-name PCS", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + ObservedGeneration: ptr.To(int64(1)), + }), + pcsgReplicas: ptr.To(int32(0)), + oldPCS: true, + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + }, + { + name: "ready cliques belong to the previous same-name scaling group", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, + ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }), + oldPCSG: true, + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + }, + { + name: "foreign clique with a malformed replica label does not affect readiness", + pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, + ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }), + foreignClique: true, + wantReady: true, + }, { name: "pinned DGD replicas take precedence over PCSG replicas", pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ @@ -2061,11 +2116,22 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := gomega.NewGomegaWithT(t) + t.Log("Build the observed PCS, scaling group, and owned ready cliques") pcs := newReadyPCS("pcs") objs := []client.Object{pcs} if tt.pcsg != nil { + tt.pcsg.UID = "current-pcsg" + tt.pcsg.Spec.Replicas = ptr.Deref(tt.pcsgReplicas, tt.pcsg.Spec.Replicas) + tt.pcsg.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + if tt.oldPCS { + tt.pcsg.OwnerReferences[0].UID = "previous-pcs" + } tt.pcsg.Status.CurrentPodCliqueSetGenerationHash = ptr.To("current") objs = append(objs, tt.pcsg) + cliqueOwner := *metav1.NewControllerRef(tt.pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup")) + if tt.oldPCSG { + cliqueOwner.UID = "previous-pcsg" + } for replica := range testPCSGReplicas { objs = append(objs, &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{ @@ -2077,6 +2143,7 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(replica), commonconsts.KubeLabelDynamoComponent: "component", }, + OwnerReferences: []metav1.OwnerReference{cliqueOwner}, }, Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, Status: grovev1alpha1.PodCliqueStatus{ @@ -2088,15 +2155,30 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { }, }) } + if tt.foreignClique { + foreign := objs[len(objs)-1].(*grovev1alpha1.PodClique).DeepCopy() + foreign.Name = "foreign-clique" + foreign.OwnerReferences[0].UID = "previous-pcsg" + foreign.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = "invalid" + objs = append(objs, foreign) + } } + + t.Log("Evaluate readiness against the current controller identities") c := newFakeGroveClient(g, objs...) readiness, checkErr := observeComponentPCSGReadiness( ctx, c, pcs, testPCSGName, "default", "component", tt.dgdReplicas, ) g.Expect(checkErr).NotTo(gomega.HaveOccurred()) + t.Log("Verify foreign resources cannot contribute readiness or replica status") g.Expect(readiness.ready).To(gomega.Equal(tt.wantReady)) g.Expect(readiness.classification).To(gomega.Equal(tt.wantClassification)) + if tt.oldPCS { + g.Expect(readiness.status).To(gomega.Equal(v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{testPCSGName}, + })) + } if tt.wantReasonContains != "" { g.Expect(readiness.reason).To(gomega.ContainSubstring(tt.wantReasonContains)) } @@ -2136,7 +2218,7 @@ func newPCSG(status grovev1alpha1.PodCliqueScalingGroupStatus) *grovev1alpha1.Po func newReadyPCS(name string) *grovev1alpha1.PodCliqueSet { return &grovev1alpha1.PodCliqueSet{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1, UID: types.UID(name)}, Status: grovev1alpha1.PodCliqueSetStatus{ ObservedGeneration: ptr.To(int64(1)), CurrentGenerationHash: ptr.To("current"), From 59134d6f442a4c7c83466d0b564c8a213ffba7f7 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:48:11 +0300 Subject: [PATCH 041/196] fix(operator): verify the LPX PodGang API prerequisite (#468) Signed-off-by: Andrew Paprotsky (cherry picked from commit 801e93e8119979aa81fd73f52709656aac6ae520) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 2 +- .../platform/components/operator/values.yaml | 6 +- deploy/helm/charts/platform/values.yaml | 6 +- deploy/operator/api/config/v1alpha1/lpx.go | 5 +- deploy/operator/internal/features/lpx.go | 17 +++- deploy/operator/internal/features/lpx_test.go | 81 +++++++++++++++++++ .../additional-resources/api-reference-k8s.md | 2 +- .../kubernetes-api/full-api-reference.mdx | 2 +- 8 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 deploy/operator/internal/features/lpx_test.go diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index b07982c39e81..9e9c58262bd2 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -170,7 +170,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | -| dynamo-operator.lpx.enabled | bool | `false` | Enable LPX integration. Startup verifies that the externally installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails otherwise. | +| dynamo-operator.lpx.enabled | bool | `false` | Enable LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs. | | dynamo-operator.lpx.modelRegistryURL | string | `""` | Configure the LPU model registry used by the LPX integration. | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. | diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 7fb30cf2ef58..79f6404b551a 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -25,9 +25,9 @@ upgradeCRD: true # LPX is an externally installed scheduler integration. Enabling it only # configures the Dynamo operator; this chart never installs LPX resources. lpx: - # -- Enable node-local LpuPipelineRequest production. Startup verifies that the - # externally installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 - # LpuPipelineRequest resources and fails otherwise. + # -- Enable node-local LpuPipelineRequest production. Startup requires the + # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest + # and scheduler.grove.io/v1alpha1 PodGang APIs. enabled: false # -- Configure the LPU model registry used by the LPX integration. modelRegistryURL: "" diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 4fe26587fba2..747e89bfae4a 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -94,9 +94,9 @@ dynamo-operator: # LPX is installed and operated separately. These values only opt the # Dynamo operator into producing node-local scheduling requests. lpx: - # -- Enable LPX integration. Startup verifies that the externally installed - # LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest - # resources and fails otherwise. + # -- Enable LPX integration. Startup requires the externally installed + # scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and + # scheduler.grove.io/v1alpha1 PodGang APIs. enabled: false # -- Configure the LPU model registry used by the LPX integration. modelRegistryURL: "" diff --git a/deploy/operator/api/config/v1alpha1/lpx.go b/deploy/operator/api/config/v1alpha1/lpx.go index 2a6ba00e94bf..a18920528add 100644 --- a/deploy/operator/api/config/v1alpha1/lpx.go +++ b/deploy/operator/api/config/v1alpha1/lpx.go @@ -6,9 +6,8 @@ package v1alpha1 // LPXConfiguration holds LPX scheduler integration and model-registry settings. type LPXConfiguration struct { // Enabled opts the operator into LPX request production. When true, startup - // verifies that the separately installed LPX API serves - // scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails - // otherwise. + // requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and + // scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. // +kubebuilder:default=false Enabled bool `json:"enabled"` diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go index 755ccbd8b886..24b141cfac18 100644 --- a/deploy/operator/internal/features/lpx.go +++ b/deploy/operator/internal/features/lpx.go @@ -7,6 +7,7 @@ import ( "context" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "k8s.io/client-go/rest" "k8s.io/utils/ptr" ) @@ -20,7 +21,7 @@ import ( // Configuration: lpx.enabled // Auto-detection: N/A; API availability is verified when explicitly enabled // Requires: LPX Scheduler serving scheduling.lpu.nvidia.com/v1alpha1 -// LpuPipelineRequest resources +// LpuPipelineRequest resources and scheduler.grove.io/v1alpha1 PodGang resources // Default: false const LPX Name = "lpx" @@ -34,6 +35,18 @@ func resolveLPX(ctx context.Context, config *rest.Config) (bool, error) { return false, err } + if _, err := resolve(ptr.To(true), available, + "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster"); err != nil { + return false, err + } + + // LPX also watches scheduler-scoped PodGangs, which external Grove installs may omit. + resource = groveschedulerv1alpha1.SchemeGroupVersion.WithResource("podgangs") + available, err = detectAPIAvailability(ctx, config, resource.Group, resource.Version, resource.Resource) + if err != nil { + return false, err + } + return resolve(ptr.To(true), available, - "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster") + "LPX is explicitly enabled in config but the scheduler.grove.io/v1alpha1 PodGang API was not detected in the cluster") } diff --git a/deploy/operator/internal/features/lpx_test.go b/deploy/operator/internal/features/lpx_test.go new file mode 100644 index 000000000000..f4b4572d20f7 --- /dev/null +++ b/deploy/operator/internal/features/lpx_test.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package features + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" +) + +func TestResolveLPX(t *testing.T) { + const ( + lprPath = "/apis/scheduling.lpu.nvidia.com/v1alpha1" + podGangPath = "/apis/scheduler.grove.io/v1alpha1" + missingLPR = "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster" + missingPodGang = "LPX is explicitly enabled in config but the scheduler.grove.io/v1alpha1 PodGang API was not detected in the cluster" + ) + tests := []struct { + name, path string + status int + resource, wantErr string + }{ + {name: "both APIs present"}, + {name: "LPR resource absent", path: lprPath, status: http.StatusOK, wantErr: missingLPR}, + {name: "PodGang group version absent", path: podGangPath, status: http.StatusNotFound, wantErr: missingPodGang}, + {name: "PodGang resource absent", path: podGangPath, status: http.StatusOK, wantErr: missingPodGang}, + {name: "PodGang subresource alone is insufficient", path: podGangPath, status: http.StatusOK, resource: "podgangs/status", wantErr: missingPodGang}, + {name: "PodGang discovery forbidden", path: podGangPath, status: http.StatusForbidden, wantErr: "discover scheduler.grove.io/v1alpha1 API resources: Forbidden"}, + {name: "PodGang discovery fails", path: podGangPath, status: http.StatusInternalServerError, wantErr: "discover scheduler.grove.io/v1alpha1 API resources: Internal Server Error"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Serve the exact LPX and PodGang API discovery paths") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resource := "lpupipelinerequests" + switch r.URL.Path { + case lprPath: + case podGangPath: + resource = "podgangs" + default: + t.Errorf("unexpected discovery path %q", r.URL.Path) + http.NotFound(w, r) + return + } + status := http.StatusOK + if r.URL.Path == tt.path { + status, resource = tt.status, tt.resource + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + var response any = metav1.Status{TypeMeta: metav1.TypeMeta{Kind: "Status", APIVersion: "v1"}, Status: metav1.StatusFailure, Code: int32(status), Message: http.StatusText(status)} + if status == http.StatusOK { + resources := []metav1.APIResource{} + if resource != "" { + resources = append(resources, metav1.APIResource{Name: resource}) + } + response = metav1.APIResourceList{TypeMeta: metav1.TypeMeta{Kind: "APIResourceList", APIVersion: "v1"}, GroupVersion: strings.TrimPrefix(r.URL.Path, "/apis/"), APIResources: resources} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("write discovery response: %v", err) + } + })) + defer server.Close() + + t.Log("Require both APIs and preserve discovery failures") + enabled, err := resolveLPX(t.Context(), &rest.Config{Host: server.URL}) + if enabled != (tt.wantErr == "") || (err == nil) != (tt.wantErr == "") { + t.Fatalf("resolveLPX() = %v, %v; want enabled=%v, error=%q", enabled, err, tt.wantErr == "", tt.wantErr) + } + if err != nil && err.Error() != tt.wantErr { + t.Errorf("resolveLPX() error = %q, want %q", err, tt.wantErr) + } + }) + } +} diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 7c6b3ae165a3..ffd9e2e0a884 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3444,7 +3444,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `enabled` _boolean_ | Enabled opts the operator into LPX request production. When true, startup
verifies that the separately installed LPX API serves
scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails
otherwise. | false | | +| `enabled` _boolean_ | Enabled opts the operator into LPX request production. When true, startup
requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and
scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. | false | | | `modelRegistryURL` _string_ | ModelRegistryURL configures the location of the LPU model registry used by LPX.
Supported values are an absolute local path (or file:// URL) and gs:// URLs. | | | diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index ee44a7387e7a..751d94ca5564 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4677,7 +4677,7 @@ LPXConfiguration holds LPX scheduler integration and model-registry settings. **Appears in:** [OperatorConfiguration](#operatorconfiguration) -Enabled opts the operator into LPX request production. When true, startup verifies that the separately installed LPX API serves scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest resources and fails otherwise. +Enabled opts the operator into LPX request production. When true, startup requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. From 7e25841921c9c351a127e5c8bc1f34a95a544f99 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 12 Sep 2026 19:38:02 +0000 Subject: [PATCH 042/196] fix(operator): reconcile LPX staging with main Restore included files to nvidia-lpu/lpu-dynamo main 801e93e8119979aa81fd73f52709656aac6ae520, including schema compaction and its generated artifacts and tests. Keep upstream CI, private-tooling and unrelated planner exclusions unchanged. Signed-off-by: Andrew Paprotsky --- .../platform/templates/validate-values.yaml | 2 +- .../platform/tests/lpx_config_test.yaml | 2 +- deploy/operator/Makefile | 16 +- deploy/operator/api/scripts/lpx-crd-schema.yq | 19 + deploy/operator/api/v1alpha1/common.go | 11 +- .../api/v1alpha1/zz_generated.deepcopy.go | 10 +- deploy/operator/api/v1beta1/common.go | 11 +- .../api/v1beta1/zz_generated.deepcopy.go | 10 +- deploy/operator/cmd/crd-apply/main_test.go | 8 +- ...nvidia.com_dynamocomponentdeployments.yaml | 10 +- .../nvidia.com_dynamographdeployments.yaml | 12419 +--------------- deploy/operator/go.mod | 2 +- deploy/operator/go.sum | 4 +- deploy/operator/internal/common/podspec.go | 68 + .../controller/dgd_lpx_handoff_test.go | 11 +- .../internal/dynamo/lpx/cyborg_render.go | 3 +- .../dynamo/lpx/hybrid_cyborg_bindings.go | 3 +- .../internal/dynamo/lpx/lpu_configmap.go | 5 +- .../operator/internal/dynamo/lpx/pod_roles.go | 13 +- deploy/operator/internal/dynamo/lpx/render.go | 3 +- .../dynamo/lpx/resolve_selected_workload.go | 3 +- .../internal/dynamo/lpx/runtime_agent.go | 5 +- ...graphdeployment_validation_envtest_test.go | 10 + .../webhook/validation/shared_v1beta1.go | 57 +- .../additional-resources/api-reference-k8s.md | 4 +- .../dynamo-component-deployment.mdx | 14 +- .../kubernetes-api/full-api-reference.mdx | 24 +- 27 files changed, 215 insertions(+), 12532 deletions(-) create mode 100644 deploy/operator/internal/common/podspec.go diff --git a/deploy/helm/charts/platform/templates/validate-values.yaml b/deploy/helm/charts/platform/templates/validate-values.yaml index fb530ecf85c7..c6bbc901cffa 100644 --- a/deploy/helm/charts/platform/templates/validate-values.yaml +++ b/deploy/helm/charts/platform/templates/validate-values.yaml @@ -151,7 +151,7 @@ kai-scheduler: (index .Values "dynamo-operator").lpx.enabled }} {{- fail ` -CONFIGURATION ERROR: The bundled Grove v0.1.0-alpha.13 does not support +CONFIGURATION ERROR: The bundled Grove chart does not support the scheduler-scoped PodGangs required by LPX workloads. Install the scheduler-scoped Grove build separately, then configure: diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index 2886c4dc30e8..765ad2972ed5 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -15,7 +15,7 @@ tests: dynamo-operator.lpx.enabled: true asserts: - failedTemplate: - errorPattern: "bundled Grove v0.1.0-alpha.13 does not support" + errorPattern: "bundled Grove chart does not support" - it: rejects LPX without Grove integration template: charts/dynamo-operator/templates/validate-values.yaml diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index ba818bc8f16c..36aba16626cd 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -46,6 +46,8 @@ CONTAINER_TOOL ?= docker SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +PYTHON ?= uv run --no-project --python '3.10' --with pydantic --with pyyaml --with black python3 + .PHONY: all all: build @@ -282,20 +284,20 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C $(YQ) eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \ fi; \ done - # Remove DGD-only LPX fields and rules from the standalone DCD schema. - $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml - + # Compact LPX schemas last: subsequent schema edits could change an aliased role. + for file in config/crd/bases/nvidia.com_dynamographdeployments.yaml config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml; do \ + $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 $$file || exit 1; \ + done .PHONY: generate generate: controller-gen generate-pydantic ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt_" paths="./..." .PHONY: generate-pydantic generate-pydantic: ## Generate Python Pydantic models from v1beta1 Go types (requires Python 3 + pydantic) - @python3 -c "import pydantic" 2>/dev/null || { echo "Error: pydantic not found. Install with: pip install pydantic"; exit 1; } @echo "Generating Pydantic models from v1beta1 DGDR types..." - @python3 api/scripts/generate_pydantic_from_go.py + $(PYTHON) api/scripts/generate_pydantic_from_go.py @echo "Running Pydantic validation tests..." - @python3 api/scripts/validate_pydantic_models.py + $(PYTHON) api/scripts/validate_pydantic_models.py .PHONY: fmt fmt: ## Run go fmt against code. @@ -662,7 +664,7 @@ generate-api-docs: crd-ref-docs ## Generate API reference documentation from CRD rm ./docs/api_reference.md @echo "✅ Concatenated header.md, api_reference.md, and footer.md" # Post-process duplicate anchors and fields pruned from standalone DCD CRDs. - python3 docs/fix-api-anchors.py ../../docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md + $(PYTHON) docs/fix-api-anchors.py ../../docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md HELM_CHART_DIR := ../helm/charts/platform diff --git a/deploy/operator/api/scripts/lpx-crd-schema.yq b/deploy/operator/api/scripts/lpx-crd-schema.yq index af0f5da3474d..52d17862ff6d 100644 --- a/deploy/operator/api/scripts/lpx-crd-schema.yq +++ b/deploy/operator/api/scripts/lpx-crd-schema.yq @@ -9,4 +9,23 @@ with(select(.metadata.name == "dynamocomponentdeployments.nvidia.com"); (select(has("x-kubernetes-validations"))."x-kubernetes-validations") |= map(select(.rule | test("\\b(self|oldSelf)\\.lpx\\b") | not)) ) +) | + +# Keep native validation and merge semantics; omit only LPX template descriptions. +# Ordinary PodTemplate and alpha extraPodSpec documentation remains unchanged. +del(.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | .. | select(has("description")) | .description) | + +# Compact only LPX templates, after every other schema transformation has finished. +# YAML requires the anchor before every alias: traversal order is not document order. +with(select([.. | select(has("lpx") and has("roles"))] | length > 0); + ([.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + {"path": (path | to_json), "line": line, "column": column}] | sort_by(.line, .column))[0].path as $first | + [.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) == $first) | to_json][0] as $schema | + (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) == $first)) anchor = "lpx-pod-template" | + + # Different role-specific constraints must never be replaced by an alias. + (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | + select((path | to_json) != $first) | select((to_json) == $schema)) alias = "lpx-pod-template" ) diff --git a/deploy/operator/api/v1alpha1/common.go b/deploy/operator/api/v1alpha1/common.go index 4d58e5317559..143f2e6f49fa 100644 --- a/deploy/operator/api/v1alpha1/common.go +++ b/deploy/operator/api/v1alpha1/common.go @@ -89,16 +89,15 @@ type ComponentRoleSpec struct { // +kubebuilder:validation:Minimum=1 Replicas *int32 `json:"replicas,omitempty"` + // PodTemplate configures Pods for this role. The enclosing component + // defines whether the template is required or may be inherited. + // +optional + PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` + // ProviderOverride configures the provider workload unit generated for this // role. It is supported only for components embedded in a DGD. // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` - - // PodTemplate defines the Pod configuration for this role. Admission permits - // it only when the enclosing component type explicitly supports role-specific - // Pod templates. LPX components support role-specific Pod templates. - // +optional - PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } // +kubebuilder:validation:XValidation:rule="!has(self.create) || self.create == false || (has(self.size) && has(self.storageClass) && has(self.volumeAccessMode))",message="When create is true, size, storageClass, and volumeAccessMode are required" diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index bc8d798ff6b5..86ffabd6e81a 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -124,16 +124,16 @@ func (in *ComponentRoleSpec) DeepCopyInto(out *ComponentRoleSpec) { *out = new(int32) **out = **in } - if in.ProviderOverride != nil { - in, out := &in.ProviderOverride, &out.ProviderOverride - *out = new(ProviderOverride) - (*in).DeepCopyInto(*out) - } if in.PodTemplate != nil { in, out := &in.PodTemplate, &out.PodTemplate *out = new(v1.PodTemplateSpec) (*in).DeepCopyInto(*out) } + if in.ProviderOverride != nil { + in, out := &in.ProviderOverride, &out.ProviderOverride + *out = new(ProviderOverride) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentRoleSpec. diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index 09c45e073fb3..4a3294e48ef6 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -134,16 +134,15 @@ type ComponentRoleSpec struct { // +kubebuilder:validation:Minimum=1 Replicas *int32 `json:"replicas,omitempty"` + // podTemplate configures Pods for this role. The enclosing component + // defines whether the template is required or may be inherited. + // +optional + PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` + // providerOverride configures the provider workload unit generated for this // role. It is supported only for components embedded in a DGD. // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` - - // podTemplate defines the Pod configuration for this role. Admission permits - // it only when the enclosing component type explicitly supports role-specific - // Pod templates. LPX components support role-specific Pod templates. - // +optional - PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } // ComponentRole returns the authored role, or nil when the component does not declare it. diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 81e3a38f05d9..51edcf67ce3c 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -184,16 +184,16 @@ func (in *ComponentRoleSpec) DeepCopyInto(out *ComponentRoleSpec) { *out = new(int32) **out = **in } - if in.ProviderOverride != nil { - in, out := &in.ProviderOverride, &out.ProviderOverride - *out = new(ProviderOverride) - (*in).DeepCopyInto(*out) - } if in.PodTemplate != nil { in, out := &in.PodTemplate, &out.PodTemplate *out = new(v1.PodTemplateSpec) (*in).DeepCopyInto(*out) } + if in.ProviderOverride != nil { + in, out := &in.ProviderOverride, &out.ProviderOverride + *out = new(ProviderOverride) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentRoleSpec. diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index 24c72fe53a4f..2af8785036d5 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -106,11 +106,17 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { if name == "dynamocomponentdeployments.nvidia.com" { t.Log("Retain ordinary template schemas and documentation in both served DCD versions") for _, version := range crd.Spec.Versions { + spec := version.Schema.OpenAPIV3Schema.Properties["spec"] + roles := spec.Properties["roles"] + require.Contains(t, roles.Items.Schema.Properties, "replicas") + require.Contains(t, roles.Items.Schema.Properties, "podTemplate") + require.NotContains(t, spec.Properties, "lpx") + templateField := "podTemplate" if version.Name == "v1alpha1" { templateField = "extraPodSpec" } - template := version.Schema.OpenAPIV3Schema.Properties["spec"].Properties[templateField] + template := spec.Properties[templateField] if template.Type != objectType || template.Description == "" { t.Errorf("%s %s is missing the documented %s schema", name, version.Name, templateField) } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 51d94fbfd721..a01677505cb0 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -11492,9 +11492,8 @@ spec: type: string podTemplate: description: |- - PodTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. LPX components support role-specific Pod templates. + PodTemplate configures Pods for this role. The enclosing component + defines whether the template is required or may be inherited. properties: metadata: description: |- @@ -28695,9 +28694,8 @@ spec: type: string podTemplate: description: |- - podTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. LPX components support role-specific Pod templates. + podTemplate configures Pods for this role. The enclosing component + defines whether the template is required or may be inherited. properties: metadata: description: |- diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index fe552efe235a..cc5dd6298568 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11880,16 +11880,9 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: - description: |- - PodTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. LPX components support role-specific Pod templates. + podTemplate: &lpx-pod-template properties: metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata properties: annotations: additionalProperties: @@ -11909,64 +11902,27 @@ spec: type: string type: object spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. format: int64 type: integer affinity: - description: If specified, the pod's scheduling constraints properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -11978,27 +11934,13 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -12012,7 +11954,6 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -12022,43 +11963,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -12070,27 +11986,13 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. properties: key: - description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. items: type: string type: array @@ -12111,51 +12013,22 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12169,70 +12042,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12246,38 +12078,20 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. format: int32 type: integer required: @@ -12287,49 +12101,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12343,70 +12126,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12420,30 +12162,15 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12452,51 +12179,22 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12510,70 +12208,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12587,38 +12244,20 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. format: int32 type: integer required: @@ -12628,49 +12267,18 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running properties: labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12684,70 +12292,29 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -12761,30 +12328,15 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12794,138 +12346,62 @@ spec: type: object type: object automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. type: boolean containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. items: - description: A single application container that you want to run within a pod. properties: args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -12934,44 +12410,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -12986,52 +12447,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13039,67 +12473,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13108,104 +12506,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13214,130 +12566,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13346,129 +12643,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. type: string ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -13479,71 +12709,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -13552,99 +12750,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -13653,34 +12803,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -13696,9 +12825,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -13707,69 +12833,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -13784,261 +12861,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14047,134 +12974,61 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -14185,64 +13039,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -14253,11 +13064,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -14267,203 +13073,87 @@ spec: - name x-kubernetes-list-type: map dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. properties: nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. items: type: string type: array x-kubernetes-list-type: atomic options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. items: - description: PodDNSConfigOption defines DNS resolver options of a pod. properties: name: - description: |- - Name is this DNS resolver option's name. - Required. type: string value: - description: Value is this DNS resolver option's value. type: string type: object type: array x-kubernetes-list-type: atomic searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. items: type: string type: array x-kubernetes-list-type: atomic type: object dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. type: boolean ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. properties: args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -14472,44 +13162,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -14524,52 +13199,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -14577,63 +13225,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: Lifecycle is not allowed for ephemeral containers. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14642,104 +13258,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14748,126 +13318,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -14876,121 +13395,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. type: string ports: - description: Ports are not allowed for ephemeral containers. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -15001,67 +13461,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -15070,97 +13502,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: Resources resize policy for the container. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -15169,33 +13555,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -15211,9 +13577,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -15222,49 +13585,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - You cannot set this field on ephemeral containers. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. You cannot set this field on - ephemeral containers. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -15279,253 +13613,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: Probes are not allowed for ephemeral containers. properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -15534,143 +13726,63 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. type: string terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -15681,64 +13793,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -15749,11 +13818,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -15763,22 +13827,14 @@ spec: - name x-kubernetes-list-type: map hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. properties: hostnames: - description: Hostnames for the above IP address. items: type: string type: array x-kubernetes-list-type: atomic ip: - description: IP address of the host file entry. type: string required: - ip @@ -15788,70 +13844,22 @@ spec: - ip x-kubernetes-list-type: map hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. type: boolean hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - When using HostNetwork you should specify ports so the scheduler is aware. - When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, - and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. - Default to false. type: boolean hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. type: boolean hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. type: boolean hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. type: string hostnameOverride: - description: |- - HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. - This field only specifies the pod's hostname and does not affect its DNS records. - When this field is set to a non-empty string: - - It takes precedence over the values set in `hostname` and `subdomain`. - - The Pod's hostname will be set to this value. - - `setHostnameAsFQDN` must be nil or set to false. - - `hostNetwork` must be set to false. - - This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. - Requires the HostnameOverride feature gate to be enabled. type: string imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -15860,144 +13868,60 @@ spec: - name x-kubernetes-list-type: map initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ items: - description: A single application container that you want to run within a pod. properties: args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: - description: |- - List of environment variables to set in the container. - Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. type: string value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: - description: Selects a key of a ConfigMap. properties: key: - description: The key to select. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. properties: key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. type: boolean path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. type: string required: - key @@ -16006,44 +13930,29 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: - description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -16058,52 +13967,25 @@ spec: - name x-kubernetes-list-type: map envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: - description: The ConfigMap to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. type: string secretRef: - description: The Secret to select from properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -16111,67 +13993,31 @@ spec: type: array x-kubernetes-list-type: atomic image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. properties: postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16180,104 +14026,58 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16286,130 +14086,75 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object sleep: - description: Sleep represents a duration that the container should sleep. properties: seconds: - description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16418,129 +14163,62 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. type: string ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. items: - description: ContainerPort represents a network port in a single container. properties: containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. type: string hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. format: int32 type: integer name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. type: string protocol: default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". type: string required: - containerPort @@ -16551,71 +14229,39 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -16624,99 +14270,51 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. items: - description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. type: string restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -16725,34 +14323,13 @@ spec: type: array x-kubernetes-list-type: atomic resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -16768,9 +14345,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -16779,69 +14353,20 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. type: string restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. items: - description: ContainerRestartRule describes how a container exit is handled. properties: action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. type: string exitCodes: - description: Represents the exit codes to check on container exits. properties: operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. type: string values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. items: format: int32 type: integer @@ -16856,261 +14381,111 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. properties: add: - description: Added capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: - description: Removed capabilities items: - description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: - description: Exec specifies a command to execute in the container. properties: command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: - description: GRPC specifies a GRPC HealthCheckRequest. properties: port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: - description: HTTPGet specifies an HTTP GET request to perform. properties: host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: - description: The header field value type: string required: - name @@ -17119,134 +14494,61 @@ spec: type: array x-kubernetes-list-type: atomic path: - description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: - description: TCPSocket specifies a connection to a TCP port. properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. type: boolean stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false type: boolean terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. type: string terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. type: string tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -17257,64 +14559,21 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. type: string mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). type: string name: - description: This must match the Name of a Volume. type: string readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. type: boolean recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). type: string subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -17325,11 +14584,6 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. type: string required: - name @@ -17339,63 +14593,15 @@ spec: - name x-kubernetes-list-type: map nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename type: string nodeSelector: additionalProperties: type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.resources - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup properties: name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null type: string required: - name @@ -17407,50 +14613,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md type: object preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. type: string priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. format: int32 type: integer priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. type: string readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates items: - description: PodReadinessGate contains the reference to a pod condition properties: conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. type: string required: - conditionType @@ -17458,73 +14632,13 @@ spec: type: array x-kubernetes-list-type: atomic resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is a stable field but requires that the - DynamicResourceAllocation feature gate is enabled. - - This field is immutable. items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - - When the DRAWorkloadResourceClaims feature gate is enabled and this Pod - belongs to a PodGroup, a PodResourceClaim is matched to a - PodGroupResourceClaim if all of their fields are equal (Name, - ResourceClaimName, and ResourceClaimTemplateName). A matched claim references - a single ResourceClaim shared across all Pods in the PodGroup, reserved for - the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual - Pods. properties: name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. type: string resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. type: string resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - When the DRAWorkloadResourceClaims feature gate is enabled and the pod - belongs to a PodGroup that defines a PodGroupResourceClaim with the same - Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the - ResourceClaim generated for the PodGroup. All pods in the group that - define an equivalent PodResourceClaim matching the - PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same - generated ResourceClaim. ResourceClaims generated for a PodGroup are - owned by the PodGroup and their lifecycles are tied to the PodGroup - instead of any individual pod. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. type: string required: - name @@ -17534,40 +14648,13 @@ spec: - name x-kubernetes-list-type: map resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. properties: claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. type: string request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. type: string required: - name @@ -17583,9 +14670,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -17594,47 +14678,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy type: string runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class type: string schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. type: string schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. properties: name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. type: string required: - name @@ -17644,217 +14699,70 @@ spec: - name x-kubernetes-list-type: map schedulingGroup: - description: |- - SchedulingGroup provides a reference to the immediate scheduling runtime - grouping object that this Pod belongs to. - This field is used by the scheduler to identify the group and apply the - correct group scheduling policies. The association with a group also - impacts other lifecycle aspects of a Pod that are relevant in a wider context - of scheduling like preemption, resource attachment, etc. If not specified, - the Pod is treated as a single unit in all of these aspects. - The group object referenced by this field may not exist at the time the - Pod is created. - This field is immutable, but a group object with the same name may be - recreated with different policies. Doing this during pod scheduling - may result in the placement not conforming to the expected policies. properties: podGroupName: - description: |- - PodGroupName specifies the name of the standalone PodGroup object - that represents the runtime instance of this group. - Must be a DNS subdomain. type: string type: object securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. properties: appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type type: object fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. type: string runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. type: string seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. type: string required: - type type: object supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. type: string sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. items: - description: Sysctl defines a kernel parameter to be set properties: name: - description: Name of a property to set type: string value: - description: Value of a property to set type: string required: - name @@ -17863,156 +14771,60 @@ spec: type: array x-kubernetes-list-type: atomic windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. type: string serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ type: string setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. type: boolean shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. type: boolean subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. type: string terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. format: int64 type: integer tolerations: - description: If specified, the pod's tolerations. items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . properties: effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array x-kubernetes-list-type: atomic topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -18026,125 +14838,27 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string type: array x-kubernetes-list-type: atomic maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. format: int32 type: integer nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. type: string nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. type: string topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. type: string whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. type: string required: - maxSkew @@ -18157,244 +14871,111 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore properties: fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). format: int32 type: integer readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: boolean volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string required: - volumeID type: object azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. properties: cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' type: string diskName: - description: diskName is the Name of the data disk in the blob storage type: string diskURI: - description: diskURI is the URI of data disk in the blob storage type: string fsType: default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. properties: readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key type: string shareName: - description: shareName is the azure share Name type: string required: - secretName - shareName type: object cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. properties: monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: boolean secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string required: - monitors type: object cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: boolean secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string required: - volumeID type: object configMap: - description: configMap represents a configMap that should populate this volume properties: defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -18404,129 +14985,67 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. properties: driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. type: string fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. type: string nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: Items is a list of downward API volume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource @@ -18539,87 +15058,21 @@ spec: x-kubernetes-list-type: atomic type: object emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir properties: medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string sizeLimit: anyOf: - type: integer - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. properties: volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. properties: metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. properties: annotations: additionalProperties: @@ -18639,42 +15092,19 @@ spec: type: string type: object spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. properties: accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 items: type: string type: array x-kubernetes-list-type: atomic dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. type: string kind: - description: Kind is the type of resource being referenced type: string name: - description: Name is the name of resource being referenced type: string required: - kind @@ -18682,60 +15112,20 @@ spec: type: object x-kubernetes-map-type: atomic dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. type: string kind: - description: Kind is the type of resource being referenced type: string name: - description: Name is the name of resource being referenced type: string namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. type: string required: - kind - name type: object resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources properties: limits: additionalProperties: @@ -18744,9 +15134,6 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -18755,37 +15142,18 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object selector: - description: selector is a label query over volumes to consider for binding. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -18799,38 +15167,16 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ type: string volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. type: string volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object required: @@ -18838,79 +15184,41 @@ spec: type: object type: object fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string lun: - description: 'lun is Optional: FC target lun number' format: int32 type: integer readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' items: type: string type: array x-kubernetes-list-type: atomic wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. items: type: string type: array x-kubernetes-list-type: atomic type: object flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. properties: driver: - description: driver is the name of the driver to use for this volume. type: string fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string - description: 'options is Optional: this field holds extra command options if any.' type: object readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -18918,230 +15226,98 @@ spec: - driver type: object flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. properties: datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated type: string datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk properties: fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk format: int32 type: integer pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: boolean required: - pdName type: object gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. properties: directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. type: string repository: - description: repository is the URL type: string revision: - description: revision is the commit hash for the specified revision. type: string required: - repository type: object glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. properties: endpoints: - description: endpoints is the endpoint name that details Glusterfs topology. type: string path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: string readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: boolean required: - endpoints - path type: object hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath properties: path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string required: - path type: object image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. properties: pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. type: string reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. type: string type: object iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication type: boolean fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi type: string initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. type: string iqn: - description: iqn is the target iSCSI Qualified Name. type: string iscsiInterface: default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). type: string lun: - description: lun represents iSCSI Target Lun number. format: int32 type: integer portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). items: type: string type: array x-kubernetes-list-type: atomic readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. type: boolean secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). type: string required: - iqn @@ -19149,161 +15325,68 @@ spec: - targetPortal type: object name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs properties: path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: boolean server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string required: - path - server type: object persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. type: boolean required: - claimName type: object photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: - description: pdID is the ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver. properties: fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean volumeID: - description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: - description: projected items for all in one resources secrets, configmaps, and downward API properties: defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. properties: clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. properties: labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. properties: key: - description: key is the label key that the selector applies to. type: string operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. items: type: string type: array @@ -19317,72 +15400,31 @@ spec: matchLabels: additionalProperties: type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. type: string optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. type: boolean path: - description: Relative path from the volume root to write the bundle. type: string signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. type: string required: - path type: object configMap: - description: configMap information about the configMap data to project properties: items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19392,69 +15434,42 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic downwardAPI: - description: downwardAPI information about the downwardAPI data to project properties: items: - description: Items is a list of DownwardAPIVolume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' type: string required: - resource @@ -19467,161 +15482,39 @@ spec: x-kubernetes-list-type: atomic type: object podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. properties: certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. type: string credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. type: string keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. type: string keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". type: string maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. format: int32 type: integer signerName: - description: Kubelet's generated CSRs will be addressed to this signer. type: string userAnnotations: additionalProperties: type: string - description: |- - userAnnotations allow pod authors to pass additional information to - the signer implementation. Kubernetes does not restrict or validate this - metadata in any way. - - These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of - the PodCertificateRequest objects that Kubelet creates. - - Entries are subject to the same validation as object metadata annotations, - with the addition that all keys must be domain-prefixed. No restrictions - are placed on values, except an overall size limitation on the entire field. - - Signers should document the keys and values they support. Signers should - deny requests that contain keys they do not recognize. type: object required: - keyType - signerName type: object secret: - description: secret information about the secret data to project properties: items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19631,42 +15524,19 @@ spec: x-kubernetes-list-type: atomic name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: optional field specify whether the Secret or its key must be defined type: boolean type: object x-kubernetes-map-type: atomic serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project properties: audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. type: string expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. format: int64 type: integer path: - description: |- - path is the path relative to the mount point of the file to project the - token into. type: string required: - path @@ -19676,176 +15546,84 @@ spec: x-kubernetes-list-type: atomic type: object quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. properties: group: - description: |- - group to map volume access to - Default is no group type: string readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. type: boolean registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes type: string tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: - description: |- - user to map volume access to - Defaults to serivceaccount user type: string volume: - description: volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd type: string image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string keyring: default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic pool: default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: boolean secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string required: - image - monitors type: object scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. properties: fsType: default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". type: string gateway: - description: gateway is the host address of the ScaleIO API Gateway. type: string protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false type: boolean storageMode: default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. type: string storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. type: string system: - description: system is the name of the storage system as configured in ScaleIO. type: string volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. type: string required: - gateway @@ -19853,52 +15631,19 @@ spec: - system type: object secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret properties: defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. properties: key: - description: key is the key to project. type: string mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. type: string required: - key @@ -19907,81 +15652,37 @@ spec: type: array x-kubernetes-list-type: atomic optional: - description: optional field specify whether the Secret or its keys must be defined type: boolean secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret type: string type: object storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. properties: fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. properties: name: default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. type: string volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. properties: fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. type: string volumePath: - description: volumePath is the path that identifies vSphere volume vmdk type: string required: - volumePath @@ -29459,8123 +25160,7 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: - description: |- - podTemplate defines the Pod configuration for this role. Admission permits - it only when the enclosing component type explicitly supports role-specific - Pod templates. LPX components support role-specific Pod templates. - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options of a pod. - properties: - name: - description: |- - Name is this DNS resolver option's name. - Required. - type: string - value: - description: Value is this DNS resolver option's value. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral containers. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - You cannot set this field on ephemeral containers. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. You cannot set this field on - ephemeral containers. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - When using HostNetwork you should specify ports so the scheduler is aware. - When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, - and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - hostnameOverride: - description: |- - HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. - This field only specifies the pod's hostname and does not affect its DNS records. - When this field is set to a non-empty string: - - It takes precedence over the values set in `hostname` and `subdomain`. - - The Pod's hostname will be set to this value. - - `setHostnameAsFQDN` must be nil or set to false. - - `hostNetwork` must be set to false. - - This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. - Requires the HostnameOverride feature gate to be enabled. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps or Secrets - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: |- - Resources resize policy for the container. - This field cannot be set on ephemeral containers. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.resources - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is a stable field but requires that the - DynamicResourceAllocation feature gate is enabled. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - - When the DRAWorkloadResourceClaims feature gate is enabled and this Pod - belongs to a PodGroup, a PodResourceClaim is matched to a - PodGroupResourceClaim if all of their fields are equal (Name, - ResourceClaimName, and ResourceClaimTemplateName). A matched claim references - a single ResourceClaim shared across all Pods in the PodGroup, reserved for - the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual - Pods. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - When the DRAWorkloadResourceClaims feature gate is enabled and the pod - belongs to a PodGroup that defines a PodGroupResourceClaim with the same - Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the - ResourceClaim generated for the PodGroup. All pods in the group that - define an equivalent PodResourceClaim matching the - PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same - generated ResourceClaim. ResourceClaims generated for a PodGroup are - owned by the PodGroup and their lifecycles are tied to the PodGroup - instead of any individual pod. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - schedulingGroup: - description: |- - SchedulingGroup provides a reference to the immediate scheduling runtime - grouping object that this Pod belongs to. - This field is used by the scheduler to identify the group and apply the - correct group scheduling policies. The association with a group also - impacts other lifecycle aspects of a Pod that are relevant in a wider context - of scheduling like preemption, resource attachment, etc. If not specified, - the Pod is treated as a single unit in all of these aspects. - The group object referenced by this field may not exist at the time the - Pod is created. - This field is immutable, but a group object with the same name may be - recreated with different policies. Doing this during pod scheduling - may result in the placement not conforming to the expected policies. - properties: - podGroupName: - description: |- - PodGroupName specifies the name of the standalone PodGroup object - that represents the runtime instance of this group. - Must be a DNS subdomain. - type: string - type: object - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. - properties: - driver: - description: driver is the name of the driver to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - properties: - endpoints: - description: endpoints is the endpoint name that details Glusterfs topology. - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver. - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. - properties: - certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. - type: string - keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". - type: string - maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. - format: int32 - type: integer - signerName: - description: Kubelet's generated CSRs will be addressed to this signer. - type: string - userAnnotations: - additionalProperties: - type: string - description: |- - userAnnotations allow pod authors to pass additional information to - the signer implementation. Kubernetes does not restrict or validate this - metadata in any way. - - These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of - the PodCertificateRequest objects that Kubelet creates. - - Entries are subject to the same validation as object metadata annotations, - with the addition that all keys must be domain-prefixed. No restrictions - are placed on values, except an overall size limitation on the entire field. - - Signers should document the keys and values they support. Signers should - deny requests that contain keys they do not recognize. - type: object - required: - - keyType - - signerName - type: object - secret: - description: secret information about the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify whether the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object + podTemplate: *lpx-pod-template providerOverride: description: |- providerOverride configures the provider workload unit generated for this diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 78a5fe44244c..4a0db3fa943b 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -7,7 +7,7 @@ require ( emperror.dev/errors v0.8.1 github.com/Masterminds/semver/v3 v3.4.0 github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 - github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13 + github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f github.com/ai-dynamo/snapshot/api v0.1.0 github.com/bsm/gomega v1.27.10 diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index dd11984f9685..83d26aae74f7 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -20,8 +20,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 h1:eVDVuyh5vDgWG6I2Mffs68g+25qW94VAG3EfDTxbhc0= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13/go.mod h1:csL4kgDc1MoD7Erjyu8QyVrbL/dv8fIZ3sYkJcc2nrQ= -github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13 h1:1pS3fH87Zr4IXTC8IR7KQxrLoChPrLv4OWOiquW/YVg= -github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.13/go.mod h1:+JD/5QmNY0FViNWzx6EWMrLDdYbl4gkO5472mqfFeck= +github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1 h1:9WJ/OFtVNQMozYVUl6OfXDz/jF4hFvdj7hL7SHauIOU= +github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1/go.mod h1:72lQ5kc40jXxyOrfuAICc8qxV0/Mwy8k8woM8LUoo9E= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f h1:PBmn3itAlMFHc0HgDOcr0IwV0uiUDQehcg7/aumIobU= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f/go.mod h1:i7tIkusSEcMQqN4IDd3cXZ6EL0Kfke6F00Nn5WbuH3Y= github.com/ai-dynamo/snapshot/api v0.1.0 h1:S8HG6K+YhloUc/usu03wiJ8fTo3Mhju/oWQVFDYVw/o= diff --git a/deploy/operator/internal/common/podspec.go b/deploy/operator/internal/common/podspec.go new file mode 100644 index 000000000000..9157e4e96fd5 --- /dev/null +++ b/deploy/operator/internal/common/podspec.go @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package common + +import corev1 "k8s.io/api/core/v1" + +// FindContainerByName returns the first container with the given name. +func FindContainerByName(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + +// HasPodResourceClaim reports whether the pod spec references a claim by name. +func HasPodResourceClaim(podSpec *corev1.PodSpec, name string) bool { + for i := range podSpec.ResourceClaims { + if podSpec.ResourceClaims[i].Name == name { + return true + } + } + return false +} + +// HasContainerResourceClaim reports whether the container references a claim by name. +func HasContainerResourceClaim(container *corev1.Container, name string) bool { + for i := range container.Resources.Claims { + if container.Resources.Claims[i].Name == name { + return true + } + } + return false +} + +// HasVolume reports whether a volume exists by name. +func HasVolume(volumes []corev1.Volume, name string) bool { + for i := range volumes { + if volumes[i].Name == name { + return true + } + } + return false +} + +// HasVolumeMount reports whether a volume mount exists by name and path. +func HasVolumeMount(mounts []corev1.VolumeMount, name, mountPath string) bool { + for i := range mounts { + if mounts[i].Name == name && mounts[i].MountPath == mountPath { + return true + } + } + return false +} + +// HasEnvValue reports whether an environment variable exists with the given value. +func HasEnvValue(env []corev1.EnvVar, name, value string) bool { + for i := range env { + if env[i].Name == name && env[i].Value == value { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index af823ee75d24..517ca9b47a12 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -729,12 +729,13 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.Equal(t, "enabled", service.Annotations["example.com/model-discovery"]) require.True(t, metav1.IsControlledBy(service, source)) - t.Log("Observe the created PCS before committing its worker hash") - require.Empty(t, source.Annotations[consts.AnnotationCurrentWorkerHashV2]) - _, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + t.Log("Observe the ordinary PCS from a fresh parent reconcile before committing its worker hash") + freshSource := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), freshSource)) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: freshSource}) require.NoError(t, err) - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - require.NotEmpty(t, source.Annotations[consts.AnnotationCurrentWorkerHashV2]) + require.Equal(t, v1beta1.DGDStatePending, result.Status.State) + source = freshSource t.Log("A pending ordinary checkpoint retains its startup and scaling gates alongside the pending child") prefill := source.GetComponentByName("prefill") diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 63b88b1aaa3d..da4421bf0205 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -9,6 +9,7 @@ import ( "slices" "strconv" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -26,7 +27,7 @@ func configureHybridCyborg( plan *MaterializationPlan, cyborgConfigMap *corev1.ConfigMap, ) error { - container := findMainContainer(cyborg.Spec.PodSpec.Containers) + container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) // Apply manifest-aware runtime bindings to the selected Cyborg container. if err := withLPUModelStorage(&cyborg.Spec.PodSpec, container, modelStorage); err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 693a48757d67..5cbe404d50bf 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -61,7 +62,7 @@ func ApplySelectedCyborgContainerDefaults( // SWA cache IDs depend on the final, user-overridable batch size and are // bound only after the main-container override is merged. applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) - lpxContainer := findMainContainer(lpxPodSpec.Containers) + lpxContainer := common.FindContainerByName(lpxPodSpec.Containers, commonconsts.MainContainerName) mountIndex := slices.IndexFunc(lpxContainer.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == commonconsts.ModelStorageVolumeName }) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 89fdaa35176f..e271fb9521d7 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/pelletier/go-toml/v2" @@ -139,7 +140,7 @@ func LPUConfigMapHash(configMap *corev1.ConfigMap) string { } func lpuModelStorageBinding(spec corev1.PodSpec) (lpuModelStorage, error) { - container := findMainContainer(spec.Containers) + container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == commonconsts.ModelStorageVolumeName }) @@ -441,7 +442,7 @@ func withLPUConfigVolume(spec *corev1.PodSpec, configMapName string, allowOverri }}, }) } - container := findMainContainer(spec.Containers) + container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) found = false for _, mount := range container.VolumeMounts { if mount.Name == lpuConfigVolumeName || mount.MountPath == lpuConfigMountPath { diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index 102a5fd3b125..6bc4b6cd9914 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -8,6 +8,7 @@ package lpx import ( "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -31,7 +32,7 @@ func configureAgentScheduling( // Preserve Agent placement while replacing authored LPU resources with the selected device. stripLPUResources(agent) agent.SchedulerName = SchedulerName - container := findMainContainer(agent.Containers) + container := common.FindContainerByName(agent.Containers, commonconsts.MainContainerName) // Model projection has already restricted the target family to XT or HX. name, amount := v2LPUResourceName, resource.MustParse("8") if targetFamily == BuildFamilyHX { @@ -46,16 +47,6 @@ func configureAgentScheduling( container.Resources.Requests[name], container.Resources.Limits[name] = amount, amount } -// findMainContainer returns the main container, or nil when absent. -func findMainContainer(containers []corev1.Container) *corev1.Container { - for i := range containers { - if containers[i].Name == commonconsts.MainContainerName { - return &containers[i] - } - } - return nil -} - func stripLPUResources(spec *corev1.PodSpec) { for index := range spec.InitContainers { stripLPUResourceRequirements(&spec.InitContainers[index].Resources) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index d0399358c3a0..4cd2a8dc1925 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -11,6 +11,7 @@ import ( "slices" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -252,7 +253,7 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo } } if workload.Pipeline() == PipelineSpecDecode || workload.Pipeline() == PipelineLPX { - agent := findMainContainer(agentPodSpec.Containers) + agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) setContainerEnv(agent, false, corev1.EnvVar{Name: lpuModelNameEnvVar}) } diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index f7dcde489f8e..00e2eb69b08c 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -16,6 +16,7 @@ import ( "k8s.io/utils/ptr" dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) @@ -151,7 +152,7 @@ func validateSelectedConductor( if conductor != nil && conductor.PodTemplate != nil { template = conductor.PodTemplate } - container := findMainContainer(template.Spec.Containers) + container := common.FindContainerByName(template.Spec.Containers, commonconsts.MainContainerName) count, err := EffectiveCyborgGPUCount(container.Resources) if err != nil { return fmt.Errorf("component %q conductor resources: %w", component.ComponentName, err) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index dfd8027c0349..8a73a472aacc 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -10,6 +10,7 @@ import ( "slices" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -115,7 +116,7 @@ func configureDirectHybridAgentRuntime( return fmt.Errorf("direct hybrid agent runtime requires an MPI SSH secret name") } - agent := findMainContainer(agentPodSpec.Containers) + agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) // Drop Nova-only flags before applying the direct Agent identity and worker defaults. agent.Args, _ = stripNovaOnlyArgs(agent.Args) hasCustomStartup := len(agent.Command) != 0 || len(agent.Args) != 0 @@ -283,7 +284,7 @@ func runtimeMainContainer( if targetFamily == BuildFamilyHX && len(podSpec.Containers) != 1 { return nil, fmt.Errorf("PodSpec requires exactly one %q container", commonconsts.MainContainerName) } - return findMainContainer(podSpec.Containers), nil + return common.FindContainerByName(podSpec.Containers, commonconsts.MainContainerName), nil } func configureNodeLocalAgentWorkerContainer( diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index f746d575a7c4..dba265aaa378 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -1104,6 +1104,16 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { }), wantWebhookErrs: []string{"spec.components[1].roles: Forbidden: roles are supported only for component shapes that define a role schema; this release supports multinode and LPX components"}, }, + { + name: "ordinary multinode roles accept matching replicas but reject role PodTemplates", + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + setBetaExplicitMultinodeRoles(worker, 4) + worker.Roles[0].Replicas = k8sptr.To(int32(1)) + worker.Roles[1].PodTemplate = worker.PodTemplate.DeepCopy() + }), + wantWebhookErrs: []string{"spec.components[1].roles[1].podTemplate: Forbidden: is not supported for this component role"}, + }, { name: "explicit multinode roles require the complete role set", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index 63452d81b084..e85d699b67d7 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -429,7 +429,6 @@ func (v *sharedValidation) validateComponentRoles( workloadProvider: workloadProvider, scope: scope, component: component, - podTemplateAllowed: component.IsLPX(), }, )...) } @@ -455,19 +454,13 @@ type componentRoleSpecValidationOptions struct { } // validateComponentRoleSpec validates role. role and fldPath must not be nil; -// the optional provider override and PodTemplate may be nil. +// options.component and optional role fields may be nil. func (v *sharedValidation) validateComponentRoleSpec( role *nvidiacomv1beta1.ComponentRoleSpec, fldPath *field.Path, options componentRoleSpecValidationOptions, ) field.ErrorList { allErrs := field.ErrorList{} - if role.PodTemplate != nil && !options.podTemplateAllowed { - allErrs = append(allErrs, field.Forbidden( - fldPath.Child("podTemplate"), - "is not supported for this component role", - )) - } // LPX renders role templates directly and has no provider-override lowering. if options.component != nil && options.component.IsLPX() { @@ -482,21 +475,28 @@ func (v *sharedValidation) validateComponentRoleSpec( return allErrs } - if role.ProviderOverride == nil { - return allErrs + if role.PodTemplate != nil && !options.podTemplateAllowed { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("podTemplate"), + "is not supported for this component role", + )) } // Validate the provider fragment against this exact multinode role. - return append(allErrs, v.validateProviderOverride( - role.ProviderOverride, - fldPath.Child("providerOverride"), - providerOverrideValidationOptions{ - supported: options.providerOverridesSupported, - workloadProvider: options.workloadProvider, - scope: options.scope, - component: options.component, - }, - )...) + if role.ProviderOverride != nil { + allErrs = append(allErrs, v.validateProviderOverride( + role.ProviderOverride, + fldPath.Child("providerOverride"), + providerOverrideValidationOptions{ + supported: options.providerOverridesSupported, + workloadProvider: options.workloadProvider, + scope: options.scope, + component: options.component, + }, + )...) + } + + return allErrs } // validateEPPConfig validates deprecated Go-EPP config. config and fldPath must not be nil. @@ -785,6 +785,8 @@ func (v *sharedValidation) validateComponentCheckpointJobConfig( // validateDynamoComponentDeploymentSharedSpecUpdate validates a component update. // newComponent, oldComponent, and fldPath must not be nil; ownerKind.Kind must not be empty. +// +//nolint:gocyclo // Update validation reports the combined upstream and LPX contracts in one pass. func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( newComponent *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, oldComponent *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, @@ -824,11 +826,13 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( apivalidation.FieldImmutableErrorMsg, )) } - allErrs = append(allErrs, validateComponentRolesUpdate( - newComponent, - oldComponent, - fldPath.Child("roles"), - )...) + if !newComponent.IsLPX() && !oldComponent.IsLPX() { + allErrs = append(allErrs, validateComponentRolesUpdate( + newComponent, + oldComponent, + fldPath.Child("roles"), + )...) + } } if newComponent.IsLPX() != oldComponent.IsLPX() { @@ -939,9 +943,6 @@ func validateComponentRolesUpdate( oldComponent *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, fldPath *field.Path, ) field.ErrorList { - if newComponent.IsLPX() || oldComponent.IsLPX() { - return nil - } // Permit representation-only implicit/explicit migrations, but require // role-specific configuration changes to happen in a subsequent update. if (newComponent.Roles == nil) != (oldComponent.Roles == nil) { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index ffd9e2e0a884..6f13b531cded 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -151,8 +151,8 @@ _Appears in:_ | --- | --- | --- | --- | | `name` _string_ | Name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | Replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
Multinode.NodeCount; leader must be 1 and worker must be
Multinode.NodeCount minus 1. | | Minimum: 1
Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate configures Pods for this role. The enclosing component
defines whether the template is required or may be inherited. | | Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | ProviderOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. LPX components support role-specific Pod templates. | | Optional: \{\}
| #### ConfigMapKeySelector @@ -1794,8 +1794,8 @@ _Appears in:_ | --- | --- | --- | --- | | `name` _string_ | name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
multinode.nodeCount; leader must be 1 and worker must be
multinode.nodeCount minus 1. | | Minimum: 1
Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate configures Pods for this role. The enclosing component
defines whether the template is required or may be inherited. | | Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. LPX components support role-specific Pod templates. | | Optional: \{\}
| #### ComponentType diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 387f48a411e4..a1344273cd1d 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -89,27 +89,25 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - Optional explicit structure of a compound component. Multinode components must contain exactly one `leader` and one `worker`. Omit the field to retain the implicit leader/worker layout. + Named Pod-producing parts of a compound component, keyed by `name`. Multinode components must contain exactly one `leader` and one `worker`; omitted role counts default to `1` and `multinode.nodeCount - 1`, respectively. Omit the field to retain the implicit leader/worker layout. LPX components use `agent` for Agent Pods and `conductor` for the conductor. With one LPX component, the conductor role is optional. In a speculative pair, the draft has only an agent role and the target declares both roles. Hybrid execution may also create Nova Pods from Agent configuration. - Standalone DCD roles accept `name` and `replicas`. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). + Standalone DCD roles cannot use LPX templates or DGD-only provider overrides. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). Semantic role name: `leader` or `worker` for ordinary multinode components, `conductor` or `agent` for LPX components. Names must be unique within the component. + + Logical cardinality of this role in one complete component instance; minimum `1`. For ordinary multinode components, admission defaults and persists an omitted value as `leader: 1` or `worker: multinode.nodeCount - 1`; an explicit value must match that fixed shape. For LPX, omitted conductor replicas use `1`, subject to runtime-specific limits. Agent replicas are derived from the model build; an explicit count must match the build. + Native Pod template for this role. LPX-only; required for the agent. Each supplied template requires a `main` container with an explicit image. An omitted conductor template reuses the same component's agent template; a single LPX component also uses this fallback when its conductor role is omitted. A speculative pair uses the target's conductor. An explicit conductor template merges with shared defaults independently, without inheriting the agent template. core/v1.PodTemplateSpec - - Logical cardinality of the role in one complete component instance. For multinode, admission defaults and persists an omitted value as `leader: 1` or `worker: multinode.nodeCount - 1`; an explicit value must match that fixed shape. - - For LPX, omitted conductor replicas use `1`, subject to runtime-specific limits. Agent replicas are derived from the model build; an explicit count must match the build. - Provider configuration for the workload unit generated for this role. Available only on ordinary multinode components embedded in a DGD. LPX roles do not support it; standalone DCD OpenAPI omits this field. @@ -326,7 +324,7 @@ The upgrade does not delete the legacy `DynamoCheckpoint` CRD, instances, `PodSn Runtime framework. - Allowed values: vllm sglang trtllm + Allowed values: vllm sglang trtllm lpu Dynamo platform version. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 751d94ca5564..6c7c06da01d9 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -154,15 +154,15 @@ Replicas is the logical cardinality of this role in one complete component insta **Validation:** Minimum: 1 Optional: \{\} - -ProviderOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. -See [ProviderOverride](#provideroverride). + +PodTemplate configures Pods for this role. The enclosing component defines whether the template is required or may be inherited. +See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). **Validation:** Optional: \{\} - -PodTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. LPX components support role-specific Pod templates. -See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). + +ProviderOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. +See [ProviderOverride](#provideroverride). **Validation:** Optional: \{\} @@ -2532,15 +2532,15 @@ replicas is the logical cardinality of this role in one complete component insta **Validation:** Minimum: 1 Optional: \{\} - -providerOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. -See [ProviderOverride](#v1beta1-provideroverride). + +podTemplate configures Pods for this role. The enclosing component defines whether the template is required or may be inherited. +See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). **Validation:** Optional: \{\} - -podTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. LPX components support role-specific Pod templates. -See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). + +providerOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. +See [ProviderOverride](#v1beta1-provideroverride). **Validation:** Optional: \{\} From f3e77303da607abffc36a9a1930a3bb158e7d298 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sat, 12 Sep 2026 23:36:46 +0300 Subject: [PATCH 043/196] fix(operator): remove LPX CRD schema compaction (#470) Signed-off-by: Andrew Paprotsky (cherry picked from commit 1d6d649719678b38b3870024aedf097051c44f96) Signed-off-by: Andrew Paprotsky --- deploy/operator/Makefile | 6 +- deploy/operator/api/scripts/lpx-crd-schema.yq | 31 - deploy/operator/cmd/crd-apply/main_test.go | 12 + deploy/operator/cmd/crd-apply/schema_test.go | 168 - .../nvidia.com_dynamographdeployments.yaml | 12417 +++++++++++++++- 5 files changed, 12429 insertions(+), 205 deletions(-) delete mode 100644 deploy/operator/api/scripts/lpx-crd-schema.yq delete mode 100644 deploy/operator/cmd/crd-apply/schema_test.go diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 36aba16626cd..6a4ecbff0226 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -253,6 +253,8 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C $(YQ) eval '(.spec.versions[] | select(.name == "v1beta1") | .schema.openAPIV3Schema.properties.spec.required) |= (. - ["name"])' -i --indent 2 config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml echo "Removing DGD-only provider overrides from standalone DynamoComponentDeployment schemas" $(YQ) eval '(.spec.versions[].schema.openAPIV3Schema.properties.spec.properties) |= del(.providerOverride) | (.spec.versions[].schema.openAPIV3Schema.properties.spec.properties.roles.items.properties) |= del(.providerOverride)' -i --indent 2 config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml + echo "Removing DGD-only LPX fields and validation rules from standalone DynamoComponentDeployment schemas" + $(YQ) eval '(.spec.versions[].schema.openAPIV3Schema.properties.spec) |= (del(.properties.lpx) | (select(has("x-kubernetes-validations"))."x-kubernetes-validations") |= map(select(.rule | test("\\b(self|oldSelf)\\.lpx\\b") | not)))' -i --indent 2 config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml # PodTemplateSpec is an external Kubernetes type, so its item-level markers cannot be declared on the API field. # Item-scoped rules avoid unbounded list comprehensions without imposing maxItems on the public API. # The annotation value bound matches Kubernetes' total annotation size limit. @@ -284,10 +286,6 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C $(YQ) eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \ fi; \ done - # Compact LPX schemas last: subsequent schema edits could change an aliased role. - for file in config/crd/bases/nvidia.com_dynamographdeployments.yaml config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml; do \ - $(YQ) --from-file api/scripts/lpx-crd-schema.yq -i --indent 2 $$file || exit 1; \ - done .PHONY: generate generate: controller-gen generate-pydantic ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt_" paths="./..." diff --git a/deploy/operator/api/scripts/lpx-crd-schema.yq b/deploy/operator/api/scripts/lpx-crd-schema.yq deleted file mode 100644 index 52d17862ff6d..000000000000 --- a/deploy/operator/api/scripts/lpx-crd-schema.yq +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# LPX is DGD-only. Preserve ordinary DCD roles while removing LPX inputs and -# their dependent rules. Retain the rejection of standalone type: lpx. -with(select(.metadata.name == "dynamocomponentdeployments.nvidia.com"); - (.spec.versions[].schema.openAPIV3Schema.properties.spec) |= ( - del(.properties.lpx) | - (select(has("x-kubernetes-validations"))."x-kubernetes-validations") |= - map(select(.rule | test("\\b(self|oldSelf)\\.lpx\\b") | not)) - ) -) | - -# Keep native validation and merge semantics; omit only LPX template descriptions. -# Ordinary PodTemplate and alpha extraPodSpec documentation remains unchanged. -del(.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | .. | select(has("description")) | .description) | - -# Compact only LPX templates, after every other schema transformation has finished. -# YAML requires the anchor before every alias: traversal order is not document order. -with(select([.. | select(has("lpx") and has("roles"))] | length > 0); - ([.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - {"path": (path | to_json), "line": line, "column": column}] | sort_by(.line, .column))[0].path as $first | - [.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) == $first) | to_json][0] as $schema | - (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) == $first)) anchor = "lpx-pod-template" | - - # Different role-specific constraints must never be replaced by an alias. - (.. | select(has("lpx") and has("roles")) | .roles.items.properties.podTemplate | - select((path | to_json) != $first) | select((to_json) == $schema)) alias = "lpx-pod-template" -) diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index 2af8785036d5..6d62a6830e6b 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -126,6 +126,18 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { for _, version := range crd.Spec.Versions { status := version.Schema.OpenAPIV3Schema.Properties["status"] if name == "dynamographdeployments.nvidia.com" { + t.Log("Preserve upstream role PodTemplate documentation in both DGD versions") + spec := version.Schema.OpenAPIV3Schema.Properties["spec"] + var component *apiextensionsv1.JSONSchemaProps + if version.Name == "v1alpha1" { + component = spec.Properties["services"].AdditionalProperties.Schema + } else { + component = spec.Properties["components"].Items.Schema + } + template := component.Properties["roles"].Items.Schema.Properties["podTemplate"] + require.NotEmpty(t, template.Description) + require.NotEmpty(t, template.Properties["spec"].Description) + if _, exists := status.Properties["modelDownload"]; exists { t.Errorf("%s %s retains status.modelDownload", name, version.Name) } diff --git a/deploy/operator/cmd/crd-apply/schema_test.go b/deploy/operator/cmd/crd-apply/schema_test.go deleted file mode 100644 index 252002f6980e..000000000000 --- a/deploy/operator/cmd/crd-apply/schema_test.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package main - -import ( - "bytes" - "encoding/json" - "os/exec" - "reflect" - "testing" - - "sigs.k8s.io/yaml" -) - -func TestLPXSchemaCompaction(t *testing.T) { - // Exercise native validation and merge metadata without copying an entire generated CRD. - const podSchema = ` -description: A role-specific Pod template. -type: object -required: [spec] -properties: - metadata: - type: object - properties: - labels: - description: Pod labels. - type: object - additionalProperties: {type: string} - spec: - description: Native PodSpec. - type: object - required: [containers] - properties: - containers: - type: array - x-kubernetes-list-type: map - x-kubernetes-list-map-keys: [name] - items: - type: object - required: [name] - x-kubernetes-validations: - - rule: self.name != '' - message: Name must not be empty. - properties: - name: {type: string} - image: {type: string} - restartPolicy: {type: string, default: Always, enum: [Always, Never]} -` - // Keep absence, equal schemas, and role-specific constraints in one behavioral matrix. - for _, test := range []struct { - name string - withLPX bool - different bool - flow bool - aliases int - }{ - {name: "no LPX"}, - {name: "identical schemas", withLPX: true, aliases: 1}, - {name: "identical schemas on one line", withLPX: true, flow: true, aliases: 1}, - {name: "different constraints", withLPX: true, different: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Render role templates in two API versions alongside unchanged ordinary schemas") - var alphaRoles, betaRoles, ordinary, undocumented map[string]any - for _, target := range []*map[string]any{&alphaRoles, &betaRoles, &ordinary, &undocumented} { - if err := yaml.Unmarshal([]byte(podSchema), target); err != nil { - t.Fatal(err) - } - } - if test.different { - betaRoles["required"] = []any{"metadata", "spec"} - } - removeSchemaDescriptions(undocumented) - input := map[string]any{ - "podTemplate": ordinary, - "extraPodSpec": map[string]any{"description": "Alpha documentation.", "properties": ordinary}, - "unrelated": map[string]any{"podTemplate": undocumented}, - } - if test.withLPX { - input["versions"] = []any{ - map[string]any{"properties": map[string]any{ - "lpx": map[string]any{"type": "object"}, - "roles": map[string]any{"items": map[string]any{"properties": map[string]any{ - "podTemplate": alphaRoles, - }}}, - }}, - map[string]any{"properties": map[string]any{ - "lpx": map[string]any{"type": "object"}, - "roles": map[string]any{"items": map[string]any{"properties": map[string]any{ - "podTemplate": betaRoles, - }}}, - }}, - } - } - original, err := yaml.Marshal(input) - if test.flow { - original, err = json.Marshal(input) - } - if err != nil { - t.Fatal(err) - } - - t.Log("Run the production transformation and decode its aliases using the installer YAML library") - output := compactLPXSchema(t, original) - var actual map[string]any - if err := yaml.Unmarshal(output, &actual); err != nil { - t.Fatalf("installer cannot decode compacted schema: %v\n%s", err, output) - } - if test.withLPX { - removeSchemaDescriptions(alphaRoles) - removeSchemaDescriptions(betaRoles) - } - if !reflect.DeepEqual(actual, input) { - t.Fatalf("schema semantics changed beyond LPX descriptions:\n%s", output) - } - - t.Log("Check alias scope and physical order, including an ordinary identical schema") - if got := bytes.Count(output, []byte("*lpx-pod-template")); got != test.aliases { - t.Errorf("alias count = %d, want %d", got, test.aliases) - } - anchor := bytes.Index(output, []byte("&lpx-pod-template")) - alias := bytes.Index(output, []byte("*lpx-pod-template")) - if test.aliases > 0 && anchor > alias { - t.Error("alias precedes its anchor") - } - if !test.withLPX && bytes.Contains(output, []byte("&lpx-pod-template")) { - t.Error("ordinary schema gained an LPX anchor") - } - - t.Log("Require a stable second transformation") - if again := compactLPXSchema(t, output); !bytes.Equal(output, again) { - t.Errorf("second transformation changed the YAML:\n%s", again) - } - }) - } -} - -func compactLPXSchema(t *testing.T, input []byte) []byte { - t.Helper() - - // Use the same pinned yq binary and expression as make manifests. - command := exec.CommandContext(t.Context(), "../../bin/yq", - "--from-file", "../../api/scripts/lpx-crd-schema.yq", "--indent", "2") - command.Stdin = bytes.NewReader(input) - output, err := command.CombinedOutput() - if err != nil { - t.Fatalf("compact LPX schema (run make yq first): %v\n%s", err, output) - } - return output -} - -func removeSchemaDescriptions(value any) { - // Remove documentation recursively without interpreting or changing any validation marker. - switch value := value.(type) { - case map[string]any: - delete(value, "description") - for _, child := range value { - removeSchemaDescriptions(child) - } - case []any: - for _, child := range value { - removeSchemaDescriptions(child) - } - } -} diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index cc5dd6298568..3fac56fec57a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11880,9 +11880,15 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: &lpx-pod-template + podTemplate: + description: |- + PodTemplate configures Pods for this role. The enclosing component + defines whether the template is required or may be inherited. properties: metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata properties: annotations: additionalProperties: @@ -11902,27 +11908,64 @@ spec: type: string type: object spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. format: int64 type: integer affinity: + description: If specified, the pod's scheduling constraints properties: nodeAffinity: + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: + description: A list of node selector requirements by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11934,13 +11977,27 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: + description: A list of node selector requirements by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11954,6 +12011,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -11963,18 +12021,43 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: + description: A list of node selector requirements by node's labels. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -11986,13 +12069,27 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: + description: A list of node selector requirements by node's fields. items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: + description: The label key that the selector applies to. type: string operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -12013,22 +12110,51 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12042,29 +12168,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12078,20 +12245,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -12101,18 +12286,49 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12126,29 +12342,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12162,15 +12419,30 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12179,22 +12451,51 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12208,29 +12509,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12244,20 +12586,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. format: int32 type: integer required: @@ -12267,18 +12627,49 @@ spec: type: array x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12292,29 +12683,70 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. items: type: string type: array x-kubernetes-list-type: atomic namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -12328,15 +12760,30 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". items: type: string type: array x-kubernetes-list-type: atomic topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string required: - topologyKey @@ -12346,62 +12793,138 @@ spec: type: object type: object automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. type: boolean containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. items: + description: A single application container that you want to run within a pod. properties: args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -12410,29 +12933,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -12447,25 +12985,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -12473,31 +13038,67 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12506,58 +13107,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12566,75 +13213,130 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12643,62 +13345,129 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -12709,39 +13478,71 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12750,51 +13551,99 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -12803,13 +13652,34 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -12825,6 +13695,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -12833,20 +13706,69 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -12861,111 +13783,261 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -12974,61 +14046,134 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -13039,21 +14184,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -13064,6 +14252,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -13073,87 +14266,203 @@ spec: - name x-kubernetes-list-type: map dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. properties: nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. items: type: string type: array x-kubernetes-list-type: atomic options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. items: + description: PodDNSConfigOption defines DNS resolver options of a pod. properties: name: + description: |- + Name is this DNS resolver option's name. + Required. type: string value: + description: Value is this DNS resolver option's value. type: string type: object type: array x-kubernetes-list-type: atomic searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. items: type: string type: array x-kubernetes-list-type: atomic type: object dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. type: boolean ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. properties: args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -13162,29 +14471,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -13199,25 +14523,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13225,31 +14576,63 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: Lifecycle is not allowed for ephemeral containers. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13258,58 +14641,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13318,75 +14747,126 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13395,62 +14875,121 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. type: string ports: + description: Ports are not allowed for ephemeral containers. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -13461,39 +15000,67 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13502,51 +15069,97 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: Resources resize policy for the container. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -13555,13 +15168,33 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -13577,6 +15210,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -13585,20 +15221,49 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + You cannot set this field on ephemeral containers. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. You cannot set this field on + ephemeral containers. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -13613,111 +15278,253 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: Probes are not allowed for ephemeral containers. properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -13726,63 +15533,143 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. type: string terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -13793,21 +15680,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -13818,6 +15748,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -13827,14 +15762,22 @@ spec: - name x-kubernetes-list-type: map hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. properties: hostnames: + description: Hostnames for the above IP address. items: type: string type: array x-kubernetes-list-type: atomic ip: + description: IP address of the host file entry. type: string required: - ip @@ -13844,22 +15787,70 @@ spec: - ip x-kubernetes-list-type: map hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. type: boolean hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + When using HostNetwork you should specify ports so the scheduler is aware. + When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, + and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. + Default to false. type: boolean hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. type: boolean hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. type: boolean hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. type: string hostnameOverride: + description: |- + HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. + This field only specifies the pod's hostname and does not affect its DNS records. + When this field is set to a non-empty string: + - It takes precedence over the values set in `hostname` and `subdomain`. + - The Pod's hostname will be set to this value. + - `setHostnameAsFQDN` must be nil or set to false. + - `hostNetwork` must be set to false. + + This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. + Requires the HostnameOverride feature gate to be enabled. type: string imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -13868,60 +15859,144 @@ spec: - name x-kubernetes-list-type: map initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ items: + description: A single application container that you want to run within a pod. properties: args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell items: type: string type: array x-kubernetes-list-type: atomic env: + description: |- + List of environment variables to set in the container. + Cannot be updated. items: + description: EnvVar represents an environment variable present in a Container. properties: name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". type: string valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: + description: Selects a key of a ConfigMap. properties: key: + description: The key to select. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object x-kubernetes-map-type: atomic fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. type: string optional: default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. type: boolean path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: + description: The name of the volume mount containing the env file. type: string required: - key @@ -13930,29 +16005,44 @@ spec: type: object x-kubernetes-map-type: atomic resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource type: object x-kubernetes-map-type: atomic secretKeyRef: + description: Selects a key of a secret in the pod's namespace properties: key: + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -13967,25 +16057,52 @@ spec: - name x-kubernetes-list-type: map envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets properties: configMapRef: + description: The ConfigMap to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the ConfigMap must be defined type: boolean type: object x-kubernetes-map-type: atomic prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. type: string secretRef: + description: The Secret to select from properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: Specify whether the Secret must be defined type: boolean type: object x-kubernetes-map-type: atomic @@ -13993,31 +16110,67 @@ spec: type: array x-kubernetes-list-type: atomic image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images type: string lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. properties: postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14026,58 +16179,104 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14086,75 +16285,130 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object sleep: + description: Sleep represents a duration that the container should sleep. properties: seconds: + description: Seconds is the number of seconds to sleep. format: int64 type: integer required: - seconds type: object tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name type: string type: object livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14163,62 +16417,129 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: + description: ContainerPort represents a network port in a single container. properties: containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: + description: What host IP to bind the external port to. type: string hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. type: string protocol: default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string required: - containerPort @@ -14229,39 +16550,71 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14270,51 +16623,99 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: + description: ContainerResizePolicy represents resource resize policy for the container. properties: resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. type: string restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. type: string required: - resourceName @@ -14323,13 +16724,34 @@ spec: type: array x-kubernetes-list-type: atomic resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -14345,6 +16767,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -14353,20 +16778,69 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. type: string restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. items: + description: ContainerRestartRule describes how a container exit is handled. properties: action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. type: string exitCodes: + description: Represents the exit codes to check on container exits. properties: operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. type: string values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. items: format: int32 type: integer @@ -14381,111 +16855,261 @@ spec: type: array x-kubernetes-list-type: atomic securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ properties: allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. type: boolean appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. properties: add: + description: Added capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: + description: Removed capabilities items: + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic type: object privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. type: boolean procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. type: boolean runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes properties: exec: + description: Exec specifies a command to execute in the container. properties: command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array x-kubernetes-list-type: atomic type: object failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer grpc: + description: GRPC specifies a GRPC HealthCheckRequest. properties: port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. format: int32 type: integer service: default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. type: string required: - port type: object httpGet: + description: HTTPGet specifies an HTTP GET request to perform. properties: host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. items: + description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. type: string value: + description: The header field value type: string required: - name @@ -14494,61 +17118,134 @@ spec: type: array x-kubernetes-list-type: atomic path: + description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: + description: TCPSocket specifies a connection to a TCP port. properties: host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer type: object stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. type: boolean stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false type: boolean terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. type: string terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. type: string tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. type: boolean volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. items: + description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. type: string name: + description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath @@ -14559,21 +17256,64 @@ spec: - devicePath x-kubernetes-list-type: map volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. items: + description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). type: string name: + description: This must match the Name of a Volume. type: string readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. type: boolean recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. type: string subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). type: string subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -14584,6 +17324,11 @@ spec: - mountPath x-kubernetes-list-type: map workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. type: string required: - name @@ -14593,15 +17338,63 @@ spec: - name x-kubernetes-list-type: map nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename type: string nodeSelector: additionalProperties: type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.resources + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup properties: name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null type: string required: - name @@ -14613,18 +17406,50 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md type: object preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. type: string priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. format: int32 type: integer priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. type: string readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates items: + description: PodReadinessGate contains the reference to a pod condition properties: conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. type: string required: - conditionType @@ -14632,13 +17457,73 @@ spec: type: array x-kubernetes-list-type: atomic resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is a stable field but requires that the + DynamicResourceAllocation feature gate is enabled. + + This field is immutable. items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + + When the DRAWorkloadResourceClaims feature gate is enabled and this Pod + belongs to a PodGroup, a PodResourceClaim is matched to a + PodGroupResourceClaim if all of their fields are equal (Name, + ResourceClaimName, and ResourceClaimTemplateName). A matched claim references + a single ResourceClaim shared across all Pods in the PodGroup, reserved for + the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual + Pods. properties: name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. type: string resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. type: string resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + When the DRAWorkloadResourceClaims feature gate is enabled and the pod + belongs to a PodGroup that defines a PodGroupResourceClaim with the same + Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the + ResourceClaim generated for the PodGroup. All pods in the group that + define an equivalent PodResourceClaim matching the + PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same + generated ResourceClaim. ResourceClaims generated for a PodGroup are + owned by the PodGroup and their lifecycles are tied to the PodGroup + instead of any individual pod. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. type: string required: - name @@ -14648,13 +17533,40 @@ spec: - name x-kubernetes-list-type: map resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. properties: claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. properties: name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. type: string request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. type: string required: - name @@ -14670,6 +17582,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -14678,18 +17593,47 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy type: string runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class type: string schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. type: string schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. properties: name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. type: string required: - name @@ -14699,70 +17643,217 @@ spec: - name x-kubernetes-list-type: map schedulingGroup: + description: |- + SchedulingGroup provides a reference to the immediate scheduling runtime + grouping object that this Pod belongs to. + This field is used by the scheduler to identify the group and apply the + correct group scheduling policies. The association with a group also + impacts other lifecycle aspects of a Pod that are relevant in a wider context + of scheduling like preemption, resource attachment, etc. If not specified, + the Pod is treated as a single unit in all of these aspects. + The group object referenced by this field may not exist at the time the + Pod is created. + This field is immutable, but a group object with the same name may be + recreated with different policies. Doing this during pod scheduling + may result in the placement not conforming to the expected policies. properties: podGroupName: + description: |- + PodGroupName specifies the name of the standalone PodGroup object + that represents the runtime instance of this group. + Must be a DNS subdomain. type: string type: object securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. properties: appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". type: string type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. type: string required: - type type: object fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. type: string runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. type: string seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. properties: level: + description: Level is SELinux level label that applies to the container. type: string role: + description: Role is a SELinux role label that applies to the container. type: string type: + description: Type is a SELinux type label that applies to the container. type: string user: + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. type: string type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. type: string required: - type type: object supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer type: array x-kubernetes-list-type: atomic supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. type: string sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. items: + description: Sysctl defines a kernel parameter to be set properties: name: + description: Name of a property to set type: string value: + description: Value of a property to set type: string required: - name @@ -14771,60 +17862,156 @@ spec: type: array x-kubernetes-list-type: atomic windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. type: string serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ type: string setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. type: boolean shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. type: boolean subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. type: string terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. format: int64 type: integer tolerations: + description: If specified, the pod's tolerations. items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array x-kubernetes-list-type: atomic topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -14838,27 +18025,125 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string type: array x-kubernetes-list-type: atomic maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. format: int32 type: integer minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. format: int32 type: integer nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. type: string nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. type: string topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. type: string whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. type: string required: - maxSkew @@ -14871,111 +18156,244 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore properties: fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). format: int32 type: integer readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: boolean volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore type: string required: - volumeID type: object azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. properties: cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' type: string diskName: + description: diskName is the Name of the data disk in the blob storage type: string diskURI: + description: diskURI is the URI of data disk in the blob storage type: string fsType: default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. properties: readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key type: string shareName: + description: shareName is the azure share Name type: string required: - secretName - shareName type: object cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. properties: monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: boolean secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it type: string required: - monitors type: object cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: boolean secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md type: string required: - volumeID type: object configMap: + description: configMap represents a configMap that should populate this volume properties: defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -14985,67 +18403,129 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. properties: driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. type: string fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. type: string nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: Items is a list of downward API volume file items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource @@ -15058,21 +18538,87 @@ spec: x-kubernetes-list-type: atomic type: object emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir properties: medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir type: string sizeLimit: anyOf: - type: integer - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. properties: volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. properties: metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. properties: annotations: additionalProperties: @@ -15092,19 +18638,42 @@ spec: type: string type: object spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. properties: accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 items: type: string type: array x-kubernetes-list-type: atomic dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. properties: apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. type: string kind: + description: Kind is the type of resource being referenced type: string name: + description: Name is the name of resource being referenced type: string required: - kind @@ -15112,20 +18681,60 @@ spec: type: object x-kubernetes-map-type: atomic dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. properties: apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. type: string kind: + description: Kind is the type of resource being referenced type: string name: + description: Name is the name of resource being referenced type: string namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. type: string required: - kind - name type: object resources: + description: |- + resources represents the minimum resources the volume should have. + Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources properties: limits: additionalProperties: @@ -15134,6 +18743,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object requests: additionalProperties: @@ -15142,18 +18754,37 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object selector: + description: selector is a label query over volumes to consider for binding. properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -15167,16 +18798,38 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 type: string volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ type: string volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. type: string volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object required: @@ -15184,41 +18837,79 @@ spec: type: object type: object fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string lun: + description: 'lun is Optional: FC target lun number' format: int32 type: integer readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' items: type: string type: array x-kubernetes-list-type: atomic wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. items: type: string type: array x-kubernetes-list-type: atomic type: object flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. properties: driver: + description: driver is the name of the driver to use for this volume. type: string fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string + description: 'options is Optional: this field holds extra command options if any.' type: object readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic @@ -15226,98 +18917,230 @@ spec: - driver type: object flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. properties: datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated type: string datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk properties: fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk format: int32 type: integer pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: string readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk type: boolean required: - pdName type: object gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. properties: directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. type: string repository: + description: repository is the URL type: string revision: + description: revision is the commit hash for the specified revision. type: string required: - repository type: object glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. properties: endpoints: + description: endpoints is the endpoint name that details Glusterfs topology. type: string path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: string readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: boolean required: - endpoints - path type: object hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath properties: path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath type: string required: - path type: object image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. properties: pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. type: string reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. type: string type: object iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi properties: chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication type: boolean fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi type: string initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. type: string iqn: + description: iqn is the target iSCSI Qualified Name. type: string iscsiInterface: default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). type: string lun: + description: lun represents iSCSI Target Lun number. format: int32 type: integer portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). items: type: string type: array x-kubernetes-list-type: atomic readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. type: boolean secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). type: string required: - iqn @@ -15325,68 +19148,161 @@ spec: - targetPortal type: object name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs properties: path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: boolean server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs type: string required: - path - server type: object persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims properties: claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. type: boolean required: - claimName type: object photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: + description: pdID is the ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver. properties: fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean volumeID: + description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: + description: projected items for all in one resources secrets, configmaps, and downward API properties: defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. properties: clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. properties: labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". properties: matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: + description: key is the label key that the selector applies to. type: string operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -15400,31 +19316,72 @@ spec: matchLabels: additionalProperties: type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. type: string optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. type: boolean path: + description: Relative path from the volume root to write the bundle. type: string signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. type: string required: - path type: object configMap: + description: configMap information about the configMap data to project properties: items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15434,42 +19391,69 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional specify whether the ConfigMap or its keys must be defined type: boolean type: object x-kubernetes-map-type: atomic downwardAPI: + description: downwardAPI information about the downwardAPI data to project properties: items: + description: Items is a list of DownwardAPIVolume file items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' properties: apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: + description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object x-kubernetes-map-type: atomic mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. properties: containerName: + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: + description: 'Required: resource to select' type: string required: - resource @@ -15482,39 +19466,161 @@ spec: x-kubernetes-list-type: atomic type: object podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. properties: certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. type: string credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. type: string keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. type: string keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". type: string maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. format: int32 type: integer signerName: + description: Kubelet's generated CSRs will be addressed to this signer. type: string userAnnotations: additionalProperties: type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. type: object required: - keyType - signerName type: object secret: + description: secret information about the secret data to project properties: items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15524,19 +19630,42 @@ spec: x-kubernetes-list-type: atomic name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: + description: optional field specify whether the Secret or its key must be defined type: boolean type: object x-kubernetes-map-type: atomic serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project properties: audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. type: string expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. format: int64 type: integer path: + description: |- + path is the path relative to the mount point of the file to project the + token into. type: string required: - path @@ -15546,84 +19675,176 @@ spec: x-kubernetes-list-type: atomic type: object quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. properties: group: + description: |- + group to map volume access to + Default is no group type: string readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. type: boolean registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes type: string tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: + description: |- + user to map volume access to + Defaults to serivceaccount user type: string volume: + description: volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd type: string image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string keyring: default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it items: type: string type: array x-kubernetes-list-type: atomic pool: default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: boolean secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic user: default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it type: string required: - image - monitors type: object scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. properties: fsType: default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". type: string gateway: + description: gateway is the host address of the ScaleIO API Gateway. type: string protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false type: boolean storageMode: default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. type: string storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. type: string system: + description: system is the name of the storage system as configured in ScaleIO. type: string volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. type: string required: - gateway @@ -15631,19 +19852,52 @@ spec: - system type: object secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret properties: defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. items: + description: Maps a string key to a path within a volume. properties: key: + description: key is the key to project. type: string mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. type: string required: - key @@ -15652,37 +19906,81 @@ spec: type: array x-kubernetes-list-type: atomic optional: + description: optional field specify whether the Secret or its keys must be defined type: boolean secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret type: string type: object storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. properties: fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. properties: name: default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string type: object x-kubernetes-map-type: atomic volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. type: string volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. properties: fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. type: string volumePath: + description: volumePath is the path that identifies vSphere volume vmdk type: string required: - volumePath @@ -25160,7 +29458,8122 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - podTemplate: *lpx-pod-template + podTemplate: + description: |- + podTemplate configures Pods for this role. The enclosing component + defines whether the template is required or may be inherited. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + You cannot set this field on ephemeral containers. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. You cannot set this field on + ephemeral containers. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + When using HostNetwork you should specify ports so the scheduler is aware. + When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, + and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + hostnameOverride: + description: |- + HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. + This field only specifies the pod's hostname and does not affect its DNS records. + When this field is set to a non-empty string: + - It takes precedence over the values set in `hostname` and `subdomain`. + - The Pod's hostname will be set to this value. + - `setHostnameAsFQDN` must be nil or set to false. + - `hostNetwork` must be set to false. + + This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. + Requires the HostnameOverride feature gate to be enabled. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.resources + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is a stable field but requires that the + DynamicResourceAllocation feature gate is enabled. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + + When the DRAWorkloadResourceClaims feature gate is enabled and this Pod + belongs to a PodGroup, a PodResourceClaim is matched to a + PodGroupResourceClaim if all of their fields are equal (Name, + ResourceClaimName, and ResourceClaimTemplateName). A matched claim references + a single ResourceClaim shared across all Pods in the PodGroup, reserved for + the PodGroup in ResourceClaimStatus.ReservedFor rather than for individual + Pods. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + When the DRAWorkloadResourceClaims feature gate is enabled and the pod + belongs to a PodGroup that defines a PodGroupResourceClaim with the same + Name and ResourceClaimTemplateName, this PodResourceClaim resolves to the + ResourceClaim generated for the PodGroup. All pods in the group that + define an equivalent PodResourceClaim matching the + PodGroupResourceClaim's Name and ResourceClaimTemplateName share the same + generated ResourceClaim. ResourceClaims generated for a PodGroup are + owned by the PodGroup and their lifecycles are tied to the PodGroup + instead of any individual pod. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + schedulingGroup: + description: |- + SchedulingGroup provides a reference to the immediate scheduling runtime + grouping object that this Pod belongs to. + This field is used by the scheduler to identify the group and apply the + correct group scheduling policies. The association with a group also + impacts other lifecycle aspects of a Pod that are relevant in a wider context + of scheduling like preemption, resource attachment, etc. If not specified, + the Pod is treated as a single unit in all of these aspects. + The group object referenced by this field may not exist at the time the + Pod is created. + This field is immutable, but a group object with the same name may be + recreated with different policies. Doing this during pod scheduling + may result in the placement not conforming to the expected policies. + properties: + podGroupName: + description: |- + PodGroupName specifies the name of the standalone PodGroup object + that represents the runtime instance of this group. + Must be a DNS subdomain. + type: string + type: object + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string or nil value indicates that no + VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, + this field can be reset to its previous value (including nil) to cancel the modification. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + properties: + endpoints: + description: endpoints is the endpoint name that details Glusterfs topology. + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS client or server. + + Kubelet generates a private key and uses it to send a + PodCertificateRequest to the named signer. Once the signer approves the + request and issues a certificate chain, Kubelet writes the key and + certificate chain to the pod filesystem. The pod does not start until + certificates have been issued for each podCertificate projected volume + source in its spec. + + Kubelet will begin trying to rotate the certificate at the time indicated + by the signer using the PodCertificateRequest.Status.BeginRefreshAt + timestamp. + + Kubelet can write a single file, indicated by the credentialBundlePath + field, or separate files, indicated by the keyPath and + certificateChainPath fields. + + The credential bundle is a single file in PEM format. The first PEM + entry is the private key (in PKCS#8 format), and the remaining PEM + entries are the certificate chain issued by the signer (typically, + signers will return their certificate chain in leaf-to-root order). + + Prefer using the credential bundle format, since your application code + can read it atomically. If you use keyPath and certificateChainPath, + your application must make two separate file reads. If these coincide + with a certificate rotation, it is possible that the private key and leaf + certificate you read may not correspond to each other. Your application + will need to check for this condition, and re-read until they are + consistent. + + The named signer controls chooses the format of the certificate it + issues; consult the signer implementation's documentation to learn how to + use the certificates it issues. + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + credentialBundlePath: + description: |- + Write the credential bundle at this path in the projected volume. + + The credential bundle is a single file that contains multiple PEM blocks. + The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private + key. + + The remaining blocks are CERTIFICATE blocks, containing the issued + certificate chain from the signer (leaf and any intermediates). + + Using credentialBundlePath lets your Pod's application code make a single + atomic read that retrieves a consistent key and certificate chain. If you + project them to separate files, your application code will need to + additionally check that the leaf certificate was issued to the key. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. When using keyPath + and certificateChainPath, your application needs to check that the key + and leaf certificate are consistent, because it is possible to read the + files mid-rotation. + type: string + keyType: + description: |- + The type of keypair Kubelet will generate for the pod. + + Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", + "ECDSAP521", and "ED25519". + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + + Kubelet copies this value verbatim into the PodCertificateRequests it + generates for this projection. + + If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver + will reject values shorter than 3600 (1 hour). The maximum allowable + value is 7862400 (91 days). + + The signer implementation is then free to issue a certificate with any + lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 + seconds (1 hour). This constraint is enforced by kube-apiserver. + `kubernetes.io` signers will never issue certificates with a lifetime + longer than 24 hours. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. + type: object + required: + - keyType + - signerName + type: object + secret: + description: secret information about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object providerOverride: description: |- providerOverride configures the provider workload unit generated for this From a1f17d9bd4918c970ce1a237b0509ff53398b3a0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 12 Sep 2026 21:01:37 +0000 Subject: [PATCH 044/196] fix(operator): align Python generation with upstream and refresh schemas Signed-off-by: Andrew Paprotsky (cherry picked from commit aa286c80a8beae7ccdca382ca5f7e3360a13a51b) --- deploy/operator/Makefile | 9 ++++---- .../dynamo-openapi/dynamo-openapi.json | 21 +++++++++++++++++++ .../dynamo-openapi/dynamo-openapi.json | 21 +++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 6a4ecbff0226..5ced3074e3ca 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -46,8 +46,6 @@ CONTAINER_TOOL ?= docker SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec -PYTHON ?= uv run --no-project --python '3.10' --with pydantic --with pyyaml --with black python3 - .PHONY: all all: build @@ -292,10 +290,11 @@ generate: controller-gen generate-pydantic ## Generate code containing DeepCopy, .PHONY: generate-pydantic generate-pydantic: ## Generate Python Pydantic models from v1beta1 Go types (requires Python 3 + pydantic) + @python3 -c "import pydantic" 2>/dev/null || { echo "Error: pydantic not found. Install with: pip install pydantic"; exit 1; } @echo "Generating Pydantic models from v1beta1 DGDR types..." - $(PYTHON) api/scripts/generate_pydantic_from_go.py + @python3 api/scripts/generate_pydantic_from_go.py @echo "Running Pydantic validation tests..." - $(PYTHON) api/scripts/validate_pydantic_models.py + @python3 api/scripts/validate_pydantic_models.py .PHONY: fmt fmt: ## Run go fmt against code. @@ -662,7 +661,7 @@ generate-api-docs: crd-ref-docs ## Generate API reference documentation from CRD rm ./docs/api_reference.md @echo "✅ Concatenated header.md, api_reference.md, and footer.md" # Post-process duplicate anchors and fields pruned from standalone DCD CRDs. - $(PYTHON) docs/fix-api-anchors.py ../../docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md + python3 docs/fix-api-anchors.py ../../docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md HELM_CHART_DIR := ../helm/charts/platform diff --git a/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json b/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json index f11f380929f5..ec25498dfb36 100644 --- a/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json +++ b/recipes/kustomize/components/dynamo-openapi/dynamo-openapi.json @@ -1563,6 +1563,27 @@ } ] }, + "nvidia.com.v1alpha1.LPXGraphDeployment": { + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + } + }, + "type": "object", + "x-kubernetes-group-version-kind": [ + { + "group": "nvidia.com", + "kind": "LPXGraphDeployment", + "version": "v1alpha1" + } + ] + }, "nvidia.com.v1beta1.DynamoComponentDeployment": { "properties": { "apiVersion": { diff --git a/recipes/templates/kustomize/components/dynamo-openapi/dynamo-openapi.json b/recipes/templates/kustomize/components/dynamo-openapi/dynamo-openapi.json index f11f380929f5..ec25498dfb36 100644 --- a/recipes/templates/kustomize/components/dynamo-openapi/dynamo-openapi.json +++ b/recipes/templates/kustomize/components/dynamo-openapi/dynamo-openapi.json @@ -1563,6 +1563,27 @@ } ] }, + "nvidia.com.v1alpha1.LPXGraphDeployment": { + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + } + }, + "type": "object", + "x-kubernetes-group-version-kind": [ + { + "group": "nvidia.com", + "kind": "LPXGraphDeployment", + "version": "v1alpha1" + } + ] + }, "nvidia.com.v1beta1.DynamoComponentDeployment": { "properties": { "apiVersion": { From ad8e4413dface6406fcd10d1920c5a84882c047b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:11:45 +0300 Subject: [PATCH 045/196] fix(operator): bound local build inventory acquisition (#473) Signed-off-by: Andrew Paprotsky (cherry picked from commit 8e79929838d6e6097c225fdfd921bb3ced1c23a5) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/build_snapshot.go | 12 +- .../dynamo/lpx/build_snapshot_test.go | 35 +++- .../internal/dynamo/lpx/model_registry.go | 95 ++++++++--- .../dynamo/lpx/model_registry_test.go | 159 ++++++++++++++++++ 4 files changed, 276 insertions(+), 25 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot.go b/deploy/operator/internal/dynamo/lpx/build_snapshot.go index 84cf45dd40fa..5b4440adcf79 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot.go @@ -224,9 +224,12 @@ func normalizeModelPath(path string) string { } // normalizeBuildFilePaths consumes a nonnil owned path slice and canonicalizes it in place. -func normalizeBuildFilePaths(paths []string) ([]string, error) { +func normalizeBuildFilePaths(ctx context.Context, paths []string) ([]string, error) { // Normalize and retain compiler evidence while validating every listed path. for index, rawPath := range paths { + if err := ctx.Err(); err != nil { + return nil, err + } path := normalizeModelPath(rawPath) if path == "" || path == "." || filepath.IsAbs(path) || path == ".." || strings.HasPrefix(path, "../") { return nil, fmt.Errorf("build inventory contains invalid relative path %q", rawPath) @@ -235,11 +238,18 @@ func normalizeBuildFilePaths(paths []string) ([]string, error) { } // Canonical ordering makes duplicate paths adjacent without a second index. + // Sorting itself is not interruptible; observe cancellation in the following checks. sort.Strings(paths) for i := 1; i < len(paths); i++ { + if err := ctx.Err(); err != nil { + return nil, err + } if paths[i] == paths[i-1] { return nil, fmt.Errorf("build inventory repeats relative path %q", paths[i]) } } + if err := ctx.Err(); err != nil { + return nil, err + } return paths, nil } diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go index 9ca75a7f724d..d54b64f91dcd 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go @@ -6,6 +6,7 @@ package lpx import ( + "context" "net/url" "os" "path/filepath" @@ -36,7 +37,7 @@ func TestAcquireBuildSnapshotTracksLocalContent(t *testing.T) { } t.Log("Inventory every file beneath the build root, including metadata, artifacts and weights") - paths, err := localBuildFilePaths(buildDir) + paths, err := localBuildFilePaths(t.Context(), buildDir, maxBuildSnapshotMetadataBytes) require.NoError(t, err) require.ElementsMatch(t, []string{ "extra-metadata.json", @@ -119,7 +120,7 @@ func TestNormalizeBuildFilePathsCanonicalizesAndRejectsAmbiguity(t *testing.T) { t.Parallel() t.Log("Canonicalize and sort an unambiguous build inventory") - inventory, err := normalizeBuildFilePaths([]string{"z/file.gas", gbuildManifestJSONFile, "./a/file.gas"}) + inventory, err := normalizeBuildFilePaths(t.Context(), []string{"z/file.gas", gbuildManifestJSONFile, "./a/file.gas"}) require.NoError(t, err) require.Equal(t, []string{"a/file.gas", gbuildManifestJSONFile, "z/file.gas"}, inventory) @@ -131,8 +132,36 @@ func TestNormalizeBuildFilePathsCanonicalizesAndRejectsAmbiguity(t *testing.T) { } { t.Run(name, func(t *testing.T) { t.Log("Reject the selected ambiguous or unsafe inventory path") - _, err := normalizeBuildFilePaths(paths) + _, err := normalizeBuildFilePaths(t.Context(), paths) require.Error(t, err) }) } } + +func TestNormalizeBuildFilePathsCancellation(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + paths []string + checks int + }{ + {name: "empty inventory", paths: []string{}, checks: 1}, + {name: "validation", paths: []string{"z", "b", "a"}, checks: 2}, + {name: "after sorting", paths: []string{"z", "b", "a"}, checks: 4}, + {name: "duplicate checking", paths: []string{"z", "b", "a"}, checks: 5}, + {name: "before returning", paths: []string{"z", "b", "a"}, checks: 6}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Log("Cancel deterministically at the selected inventory-normalization stage") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + normalizeCtx := &cancelAfterChecksContext{Context: ctx, cancel: cancel, remaining: tt.checks} + + t.Log("Return cancellation without publishing a normalized inventory") + paths, err := normalizeBuildFilePaths(normalizeCtx, tt.paths) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, paths) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go index 3d2224d4c701..10515db3fcc1 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -305,7 +305,7 @@ func (r *ModelRegistry) listBuildFiles(ctx context.Context, buildURL *url.URL) ( ) switch buildURL.Scheme { case BuildSchemeFile: - paths, err = localBuildFilePaths(buildURL.Path) + paths, err = localBuildFilePaths(ctx, buildURL.Path, maxBuildSnapshotMetadataBytes) case BuildSchemeGCS: if r.mxClient == nil { return nil, fmt.Errorf("Model Express client is required for GCS model registry reads of %q", buildURL.String()) @@ -332,35 +332,88 @@ func (r *ModelRegistry) listBuildFiles(ctx context.Context, buildURL *url.URL) ( if err != nil { return nil, err } - return normalizeBuildFilePaths(paths) + return normalizeBuildFilePaths(ctx, paths) } -func localBuildFilePaths(root string) ([]string, error) { - cleanRoot := filepath.Clean(root) - resolvedRoot, err := filepath.EvalSymlinks(cleanRoot) +// localBuildFilePaths bounds visited relative path bytes; maxBytes must be non-negative. +// Cancellation is cooperative and cannot interrupt an in-flight filesystem call. +func localBuildFilePaths(ctx context.Context, root string, maxBytes int) ([]string, error) { + // Resolve a symlinked build root only while acquisition is still active. + if err := ctx.Err(); err != nil { + return nil, err + } + resolvedRoot, err := filepath.EvalSymlinks(filepath.Clean(root)) if err != nil { return nil, fmt.Errorf("resolving build directory %q: %w", root, err) } - // Keep a malformed manifest directory in the inventory so its required file read fails. - manifestPath := filepath.Join(resolvedRoot, gbuildManifestV2CapnpFile) + // Reject non-directory roots without opening a potentially blocking FIFO. + if err := ctx.Err(); err != nil { + return nil, err + } + info, err := os.Stat(resolvedRoot) + if err != nil { + return nil, fmt.Errorf("checking build directory %q: %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("build root %q is not a directory", root) + } + + // Charge directories as well as files so pending traversal cannot grow without bound. paths := make([]string, 0) - err = filepath.WalkDir(resolvedRoot, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() && path != manifestPath { - return nil - } - rel, err := filepath.Rel(resolvedRoot, path) + pending := []string{""} + remaining := maxBytes + for len(pending) > 0 { + relativeDir := pending[len(pending)-1] + pending[len(pending)-1] = "" + pending = pending[:len(pending)-1] + + // Finish and close one directory before opening another, including on failure. + err := func() error { + if err := ctx.Err(); err != nil { + return err + } + dir, err := os.Open(filepath.Join(resolvedRoot, relativeDir)) + if err != nil { + return err + } + defer func() { _ = dir.Close() }() + + // Batches avoid WalkDir's whole-directory allocation before entry callbacks. + for { + if err := ctx.Err(); err != nil { + return err + } + entries, err := dir.ReadDir(128) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + + // Budget each path before retaining it, without following child symlinks. + for _, entry := range entries { + relativePath := filepath.Join(relativeDir, entry.Name()) + if len(relativePath) > remaining { + return fmt.Errorf("build inventory exceeds its %d-byte path limit", maxBytes) + } + remaining -= len(relativePath) + + // Retain malformed manifest directories so the required file read fails. + if entry.IsDir() { + pending = append(pending, relativePath) + if relativePath != gbuildManifestV2CapnpFile { + continue + } + } + paths = append(paths, filepath.ToSlash(relativePath)) + } + if errors.Is(err, io.EOF) { + return ctx.Err() + } + } + }() if err != nil { - return err + return nil, fmt.Errorf("listing build files under %q: %w", root, err) } - paths = append(paths, filepath.ToSlash(rel)) - return nil - }) - if err != nil { - return nil, fmt.Errorf("listing build files under %q: %w", root, err) } return paths, nil } diff --git a/deploy/operator/internal/dynamo/lpx/model_registry_test.go b/deploy/operator/internal/dynamo/lpx/model_registry_test.go index 9e91d0833254..06e52f2e3ee0 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry_test.go @@ -8,6 +8,7 @@ package lpx import ( "context" "errors" + "fmt" "io" "net/url" "os" @@ -15,6 +16,7 @@ import ( "strings" "syscall" "testing" + "time" modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" "github.com/stretchr/testify/require" @@ -105,6 +107,163 @@ func TestLocalModelRegistrySnapshot_ReportsMetadataDirectories(t *testing.T) { require.ErrorContains(t, err, gbuildManifestV2CapnpFile) } +func TestAcquireLocalBuildSnapshotCancellation(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + timeout time.Duration + wantErr error + }{ + {name: "canceled", timeout: time.Hour, wantErr: context.Canceled}, + {name: "expired", timeout: -time.Second, wantErr: context.DeadlineExceeded}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Log("Create a valid local build with an already canceled or expired acquisition context") + buildDir := t.TempDir() + writeManifestV2Payload(t, buildDir, manifestV2Payload(t)) + ctx, cancel := context.WithTimeout(t.Context(), tt.timeout) + defer cancel() + if tt.wantErr == context.Canceled { + cancel() + } + + t.Log("Reject the acquisition without returning a snapshot") + registry := &ModelRegistry{} + snapshot, err := registry.AcquireBuildSnapshot(ctx, buildDir) + require.ErrorIs(t, err, tt.wantErr) + require.Nil(t, snapshot) + }) + } +} + +func TestLocalBuildFilePathsBudget(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + directories []string + files []string + maxBytes int + wantErr bool + }{ + {name: "empty"}, + {name: "files exact", files: []string{"a", "b"}, maxBytes: 2}, + {name: "files over", files: []string{"a", "b"}, maxBytes: 1, wantErr: true}, + {name: "nested exact", directories: []string{"d"}, files: []string{"d/f"}, maxBytes: 4}, + {name: "nested over", directories: []string{"d"}, files: []string{"d/f"}, maxBytes: 3, wantErr: true}, + {name: "directories exact", directories: []string{"a", "b"}, maxBytes: 2}, + {name: "directories over", directories: []string{"a", "b"}, maxBytes: 1, wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Log("Create the selected tree of files and empty directories") + root := t.TempDir() + for _, directory := range tt.directories { + require.NoError(t, os.MkdirAll(filepath.Join(root, directory), 0o700)) + } + for _, file := range tt.files { + require.NoError(t, os.WriteFile(filepath.Join(root, file), nil, 0o600)) + } + + t.Log("Charge every visited relative path and never return a truncated inventory") + paths, err := localBuildFilePaths(t.Context(), root, tt.maxBytes) + if tt.wantErr { + require.ErrorContains(t, err, "path limit") + require.Nil(t, paths) + return + } + require.NoError(t, err) + require.ElementsMatch(t, tt.files, paths) + }) + } +} + +type cancelAfterChecksContext struct { + context.Context + cancel context.CancelFunc + remaining int +} + +func (ctx *cancelAfterChecksContext) Err() error { + // Cancel the real context deterministically at the selected cooperative check. + ctx.remaining-- + if ctx.remaining == 0 { + ctx.cancel() + } + return ctx.Context.Err() +} + +func TestLocalBuildFilePathsBatchesAndCancellation(t *testing.T) { + t.Parallel() + + t.Log("Create more files than fit in one directory batch") + root := t.TempDir() + files := make([]string, 300) + for index := range files { + files[index] = fmt.Sprintf("%03d", index) + require.NoError(t, os.WriteFile(filepath.Join(root, files[index]), nil, 0o600)) + } + + t.Log("Read all batches at the exact aggregate path limit") + paths, err := localBuildFilePaths(t.Context(), root, 3*len(files)) + require.NoError(t, err) + require.ElementsMatch(t, files, paths) + + t.Log("Cancel after traversal has passed the first batch without relying on a timer") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + walkCtx := &cancelAfterChecksContext{Context: ctx, cancel: cancel, remaining: 5} + paths, err = localBuildFilePaths(walkCtx, root, 3*len(files)) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, paths) +} + +func TestLocalBuildFilePathsSymlinks(t *testing.T) { + t.Parallel() + + t.Log("Create a symlinked root with a file, a child directory link and a dangling link") + parent := t.TempDir() + root := filepath.Join(parent, "build") + require.NoError(t, os.Mkdir(root, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "file"), nil, 0o600)) + require.NoError(t, os.Symlink(".", filepath.Join(root, "loop"))) + require.NoError(t, os.Symlink("missing", filepath.Join(root, "dangling"))) + rootLink := filepath.Join(parent, "linked-build") + require.NoError(t, os.Symlink(root, rootLink)) + + t.Log("Resolve only the root link and inventory child links without following them") + paths, err := localBuildFilePaths(t.Context(), rootLink, maxBuildSnapshotMetadataBytes) + require.NoError(t, err) + require.ElementsMatch(t, []string{"file", "loop", "dangling"}, paths) +} + +func TestLocalBuildFilePathsRejectsNonDirectoryRoot(t *testing.T) { + t.Parallel() + + for _, name := range []string{"file", "fifo"} { + t.Run(name, func(t *testing.T) { + t.Log("Create a non-directory build root") + root := filepath.Join(t.TempDir(), name) + if name == "fifo" { + require.NoError(t, syscall.Mkfifo(root, 0o600)) + + t.Log("Keep both FIFO ends open so a missing root check fails without hanging the test") + fifo, err := os.OpenFile(root, os.O_RDWR|syscall.O_NONBLOCK, 0) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, fifo.Close()) }) + } else { + require.NoError(t, os.WriteFile(root, nil, 0o600)) + } + + t.Log("Reject the root before opening it for directory reads") + paths, err := localBuildFilePaths(t.Context(), root, maxBuildSnapshotMetadataBytes) + require.ErrorContains(t, err, "build root ") + require.ErrorContains(t, err, "not a directory") + require.Nil(t, paths) + }) + } +} + func TestReadBuildFileBoundsLocalFileAtAcquisition(t *testing.T) { t.Parallel() From a23b576f6897bebd66c884902b8af2aee169d6c1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:11:46 +0300 Subject: [PATCH 046/196] fix(operator): reject conductor SSH key volume collisions (#475) Signed-off-by: Andrew Paprotsky (cherry picked from commit 6431eae2b76b80fa8eb57ccf29ddcfc0d6dc77cd) Signed-off-by: Andrew Paprotsky --- .../dgd_lpx_handoff_envtest_test.go | 28 +++++++++++---- .../internal/dynamo/lpx/runtime_ssh.go | 9 ++++- .../internal/dynamo/lpx/runtime_ssh_test.go | 35 +++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index df931c557d7d..a8ca9975eb60 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -189,8 +190,13 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { }, 20*time.Second, 50*time.Millisecond) t.Log("Change LPX intent so the failing child generation is newer than its completed observation") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "example/lpu-runtime:1.4.1" - require.NoError(t, env.Client().Update(t.Context(), source)) + require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := env.Client().Get(t.Context(), key, source); err != nil { + return err + } + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "example/lpu-runtime:1.4.1" + return env.Client().Update(t.Context(), source) + })) child := &v1alpha1.LPXGraphDeployment{} require.EventuallyWithT(t, func(c *assert.CollectT) { if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { @@ -223,8 +229,13 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { require.Empty(t, requests.Items) t.Log("Accept an LPX runtime-invalid edit and report its exact rejection on the public DGD") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "manual-placement" - require.NoError(t, env.Client().Update(t.Context(), source)) + require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := env.Client().Get(t.Context(), key, source); err != nil { + return err + } + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "manual-placement" + return env.Client().Update(t.Context(), source) + })) require.EventuallyWithT(t, func(c *assert.CollectT) { if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { return @@ -246,8 +257,13 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { }, 20*time.Second, 50*time.Millisecond) t.Log("Repair the accepted configuration so the same DGD and child can reconcile again") - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "" - require.NoError(t, env.Client().Update(t.Context(), source)) + require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := env.Client().Get(t.Context(), key, source); err != nil { + return err + } + source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "" + return env.Client().Update(t.Context(), source) + })) t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") require.NoError(t, env.Client().Delete(t.Context(), quota)) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index 62e9401c10ba..5ae01039a282 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -62,13 +62,20 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se return err } + // Reject authored destination storage before changing the source Secret volume. + for _, volume := range podSpec.Volumes { + if volume.Name == conductorSSHKeyVolumeName { + return fmt.Errorf("selected LPX podTemplate volume %q is reserved for the conductor SSH key", conductorSSHKeyVolumeName) + } + } + // Mount the source Secret and the writable destination used by OpenMPI. if err := addSSHVolume(podSpec, secretName, 0600); err != nil { return err } // Keep the copied private key in a Pod-local writable volume. - podSpec.Volumes = setVolumeByName(podSpec.Volumes, corev1.Volume{ + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go index e0e7493ef248..417282840ec1 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go @@ -46,6 +46,41 @@ func TestAddSSHVolumePreservesAuthoredStorage(t *testing.T) { } } +func TestAddConductorSSHKeyRejectsAuthoredKeyVolume(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + source corev1.VolumeSource + }{ + {"Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}}, + {"PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, + {"EmptyDir", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Mount authored storage using the reserved conductor key-volume name") + pod := corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, Image: "lpu-runtime", + VolumeMounts: []corev1.VolumeMount{{Name: conductorSSHKeyVolumeName, MountPath: "/application-data"}}, + }}, + Volumes: []corev1.Volume{ + {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: "mpi.ssh", DefaultMode: ptr.To(int32(0644)), + }}}, + {Name: conductorSSHKeyVolumeName, VolumeSource: test.source}, + }, + } + before := pod.DeepCopy() + + t.Log("Reject the collision before changing storage, mounts or source Secret permissions") + err := addConductorSSHKey(&pod, &pod.Containers[0], "mpi.ssh") + require.ErrorContains(t, err, `volume "single-v2-ssh-key" is reserved for the conductor SSH key`) + require.Equal(t, *before, pod) + }) + } +} + func TestXTRuntimesRejectSSHVolumeCollision(t *testing.T) { t.Log("Start each XT role with an authored volume using the fixed SSH name") base := corev1.PodSpec{ From c2eee544b3c4a3bdca72b3b65e41dd8de43f000f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:11:46 +0300 Subject: [PATCH 047/196] fix(operator): reject ignored LPX draft minimum availability (#476) Signed-off-by: Andrew Paprotsky (cherry picked from commit a98fdd8543c273b5dd8d0c72ee271fd70fb9011d) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/selection_test.go | 78 +++++++++++++++++++ .../dynamo/lpx/selection_validation.go | 4 + 2 files changed, 82 insertions(+) diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go index 8b66040d9060..265d6a0c95d2 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_test.go +++ b/deploy/operator/internal/dynamo/lpx/selection_test.go @@ -10,8 +10,14 @@ import ( dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" "github.com/stretchr/testify/require" + admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { @@ -115,3 +121,75 @@ func TestSelectedModelNamesRequiresSharedConductorOwner(t *testing.T) { require.Nil(t, ServingComponent(dgd)) require.Equal(t, before, dgd) } + +func TestSelectedMinAvailableOwnership(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + draftMin *int32 + targetMin *int32 + singleton bool + implicitConductor bool + wantErr bool + }{ + {name: "omitted draft"}, + {name: "default draft", draftMin: ptr.To(int32(1))}, + {name: "nondefault draft", draftMin: ptr.To(int32(2)), wantErr: true}, + {name: "target", targetMin: ptr.To(int32(1))}, + {name: "singleton", targetMin: ptr.To(int32(2)), singleton: true}, + {name: "implicit singleton conductor", targetMin: ptr.To(int32(2)), singleton: true, implicitConductor: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Put a non-LPX component before the target and reverse the authored LPX order") + target := testLPXComponent("target", "target-build", + dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor}, + dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}) + target.MinAvailable = test.targetMin + if test.singleton { + target.Replicas = ptr.To(int32(2)) + } + if test.implicitConductor { + target.Roles = target.Roles[1:] + } + dgd := newSelectedTestDGD(t, "min-available", + dynamov1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", MinAvailable: ptr.To(int32(1))}, target) + if !test.singleton { + draft := testLPXComponent("draft", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}) + draft.Replicas = ptr.To(int32(2)) + draft.MinAvailable = test.draftMin + dgd.Spec.Components = append(dgd.Spec.Components, draft) + } + + for _, phase := range []string{"authored", "defaulted"} { + t.Run(phase, func(t *testing.T) { + if phase == "defaulted" { + t.Log("Apply the real Grove CREATE defaulter before selected-LPX validation") + ctx := features.WithGate(t.Context(), features.Gates{Grove: true}) + ctx = admission.NewContextWithRequest(ctx, admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Kind: metav1.GroupVersionKind(dynamov1beta1.DynamoGraphDeploymentGVK), + }}) + require.NoError(t, defaulting.NewDGDDefaulter("test").Default(ctx, dgd)) + if !test.singleton { + require.Equal(t, ptr.To(ptr.Deref(test.draftMin, 1)), dgd.Spec.Components[2].MinAvailable) + } + } + before := dgd.DeepCopy() + + t.Log("Reject only nondefault draft availability at its authored index without mutation") + errs := ValidateSelectedIntent(dgd) + require.Equal(t, before, dgd) + if test.wantErr { + require.Len(t, errs, 1) + require.Equal(t, field.ErrorTypeForbidden, errs[0].Type) + require.Equal(t, "spec.components[2].minAvailable", errs[0].Field) + require.Contains(t, errs[0].Detail, "must be omitted or 1") + return + } + require.Empty(t, errs) + }) + } + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index 1ce092898239..3978b968dc6a 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -93,6 +93,10 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC if component.ModelRef != nil { allErrs = append(allErrs, field.Forbidden(componentPath.Child("modelRef"), "the shared target owns the serving endpoint")) } + // Grove persists its immutable default of 1 even for drafts. + if ptr.Deref(component.MinAvailable, 1) != 1 { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("minAvailable"), "draft minAvailable must be omitted or 1; the shared target owns minimum availability")) + } } else if replicas != 1 { allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "shared target replicas must be one")) } From 9dd047079d410dfca24d625aba332f6caead3b75 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sun, 13 Sep 2026 18:20:45 +0000 Subject: [PATCH 048/196] fix(operator): reject non-regular local manifests Signed-off-by: Andrew Paprotsky (cherry picked from commit e922a166aa41f8d5b9ccd1241c3f071fb262427f) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/model_registry.go | 15 ++++++- .../dynamo/lpx/model_registry_test.go | 40 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go index 10515db3fcc1..ad64b2118d58 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "strings" + "syscall" modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" ) @@ -191,15 +192,25 @@ func (r *ModelRegistry) readBuildFileBounded( ) ([]byte, error) { switch buildURL.Scheme { case BuildSchemeFile: + // Open nonblocking so a FIFO cannot stall before file-type validation. fileURL := buildURL.JoinPath(relativePath) - file, err := os.Open(filepath.Clean(fileURL.Path)) + file, err := os.OpenFile(filepath.Clean(fileURL.Path), os.O_RDONLY|syscall.O_NONBLOCK, 0) if err != nil { return nil, fmt.Errorf("reading %s: %w", fileURL.String(), err) } defer func() { _ = file.Close() }() - if info, statErr := file.Stat(); statErr == nil && info.Size() > int64(maxBytes) { + + // Validate the opened file, not a path that could be replaced. + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("checking %s: %w", fileURL.String(), err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("reading %s: not a regular file", fileURL.String()) + } + if info.Size() > int64(maxBytes) { return nil, fmt.Errorf( "%w: %s is %d bytes, limit is %d", errBuildFileTooLarge, diff --git a/deploy/operator/internal/dynamo/lpx/model_registry_test.go b/deploy/operator/internal/dynamo/lpx/model_registry_test.go index 06e52f2e3ee0..9dfb8ba8e30b 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry_test.go @@ -12,6 +12,7 @@ import ( "io" "net/url" "os" + "os/exec" "path/filepath" "strings" "syscall" @@ -137,6 +138,38 @@ func TestAcquireLocalBuildSnapshotCancellation(t *testing.T) { } } +func TestAcquireLocalBuildSnapshotRejectsFIFOManifest(t *testing.T) { + t.Parallel() + + const buildDirEnv = "DYNAMO_TEST_FIFO_MANIFEST_BUILD_DIR" + if buildDir := os.Getenv(buildDirEnv); buildDir != "" { + t.Log("Reject the FIFO manifest while the acquisition context is still active") + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + snapshot, err := (&ModelRegistry{}).AcquireBuildSnapshot(ctx, buildDir) + require.ErrorContains(t, err, "not a regular file") + require.ErrorContains(t, err, gbuildManifestV2CapnpFile) + require.Nil(t, snapshot) + require.NoError(t, ctx.Err()) + return + } + + t.Log("Create a valid build directory containing a FIFO manifest with no writer") + buildDir := t.TempDir() + require.NoError(t, syscall.Mkfifo(filepath.Join(buildDir, gbuildManifestV2CapnpFile), 0o600)) + + t.Log("Bound the subprocess so a regressed FIFO open cannot hang or leak a goroutine") + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, executable, "-test.run=^"+t.Name()+"$", "-test.v") + cmd.Env = append(os.Environ(), buildDirEnv+"="+buildDir) + output, err := cmd.CombinedOutput() + require.NoError(t, ctx.Err(), "FIFO manifest acquisition blocked past its deadline: %s", output) + require.NoError(t, err, "%s", output) +} + func TestLocalBuildFilePathsBudget(t *testing.T) { t.Parallel() @@ -279,6 +312,13 @@ func TestReadBuildFileBoundsLocalFileAtAcquisition(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("12345"), data) + t.Log("Read the same metadata through a symlink at the inclusive acquisition limit") + const metadataLink = "metadata-link.bin" + require.NoError(t, os.Symlink(metadataFile, filepath.Join(buildDir, metadataLink))) + data, err = registry.readBuildFileBounded(t.Context(), buildURL, metadataLink, 5) + require.NoError(t, err) + require.Equal(t, []byte("12345"), data) + t.Log("Reject the same file above a four-byte limit") _, err = registry.readBuildFileBounded(t.Context(), buildURL, metadataFile, 4) require.ErrorIs(t, err, errBuildFileTooLarge) From 74eac8ffb9c3b98387ef16b5dd5a293fcee76af6 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 06:25:05 +0000 Subject: [PATCH 049/196] fix(operator): add copyright headers to LPX fixtures Signed-off-by: Andrew Paprotsky (cherry picked from commit 686a9672df63f7039fb53cd042f9d606f402d608) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/render_test.go | 4 +++- .../dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml | 3 +++ .../internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml | 3 +++ .../dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml | 3 +++ .../dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml | 3 +++ .../from_dgd_yaml/node-local-v2-specdecode.input.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v2-specdecode.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml | 3 +++ .../from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 3 +++ .../from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml | 3 +++ .../testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml | 3 +++ .../dynamo/testdata/from_dgd_yaml/single_v2.input.yaml | 3 +++ .../internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml | 3 +++ .../from_dgd_yaml/single_v2_manifest_defaults.input.yaml | 3 +++ 18 files changed, 54 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 8fe2cd112c41..60034ab3f018 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1081,7 +1081,9 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { } goldenPath, err := filepath.Abs("../../dynamo/testdata/" + name + ".yaml") require.NoError(t, err) - golden.Assert(t, strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) + const header = "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" + + "# SPDX-License-Identifier: Apache-2.0\n\n" + golden.Assert(t, header+strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) }) } } diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index 1a64ce9c8084..af6952134982 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index e12bfe489032..6734957ab721 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: lpu_servers: |- lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index bb3c89febf59..26b02660fe76 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 8737b165090e..c3e8a6b3256d 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: lpu_servers: lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 tokenizer_dir: /testdata/node-local-v2-connected-lpx/tokenizer diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 678289555aea..6395ae8aa487 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index b3b0fba58bcf..4459148b4d01 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index 29a4b9bce171..96cae27b567c 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index fb5128c5b013..92ba3c6a06e5 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index b694a9cbbb8d..94f9d94e4a01 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 79e024cbf2fb..1f929dab902b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index e28d188b83aa..8593af620cdc 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index c97b7f142938..156e82ce76d5 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 2e1dd2f3a4a4..7b3e4306421b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 1a571d96eddd..cb1081964b50 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml index 1574076d33a7..f33a56b3dfda 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 84fd350f47b0..e9620c4336d0 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + data: datacenter.toml: | model_base_paths = ['/nfs'] diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml index ec1ee0c8ae71..601e82652695 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: From 9469d084e7dc28bae272ba653f91a192d6dcc414 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 06:45:04 +0000 Subject: [PATCH 050/196] fix(operator): validate LPX host-device volumes Signed-off-by: Andrew Paprotsky (cherry picked from commit c165b758004025d217d17e86c15f4405c04cddd1) Signed-off-by: Andrew Paprotsky --- .../operator/internal/dynamo/lpx/lpu_pod.go | 20 ++++++- .../internal/dynamo/lpx/pod_roles_test.go | 59 +++++++++++++++++++ .../internal/dynamo/lpx/runtime_agent.go | 8 ++- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index 32875f80f6d3..7118a57e8377 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -6,6 +6,7 @@ package lpx import ( + "fmt" "slices" corev1 "k8s.io/api/core/v1" @@ -18,14 +19,27 @@ const ( defaultLPUAgentCPUs int64 = 62 ) -func applyLPUHostDeviceVolumes(podSpec *corev1.PodSpec, replaceExisting bool) { +func applyLPUHostDeviceVolumes(podSpec *corev1.PodSpec, replaceExisting bool) error { apply := appendVolumeIfMissing if replaceExisting { apply = setVolumeByName } - podSpec.Volumes = apply(podSpec.Volumes, lpuHostDeviceVolume("host-dev", "/dev")) - podSpec.Volumes = apply(podSpec.Volumes, lpuHostDeviceVolume("host-sys", "/sys")) + // Keep authored storage only when it exposes the required host device directory. + for _, required := range []corev1.Volume{lpuHostDeviceVolume("host-dev", "/dev"), lpuHostDeviceVolume("host-sys", "/sys")} { + if !replaceExisting { + for _, existing := range podSpec.Volumes { + hostPath := existing.HostPath + if existing.Name == required.Name && (hostPath == nil || hostPath.Path != required.HostPath.Path || + hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && + *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate) { + return fmt.Errorf("selected LPX podTemplate volume %q must use directory hostPath %q", required.Name, required.HostPath.Path) + } + } + } + podSpec.Volumes = apply(podSpec.Volumes, required) + } + return nil } func lpuHostDeviceVolume(name, path string) corev1.Volume { diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 8eb9bc771ea5..f9b674d1f707 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -396,6 +396,65 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { } } +func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { + for _, test := range []struct { + name string + volume corev1.Volume + wantErr bool + }{ + {"ConfigMap", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "application"}}, + }}, true}, + {"Secret", corev1.Volume{Name: "host-sys", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "application"}, + }}, true}, + {"wrong host directory", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/application"}, + }}, true}, + {"wrong host type", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathFile)}, + }}, true}, + {"compatible host directory", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}, + }}, false}, + {"compatible host type", corev1.Volume{Name: "host-sys", VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: ptr.To(corev1.HostPathDirectoryOrCreate)}, + }}, false}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Give an authored sidecar a volume name reserved by the Agent runtime") + base := corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: commonconsts.MainContainerName, Image: "lpu-runtime"}, + {Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: test.volume.Name, MountPath: "/application"}}}, + }, + Volumes: []corev1.Volume{test.volume}, + } + configureAgentScheduling(&base, BuildFamilyXT) + direct, agent := base.DeepCopy(), base.DeepCopy() + + t.Log("Reject incompatible bindings through both XT runtime entry points") + errors := []error{ + configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret"), + configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), + } + for _, err := range errors { + if test.wantErr { + require.ErrorContains(t, err, "must use directory hostPath") + } else { + require.NoError(t, err) + } + } + + t.Log("Never rebind the authored volume or its sidecar mount") + for _, pod := range []*corev1.PodSpec{direct, agent} { + require.Equal(t, test.volume.VolumeSource, testVolumeByName(t, pod.Volumes, test.volume.Name)) + require.Equal(t, base.Containers[1], pod.Containers[1]) + } + }) + } +} + func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { t.Log("Define custom entrypoint, probe and Pod security preservation cases") customStartup := testExecProbe("custom-startup") diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 8a73a472aacc..9308250ea2f7 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -176,7 +176,9 @@ func configureDirectHybridAgentRuntime( if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { return err } - applyLPUHostDeviceVolumes(agentPodSpec, false) + if err := applyLPUHostDeviceVolumes(agentPodSpec, false); err != nil { + return err + } agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ Name: "hugepages", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ @@ -250,7 +252,9 @@ func configureNodeLocalAgentRuntime( addRuntimeTemporaryStorage(agentPodSpec, agent, !isXT) agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) - applyLPUHostDeviceVolumes(agentPodSpec, !isXT) + if err := applyLPUHostDeviceVolumes(agentPodSpec, !isXT); err != nil { + return err + } if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { return err } From f43dc3e80c226dff38e8adb41983435e876cbd71 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 06:54:38 +0000 Subject: [PATCH 051/196] fix(operator): reject read-only LPX Agent temporary storage Signed-off-by: Andrew Paprotsky (cherry picked from commit 113925f7db3a079fe0bc1c5a7bcad9ca92f0bf2b) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/pod_roles_test.go | 73 +++++++++++++++++++ .../internal/dynamo/lpx/runtime_agent.go | 50 +++++++++++-- .../internal/dynamo/lpx/runtime_startup.go | 3 +- 3 files changed, 116 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index f9b674d1f707..81c5287a4017 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -455,6 +455,79 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { } } +func TestXTRuntimesValidateTemporaryStorage(t *testing.T) { + for _, test := range []struct { + name string + source corev1.VolumeSource + readOnly bool + wantErr bool + }{ + {"emptyDir", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, false, false}, + {"read-only mount", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, true, true}, + {"hostPath", corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/scratch"}}, false, false}, + {"PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "scratch"}}, false, false}, + {"CSI", corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{Driver: "storage.example.com"}}, false, false}, + {"AzureDisk", corev1.VolumeSource{AzureDisk: &corev1.AzureDiskVolumeSource{DiskName: "scratch", DataDiskURI: "disk-uri"}}, false, false}, + {"ConfigMap", corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}, false, true}, + {"Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{}}, false, true}, + {"DownwardAPI", corev1.VolumeSource{DownwardAPI: &corev1.DownwardAPIVolumeSource{}}, false, true}, + {"Projected", corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{}}, false, true}, + {"Image", corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "runtime:latest"}}, false, true}, + {"read-only PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ReadOnly: true}}, false, true}, + {"read-only NFS", corev1.VolumeSource{NFS: &corev1.NFSVolumeSource{ReadOnly: true}}, false, true}, + {"read-only CSI", corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{ReadOnly: ptr.To(true)}}, false, true}, + {"read-only ISCSI", corev1.VolumeSource{ISCSI: &corev1.ISCSIVolumeSource{ReadOnly: true}}, false, true}, + {"read-only FC", corev1.VolumeSource{FC: &corev1.FCVolumeSource{ReadOnly: true}}, false, true}, + {"read-only FlexVolume", corev1.VolumeSource{FlexVolume: &corev1.FlexVolumeSource{ReadOnly: true}}, false, true}, + {"read-only GCE", corev1.VolumeSource{GCEPersistentDisk: &corev1.GCEPersistentDiskVolumeSource{ReadOnly: true}}, false, true}, + {"read-only AWS", corev1.VolumeSource{AWSElasticBlockStore: &corev1.AWSElasticBlockStoreVolumeSource{ReadOnly: true}}, false, true}, + {"read-only Cinder", corev1.VolumeSource{Cinder: &corev1.CinderVolumeSource{ReadOnly: true}}, false, true}, + {"read-only AzureFile", corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{ReadOnly: true}}, false, true}, + {"read-only AzureDisk", corev1.VolumeSource{AzureDisk: &corev1.AzureDiskVolumeSource{ReadOnly: ptr.To(true)}}, false, true}, + {"read-only Portworx", corev1.VolumeSource{PortworxVolume: &corev1.PortworxVolumeSource{ReadOnly: true}}, false, true}, + } { + for _, name := range []string{"tmp", "scratch"} { + t.Run(test.name+"/"+name, func(t *testing.T) { + t.Log("Use either a named tmp volume or an existing /tmp mount") + base := corev1.PodSpec{ + Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, + Volumes: []corev1.Volume{{Name: name, VolumeSource: test.source}}, + } + if name != "tmp" || test.readOnly { + base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: name, MountPath: "/tmp", ReadOnly: test.readOnly}} + } + if name != "tmp" { + base.Volumes = append(base.Volumes, corev1.Volume{Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "unused"}}}) + } + configureAgentScheduling(&base, BuildFamilyXT) + direct, agent := base.DeepCopy(), base.DeepCopy() + + t.Log("Require writable temporary storage through both XT runtime entry points") + for _, err := range []error{ + configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret"), + configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), + } { + if test.wantErr { + require.ErrorContains(t, err, "requires writable storage") + } else { + require.NoError(t, err) + } + } + + t.Log("Preserve authored storage and the selected mount") + for _, pod := range []*corev1.PodSpec{direct, agent} { + for _, volume := range base.Volumes { + require.Equal(t, volume.VolumeSource, testVolumeByName(t, pod.Volumes, volume.Name)) + } + if !test.wantErr { + require.Contains(t, pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: name, MountPath: "/tmp"}) + } + } + }) + } + } +} + func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { t.Log("Define custom entrypoint, probe and Pod security preservation cases") customStartup := testExecProbe("custom-startup") diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 9308250ea2f7..a5f5fc233dd4 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -162,7 +162,9 @@ func configureDirectHybridAgentRuntime( }, ) setNodeLocalPodIPEnv(agent, true) - addRuntimeTemporaryStorage(agentPodSpec, agent, false) + if err := addRuntimeTemporaryStorage(agentPodSpec, agent, false); err != nil { + return err + } updateWorkerPodSpec(agentPodSpec) // Keep volume permissions and other Pod settings while forcing the Agent's root identity. @@ -249,7 +251,9 @@ func configureNodeLocalAgentRuntime( agent.Name = lpuAgentContainerName retargetMainContainerReferences(agentPodSpec, agent) configureNodeLocalAgentWorkerContainer(agent, isXT, preserveAgentEntrypoint) - addRuntimeTemporaryStorage(agentPodSpec, agent, !isXT) + if err := addRuntimeTemporaryStorage(agentPodSpec, agent, !isXT); err != nil { + return err + } agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) if err := applyLPUHostDeviceVolumes(agentPodSpec, !isXT); err != nil { @@ -435,7 +439,7 @@ func retargetResourceFieldReferencesInDownwardAPI(items []corev1.DownwardAPIVolu } } -func addRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container, replaceExisting bool) { +func addRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container, replaceExisting bool) error { // Give conductor expansion and Agent setup a pod-lifetime writable workspace. volume := corev1.Volume{ Name: runtimeTemporaryStorageVolumeName, @@ -447,12 +451,42 @@ func addRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Conta if replaceExisting { podSpec.Volumes = setVolumeByName(podSpec.Volumes, volume) container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) - return + return nil } - if !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.MountPath == runtimeTemporaryStorageMountPath - }) { - container.VolumeMounts = append(container.VolumeMounts, mount) + + // Validate the selected binding without rebinding authored storage used by other containers. + for _, existing := range container.VolumeMounts { + if existing.MountPath == mount.MountPath { + mount = existing + } } + readOnly := mount.ReadOnly + for _, existing := range podSpec.Volumes { + if existing.Name == mount.Name { + readOnly = readOnly || volumeIsReadOnly(existing.VolumeSource) + } + } + if readOnly { + return fmt.Errorf("LPX runtime requires writable storage at %q", mount.MountPath) + } + container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) podSpec.Volumes = appendVolumeIfMissing(podSpec.Volumes, volume) + return nil +} + +func volumeIsReadOnly(source corev1.VolumeSource) bool { + // These sources force a read-only mount even when VolumeMount.ReadOnly is false. + return source.ConfigMap != nil || source.Secret != nil || source.DownwardAPI != nil || source.Projected != nil || source.Image != nil || + source.PersistentVolumeClaim != nil && source.PersistentVolumeClaim.ReadOnly || + source.NFS != nil && source.NFS.ReadOnly || + source.CSI != nil && ptr.Deref(source.CSI.ReadOnly, false) || + source.ISCSI != nil && source.ISCSI.ReadOnly || + source.FC != nil && source.FC.ReadOnly || + source.FlexVolume != nil && source.FlexVolume.ReadOnly || + source.GCEPersistentDisk != nil && source.GCEPersistentDisk.ReadOnly || + source.AWSElasticBlockStore != nil && source.AWSElasticBlockStore.ReadOnly || + source.Cinder != nil && source.Cinder.ReadOnly || + source.AzureFile != nil && source.AzureFile.ReadOnly || + source.AzureDisk != nil && ptr.Deref(source.AzureDisk.ReadOnly, false) || + source.PortworxVolume != nil && source.PortworxVolume.ReadOnly } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup.go b/deploy/operator/internal/dynamo/lpx/runtime_startup.go index ee6b7d88c86c..adca8f2a9aeb 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup.go @@ -61,6 +61,5 @@ func addRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Containe if mount.Name != runtimeTemporaryStorageVolumeName { return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) } - addRuntimeTemporaryStorage(podSpec, container, false) - return nil + return addRuntimeTemporaryStorage(podSpec, container, false) } From eab93bb1868da8667fa303b76f5dc58b103a73b4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 08:18:50 +0000 Subject: [PATCH 052/196] refactor(operator): remove strategic PodTemplate merging Signed-off-by: Andrew Paprotsky (cherry picked from commit 64211f0d3676cb43775c9bb93880b281a1324fa1) Signed-off-by: Andrew Paprotsky --- .../operator/templates/_validation.tpl | 10 - .../operator/templates/deployment.yaml | 1 - .../operator/templates/operator-config.yaml | 5 - .../platform/components/operator/values.yaml | 1 - deploy/operator/api/config/v1alpha1/types.go | 16 +- .../config/v1alpha1/zz_generated.deepcopy.go | 16 - .../api/config/validation/validation.go | 7 - .../api/config/validation/validation_test.go | 35 -- .../conversion_field_coverage_test.go | 2 - .../dynamocomponentdeployment_types.go | 34 -- .../dynamographdeployment_types_test.go | 48 --- .../api/v1alpha1/shared_spec_conversion.go | 5 - .../dynamocomponentdeployment_types.go | 26 +- ...nvidia.com_dynamocomponentdeployments.yaml | 18 - .../nvidia.com_dynamographdeployments.yaml | 25 +- .../dgd_lpx_handoff_envtest_test.go | 19 +- .../controller/lpx/component_roles_test.go | 1 + .../internal/controller/lpx/lifecycle_test.go | 34 +- .../internal/controller/lpx/render_test.go | 319 +++++++++--------- deploy/operator/internal/dynamo/graph.go | 35 +- deploy/operator/internal/dynamo/graph_lpx.go | 33 +- .../internal/dynamo/graph_lpx_test.go | 5 +- deploy/operator/internal/dynamo/graph_test.go | 34 +- .../internal/dynamo/lpx/conductor_runtime.go | 1 - .../internal/dynamo/lpx/cyborg_render.go | 13 +- .../dynamo/lpx/hybrid_cyborg_bindings.go | 30 -- .../internal/dynamo/lpx/lpu_configmap.go | 39 +-- .../operator/internal/dynamo/lpx/lpu_pod.go | 60 +--- .../internal/dynamo/lpx/pod_mutation.go | 49 --- .../internal/dynamo/lpx/pod_roles_test.go | 205 ++++++++++- deploy/operator/internal/dynamo/lpx/render.go | 15 + .../internal/dynamo/lpx/render_test.go | 68 +++- .../internal/dynamo/lpx/runtime_agent.go | 124 +++---- .../internal/dynamo/lpx/runtime_ssh.go | 71 ++-- .../internal/dynamo/lpx/runtime_ssh_test.go | 42 ++- .../internal/dynamo/lpx/runtime_startup.go | 26 +- .../dynamo/lpx/runtime_startup_test.go | 28 +- deploy/operator/internal/dynamo/patch.go | 129 ------- deploy/operator/internal/dynamo/patch_test.go | 294 ---------------- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 22 +- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 62 ++-- .../node-local-v2-hybrid.input.yaml | 17 + .../from_dgd_yaml/node-local-v2-hybrid.yaml | 58 ++-- .../node-local-v2-lpu-only.input.yaml | 28 ++ .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 44 +-- .../node-local-v2-specdecode.input.yaml | 33 ++ .../node-local-v2-specdecode.yaml | 96 +++--- .../node-local-v3-hx-hybrid.input.yaml | 14 + .../node-local-v3-hx-hybrid.yaml | 48 ++- .../node-local-v3-hx-lpu-only.input.yaml | 25 ++ .../node-local-v3-hx-lpu-only.yaml | 44 +-- .../node-local-v3-hx-specdecode.input.yaml | 32 ++ .../node-local-v3-hx-specdecode.yaml | 96 +++--- .../from_dgd_yaml/single_v2.input.yaml | 34 +- .../testdata/from_dgd_yaml/single_v2.yaml | 44 +-- .../single_v2_manifest_defaults.input.yaml | 26 +- .../model-deployment/deploy-with-dgd.md | 57 +++- .../additional-resources/api-reference-k8s.md | 67 +--- .../kubernetes-api/full-api-reference.mdx | 84 +---- 59 files changed, 1233 insertions(+), 1621 deletions(-) delete mode 100644 deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go delete mode 100644 deploy/operator/internal/dynamo/lpx/pod_mutation.go delete mode 100644 deploy/operator/internal/dynamo/patch.go delete mode 100644 deploy/operator/internal/dynamo/patch_test.go diff --git a/deploy/helm/charts/platform/components/operator/templates/_validation.tpl b/deploy/helm/charts/platform/components/operator/templates/_validation.tpl index a76020aa934f..829733970a5e 100644 --- a/deploy/helm/charts/platform/components/operator/templates/_validation.tpl +++ b/deploy/helm/charts/platform/components/operator/templates/_validation.tpl @@ -117,13 +117,3 @@ Validation for discoveryBackend configuration {{- fail (printf "VALIDATION ERROR: discoveryBackend must be 'kubernetes' (default) or 'etcd'. Got: '%s'" $discoveryBackend) -}} {{- end -}} {{- end -}} - -{{/* -Validation for defaultExtraPodSpecMergeStrategy configuration -*/}} -{{- define "dynamo-operator.validateExtraPodSpecMergeStrategy" -}} -{{- $strategy := default "override" .Values.controllerManager.manager.defaultExtraPodSpecMergeStrategy -}} -{{- if and (ne $strategy "override") (ne $strategy "strategic") -}} - {{- fail (printf "VALIDATION ERROR: controllerManager.manager.defaultExtraPodSpecMergeStrategy must be 'override' or 'strategic'. Got: '%s'" $strategy) -}} -{{- end -}} -{{- end -}} diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 79d53e438f73..51fa293dba59 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -17,7 +17,6 @@ {{- include "dynamo-operator.validateClusterWideInstallation" . -}} {{- include "dynamo-operator.validateConfiguration" . -}} {{- include "dynamo-operator.validateDiscoveryBackend" . -}} -{{- include "dynamo-operator.validateExtraPodSpecMergeStrategy" . -}} --- apiVersion: apps/v1 diff --git a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml index 1139fa4069d4..fd8470b34862 100644 --- a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml @@ -205,11 +205,6 @@ data: {{- end }} {{- end }} {{- end }} - {{- $defaultExtraPodSpecMergeStrategy := default "override" .Values.controllerManager.manager.defaultExtraPodSpecMergeStrategy }} - {{- if ne ($defaultExtraPodSpecMergeStrategy | toString) "override" }} - podGeneration: - defaultExtraPodSpecMergeStrategy: {{ $defaultExtraPodSpecMergeStrategy | quote }} - {{- end }} {{- if and .Values.discoveryBackend (ne (.Values.discoveryBackend | toString) "kubernetes") }} discovery: backend: {{ .Values.discoveryBackend | quote }} diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 79f6404b551a..498d7c411eb4 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -80,7 +80,6 @@ controllerManager: namespace: "" manager: - defaultExtraPodSpecMergeStrategy: override containerSecurityContext: allowPrivilegeEscalation: false capabilities: diff --git a/deploy/operator/api/config/v1alpha1/types.go b/deploy/operator/api/config/v1alpha1/types.go index 5f295000edb9..01d310e3016d 100644 --- a/deploy/operator/api/config/v1alpha1/types.go +++ b/deploy/operator/api/config/v1alpha1/types.go @@ -17,10 +17,7 @@ package v1alpha1 -import ( - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // +kubebuilder:object:root=true @@ -62,9 +59,6 @@ type OperatorConfiguration struct { // Checkpoint/restore configuration Checkpoint CheckpointConfiguration `json:"checkpoint"` - // Pod generation defaults - PodGeneration PodGenerationConfiguration `json:"podGeneration"` - // Discovery backend configuration Discovery DiscoveryConfiguration `json:"discovery"` @@ -396,14 +390,6 @@ func (c *CheckpointConfiguration) EffectiveSeccompProfile() string { return c.Seccomp.Profile } -// PodGenerationConfiguration holds pod generation defaults applied by the operator. -type PodGenerationConfiguration struct { - // DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a - // component leaves spec.extraPodSpecMergeStrategy unset. - // +kubebuilder:default="override" - DefaultExtraPodSpecMergeStrategy nvidiacomv1alpha1.ExtraPodSpecMergeStrategy `json:"defaultExtraPodSpecMergeStrategy,omitempty"` -} - // DiscoveryConfiguration holds discovery backend settings. type DiscoveryConfiguration struct { // Backend is the discovery backend: "kubernetes" or "etcd" diff --git a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go index 42a171870f91..952d37192b26 100644 --- a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go @@ -371,7 +371,6 @@ func (in *OperatorConfiguration) DeepCopyInto(out *OperatorConfiguration) { out.RBAC = in.RBAC out.MPI = in.MPI in.Checkpoint.DeepCopyInto(&out.Checkpoint) - out.PodGeneration = in.PodGeneration out.Discovery = in.Discovery in.GPU.DeepCopyInto(&out.GPU) out.Logging = in.Logging @@ -416,21 +415,6 @@ func (in *OrchestratorConfiguration) DeepCopy() *OrchestratorConfiguration { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodGenerationConfiguration) DeepCopyInto(out *PodGenerationConfiguration) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGenerationConfiguration. -func (in *PodGenerationConfiguration) DeepCopy() *PodGenerationConfiguration { - if in == nil { - return nil - } - out := new(PodGenerationConfiguration) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RBACConfiguration) DeepCopyInto(out *RBACConfiguration) { *out = *in diff --git a/deploy/operator/api/config/validation/validation.go b/deploy/operator/api/config/validation/validation.go index 02cd25dd8d47..67020970d175 100644 --- a/deploy/operator/api/config/validation/validation.go +++ b/deploy/operator/api/config/validation/validation.go @@ -40,13 +40,6 @@ func ValidateOperatorConfiguration(config *configv1alpha1.OperatorConfiguration) allErrs = append(allErrs, validateDiscovery(&config.Discovery, field.NewPath("discovery"))...) allErrs = append(allErrs, validateRBAC(config)...) allErrs = append(allErrs, validateOrchestrators(&config.Orchestrators, field.NewPath("orchestrators"))...) - if strategy := config.PodGeneration.DefaultExtraPodSpecMergeStrategy; strategy != "" && !strategy.IsValid() { - allErrs = append(allErrs, field.NotSupported( - field.NewPath("podGeneration").Child("defaultExtraPodSpecMergeStrategy"), - strategy, - []string{"override", "strategic"}, - )) - } allErrs = append(allErrs, validateIngress(&config.Ingress, field.NewPath("ingress"))...) allErrs = append(allErrs, validateServiceMesh(&config.ServiceMesh, field.NewPath("serviceMesh"))...) diff --git a/deploy/operator/api/config/validation/validation_test.go b/deploy/operator/api/config/validation/validation_test.go index 947deb8a7edd..e713c8a68dc3 100644 --- a/deploy/operator/api/config/validation/validation_test.go +++ b/deploy/operator/api/config/validation/validation_test.go @@ -362,38 +362,3 @@ func TestValidateInfrastructure_TLS(t *testing.T) { } }) } - -func TestValidateOperatorConfiguration_InvalidDefaultExtraPodSpecMergeStrategy(t *testing.T) { - cfg := validConfig() - cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy = "invalid" - - errs := ValidateOperatorConfiguration(cfg) - if len(errs) != 1 { - t.Errorf("expected 1 error for invalid extraPodSpec merge strategy, got %d: %v", len(errs), errs) - } -} - -func TestValidateOperatorConfiguration_PodGenerationCamelCaseConfigKey(t *testing.T) { - cfg := validConfig() - rawConfig := []byte(`{ - "podGeneration": { - "defaultExtraPodSpecMergeStrategy": "strategic" - } - }`) - - if err := json.Unmarshal(rawConfig, cfg); err != nil { - t.Fatalf("failed to unmarshal podGeneration config: %v", err) - } - - if cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy != "strategic" { - t.Fatalf( - "expected strategic merge strategy from podGeneration key, got %q", - cfg.PodGeneration.DefaultExtraPodSpecMergeStrategy, - ) - } - - errs := ValidateOperatorConfiguration(cfg) - if len(errs) != 0 { - t.Errorf("expected no errors for camelCase podGeneration config, got: %v", errs) - } -} diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index c6eda7f7503c..50da73799674 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -59,7 +59,6 @@ DynamoComponentDeploymentSpec.experimental.gpuMemoryService.extraClientPods.name DynamoComponentDeploymentSpec.experimental.gpuMemoryService.extraClientPods.podTemplate DynamoComponentDeploymentSpec.experimental.gpuMemoryService.mode DynamoComponentDeploymentSpec.experimental.grove.forceScalingGroup -DynamoComponentDeploymentSpec.extraPodSpecMergeStrategy DynamoComponentDeploymentSpec.frontendSidecar DynamoComponentDeploymentSpec.globalDynamoNamespace DynamoComponentDeploymentSpec.lpx.buildId @@ -134,7 +133,6 @@ DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.extraClientPo DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.extraClientPods.podTemplate DynamoGraphDeploymentSpec.components.experimental.gpuMemoryService.mode DynamoGraphDeploymentSpec.components.experimental.grove.forceScalingGroup -DynamoGraphDeploymentSpec.components.extraPodSpecMergeStrategy DynamoGraphDeploymentSpec.components.frontendSidecar DynamoGraphDeploymentSpec.components.globalDynamoNamespace DynamoGraphDeploymentSpec.components.lpx.buildId diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 4404ac6dcdcd..04c3c9fc46eb 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -35,36 +35,6 @@ const ( DynamoGraphDeploymentConditionTypeDynamoComponentReady = "DynamoComponentReady" ) -// +kubebuilder:validation:Enum=override;strategic -type ExtraPodSpecMergeStrategy string - -const ( - ExtraPodSpecMergeStrategyOverride ExtraPodSpecMergeStrategy = "override" - ExtraPodSpecMergeStrategyStrategic ExtraPodSpecMergeStrategy = "strategic" - DefaultExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy = ExtraPodSpecMergeStrategyOverride -) - -func (s ExtraPodSpecMergeStrategy) IsValid() bool { - return s == ExtraPodSpecMergeStrategyOverride || s == ExtraPodSpecMergeStrategyStrategic -} - -func ResolveExtraPodSpecMergeStrategy( - explicitStrategy ExtraPodSpecMergeStrategy, - defaultStrategy ExtraPodSpecMergeStrategy, -) (ExtraPodSpecMergeStrategy, error) { - resolved := explicitStrategy - if resolved == "" { - resolved = defaultStrategy - } - if resolved == "" { - resolved = DefaultExtraPodSpecMergeStrategy - } - if !resolved.IsValid() { - return "", fmt.Errorf("invalid extraPodSpec merge strategy %q", resolved) - } - return resolved, nil -} - // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. @@ -170,10 +140,6 @@ type DynamoComponentDeploymentSharedSpec struct { // extraPodSpec and provide a non-empty mainContainer image. Existing components created // without extraPodSpec may remain unchanged. ExtraPodSpec *ExtraPodSpec `json:"extraPodSpec,omitempty"` - // ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the - // operator-generated pod and container defaults. - // +optional - ExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy `json:"extraPodSpecMergeStrategy,omitempty"` // LivenessProbe to detect and restart unhealthy containers. LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go deleted file mode 100644 index a207164175ea..000000000000 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types_test.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package v1alpha1 - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestExtraPodSpecMergeStrategy_RoundTripAndValidation(t *testing.T) { - dgd := DynamoGraphDeployment{ - Spec: DynamoGraphDeploymentSpec{ - Services: map[string]*DynamoComponentDeploymentSharedSpec{ - "frontend": { - ComponentType: "frontend", - ExtraPodSpecMergeStrategy: ExtraPodSpecMergeStrategyStrategic, - }, - }, - }, - } - - data, err := json.Marshal(dgd) - require.NoError(t, err) - - var restored DynamoGraphDeployment - require.NoError(t, json.Unmarshal(data, &restored)) - - assert.Equal( - t, - ExtraPodSpecMergeStrategyStrategic, - restored.Spec.Services["frontend"].ExtraPodSpecMergeStrategy, - ) - assert.True(t, ExtraPodSpecMergeStrategyOverride.IsValid()) - assert.True(t, ExtraPodSpecMergeStrategyStrategic.IsValid()) - assert.False(t, ExtraPodSpecMergeStrategy("invalid").IsValid()) -} - -func TestResolveExtraPodSpecMergeStrategy_DefaultsToOverride(t *testing.T) { - got, err := ResolveExtraPodSpecMergeStrategy("", "") - require.NoError(t, err) - assert.Equal(t, ExtraPodSpecMergeStrategyOverride, got) -} diff --git a/deploy/operator/api/v1alpha1/shared_spec_conversion.go b/deploy/operator/api/v1alpha1/shared_spec_conversion.go index 8208eb2c408d..184c44bf4fd3 100644 --- a/deploy/operator/api/v1alpha1/shared_spec_conversion.go +++ b/deploy/operator/api/v1alpha1/shared_spec_conversion.go @@ -160,7 +160,6 @@ func ConvertFromDynamoComponentDeploymentSharedSpec(src *DynamoComponentDeployme dst.GlobalDynamoNamespace = src.GlobalDynamoNamespace dst.Replicas = src.Replicas dst.MinAvailable = src.MinAvailable - dst.ExtraPodSpecMergeStrategy = v1beta1.ExtraPodSpecMergeStrategy(src.ExtraPodSpecMergeStrategy) if src.Multinode != nil { dst.Multinode = &v1beta1.MultinodeSpec{} @@ -250,9 +249,6 @@ func restoreSharedAlphaOnlySimpleFields(dst *DynamoComponentDeploymentSharedSpec cp := *preserved.Autoscaling dst.Autoscaling = &cp } - if dst.ExtraPodSpecMergeStrategy == "" && preserved.ExtraPodSpecMergeStrategy != "" { - dst.ExtraPodSpecMergeStrategy = preserved.ExtraPodSpecMergeStrategy - } if dst.Ingress == nil && preserved.Ingress != nil { cp := *preserved.Ingress dst.Ingress = &cp @@ -573,7 +569,6 @@ func ConvertToDynamoComponentDeploymentSharedSpec(src *v1beta1.DynamoComponentDe dst.LPX = src.LPX dst.RuntimeVersionOverride = src.RuntimeVersionOverride dst.ServiceName = src.ComponentName - dst.ExtraPodSpecMergeStrategy = ExtraPodSpecMergeStrategy(src.ExtraPodSpecMergeStrategy) // sharedMemorySize -> SharedMemorySpec. if src.SharedMemorySize != nil { diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 6ea40f210459..b66520edf29c 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -29,18 +29,6 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) -// +kubebuilder:validation:Enum=override;strategic -type ExtraPodSpecMergeStrategy string - -const ( - ExtraPodSpecMergeStrategyOverride ExtraPodSpecMergeStrategy = "override" - ExtraPodSpecMergeStrategyStrategic ExtraPodSpecMergeStrategy = "strategic" -) - -func (s ExtraPodSpecMergeStrategy) IsValid() bool { - return s == ExtraPodSpecMergeStrategyOverride || s == ExtraPodSpecMergeStrategyStrategic -} - const ( // DynamoComponentDeploymentConditionTypeAvailable indicates the component is // available and serving traffic. @@ -79,10 +67,9 @@ type DynamoComponentDeploymentSpec struct { // volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are // replaced with a single `podTemplate` field holding a native // `corev1.PodTemplateSpec`. The operator injects its defaults into the -// container named `"main"` and merges user overrides according to -// `extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and -// pod-level configuration directly in `podTemplate` without any -// `extraPodSpec`-style escape hatch. +// container named `"main"` and merges user overrides. Users can add sidecars, +// init containers, and pod-level configuration directly in `podTemplate` +// without any `extraPodSpec`-style escape hatch. // +kubebuilder:validation:XValidation:rule="!has(self.eppConfig) || (has(self.type) && self.type == 'epp')",message="eppConfig may only be set when type is epp" // +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" @@ -155,13 +142,6 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` - // extraPodSpecMergeStrategy controls how podTemplate is merged with the - // operator-generated pod and container defaults. When omitted, the operator - // uses its configured defaultExtraPodSpecMergeStrategy, which defaults to - // "override". - // +optional - ExtraPodSpecMergeStrategy ExtraPodSpecMergeStrategy `json:"extraPodSpecMergeStrategy,omitempty"` - // replicas is the desired number of Pods for this component. When // `scalingAdapter` is set on this component, this field is managed by // the DynamoGraphDeploymentScalingAdapter and should not be modified diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index a01677505cb0..959abf0c6d87 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -10703,14 +10703,6 @@ spec: x-kubernetes-list-type: map required: [] type: object - extraPodSpecMergeStrategy: - description: |- - ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the - operator-generated pod and container defaults. - enum: - - override - - strategic - type: string failover: description: |- Failover configures GMS (GPU Memory Service) failover for this service. @@ -20431,16 +20423,6 @@ spec: type: boolean type: object type: object - extraPodSpecMergeStrategy: - description: |- - extraPodSpecMergeStrategy controls how podTemplate is merged with the - operator-generated pod and container defaults. When omitted, the operator - uses its configured defaultExtraPodSpecMergeStrategy, which defaults to - "override". - enum: - - override - - strategic - type: string frontendSidecar: description: |- frontendSidecar optionally designates a container in diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 3fac56fec57a..01f87244d468 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11044,14 +11044,6 @@ spec: x-kubernetes-list-type: map required: [] type: object - extraPodSpecMergeStrategy: - description: |- - ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the - operator-generated pod and container defaults. - enum: - - override - - strategic - type: string failover: description: |- Failover configures GMS (GPU Memory Service) failover for this service. @@ -20679,10 +20671,9 @@ spec: volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the - container named `"main"` and merges user overrides according to - `extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and - pod-level configuration directly in `podTemplate` without any - `extraPodSpec`-style escape hatch. + container named `"main"` and merges user overrides. Users can add sidecars, + init containers, and pod-level configuration directly in `podTemplate` + without any `extraPodSpec`-style escape hatch. properties: compilationCache: description: |- @@ -21148,16 +21139,6 @@ spec: type: boolean type: object type: object - extraPodSpecMergeStrategy: - description: |- - extraPodSpecMergeStrategy controls how podTemplate is merged with the - operator-generated pod and container defaults. When omitted, the operator - uses its configured defaultExtraPodSpecMergeStrategy, which defaults to - "override". - enum: - - override - - strategic - type: string frontendSidecar: description: |- frontendSidecar optionally designates a container in diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index a8ca9975eb60..b6eef40688cd 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -170,9 +170,24 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "example/lpu-runtime:1.4.0", - VolumeMounts: []corev1.VolumeMount{{Name: consts.ModelStorageVolumeName, MountPath: "/models"}}, + VolumeMounts: []corev1.VolumeMount{ + {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, + {Name: "config", MountPath: "/configs"}, + {Name: "host-dev", MountPath: "/dev"}, + {Name: "host-sys", MountPath: "/sys"}, + {Name: "hugepages", MountPath: "/dev/hugepages"}, + {Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}, + {Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}, + }, }}, - Volumes: []corev1.Volume{{Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + Volumes: []corev1.Volume{ + {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, + {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, + {Name: "hugepages", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumHugePages}}}, + {Name: "ssh-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, + {Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, }}, }}, }}}, diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go index b171c5d33bab..5c0aaf578277 100644 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -21,6 +21,7 @@ import ( func TestLPXRoleDependenciesIncludeDraftAndConductorFallback(t *testing.T) { t.Log("Give both components consumed Agent claims while the target conductor uses its Agent fallback") source := newLPXSpecDecodeTestSource() + lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil for _, component := range lpx.Components(source) { pod := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(component.ComponentName + "-gpu")}} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 9346622a95fb..acfc674f15de 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -2176,16 +2176,24 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "lpu-runtime", - VolumeMounts: []corev1.VolumeMount{{ - Name: consts.ModelStorageVolumeName, MountPath: "/models", - }}, - }}, - Volumes: []corev1.Volume{{ - Name: consts.ModelStorageVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + VolumeMounts: []corev1.VolumeMount{ + {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, + {Name: "config", MountPath: "/configs"}, + {Name: "host-dev", MountPath: "/dev"}, + {Name: "host-sys", MountPath: "/sys"}, + {Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}, + {Name: "hugepages", MountPath: "/dev/hugepages"}, + {Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}, }, }}, + Volumes: []corev1.Volume{ + {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathDirectory)}}}, + {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: ptr.To(corev1.HostPathDirectory)}}}, + {Name: "hugepages", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumHugePages}}}, + {Name: "ssh-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret", DefaultMode: ptr.To[int32](0644)}}}, + {Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, }}, }}, }}, @@ -2197,7 +2205,15 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D Replicas: &one, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ ResourceClaims: []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("gpu")}}, - Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "cyborg-runtime"}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "cyborg-runtime", + VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/configs"}, + {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, }}, } } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 60034ab3f018..7d7b2dba81f9 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -174,11 +174,13 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { {"node-local-v2-specdecode", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, {"node-local-v3-hx-lpu-only", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, {"node-local-v3-hx-hybrid", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, + {"node-local-v3-hx-hybrid-inherited", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, {"node-local-v3-hx-specdecode", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, } { t.Run(test.name, func(t *testing.T) { t.Log("Resolve the authored engine and its exact child without changing either input") - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + test.name + ".input.yaml") + fixture := strings.TrimSuffix(test.name, "-inherited") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + fixture + ".input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -213,7 +215,13 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { component := lpx.ServingComponent(source) component.ModelRef = &v1beta1.ModelReference{Name: "test/model"} component.MinAvailable = ptr.To(int32(1)) - hybrid := strings.HasSuffix(test.name, "-hybrid") + hybrid := strings.HasSuffix(fixture, "-hybrid") + inherited := fixture != test.name + if inherited { + t.Log("Let the HX GPU conductor inherit the Agent template and its authored config mount") + component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil + component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = resource.MustParse("1") + } singleXT := test.name == "node-local-v2-lpu-only" if hybrid { component.Replicas = ptr.To(int32(3)) @@ -223,8 +231,9 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { if singleXT { t.Log("Keep XT's authored readonly mount at the canonical config path") main := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0] - main.VolumeMounts = append(main.VolumeMounts, - corev1.VolumeMount{Name: "config", MountPath: "/configs", ReadOnly: true}) + configMount := slices.IndexFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" }) + require.GreaterOrEqual(t, configMount, 0) + main.VolumeMounts[configMount].ReadOnly = true } t.Log("Preserve role metadata and replica units through alpha/beta conversion before freezing the child") @@ -389,6 +398,15 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, "from-component", clique.Labels["shared-label"]) require.Equal(t, "false", clique.Annotations[commonconsts.KubeAnnotationEnableMetrics]) require.Equal(t, "kubernetes", clique.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend]) + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != lpxv1alpha1.PodRoleCyborgWorker || test.wantFamily == lpxv1alpha1.TargetFamilyXt8888 || inherited { + t.Log("Bind the generated immutable ConfigMap alongside authored volumes and mounts") + configIndex := slices.IndexFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "config" }) + require.GreaterOrEqual(t, configIndex, 0) + configVolume := clique.Spec.PodSpec.Volumes[configIndex] + require.NotNil(t, configVolume.ConfigMap) + generatedConfig := getResource[*corev1.ConfigMap](t, firstResources, configVolume.ConfigMap.Name) + require.Equal(t, ptr.To(true), generatedConfig.Immutable) + } switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { case lpxv1alpha1.PodRoleAgent: index, found := projectionIndices[clique.Name] @@ -459,8 +477,10 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { } target := lpx.ServingComponent(source) wantImage := target.ComponentName + "-runtime" + target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil + template := target.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate if explicit { - template := target.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.DeepCopy() + template = template.DeepCopy() wantImage = "independent-conductor-runtime" template.Spec.Containers[0].Image = wantImage template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} @@ -473,6 +493,8 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { }} target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = template } + template.Spec.Containers[0].VolumeMounts = append(template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}) + template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) before := source.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) require.NoError(t, err) @@ -490,6 +512,11 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { conductors := 0 for _, clique := range pcs.Spec.Template.Cliques { container := clique.Spec.PodSpec.Containers[0] + for _, mount := range container.VolumeMounts { + require.True(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { + return volume.Name == mount.Name + }), "%s mount %s requires a volume", clique.Name, mount.Name) + } if clique.Name == plan.ConductorTemplate { conductors++ require.Equal(t, wantImage, container.Image) @@ -1252,7 +1279,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test {Key: "lpu.nvidia.com/lpu", Operator: corev1.TolerationOpExists}, {Key: "lpu.nvidia.com/node-v2", Operator: corev1.TolerationOpExists}, }, agent.Spec.PodSpec.Tolerations) - for _, name := range []string{"config", "tmp", "hugepages", "host-dev", "host-sys", "ssh-secret"} { + for _, name := range []string{"config", "hugepages", "host-dev", "host-sys", "ssh-secret"} { require.True(t, slices.ContainsFunc(agent.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }), "missing volume %s", name) } require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "host-dev" })) @@ -1271,7 +1298,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, []string{"/bin/sh", "-ec"}, cyborgMain.Command) require.Contains(t, cyborgMain.Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) require.Equal(t, []string{"--", "/configs/lpu_servers", "/tmp/lpu_servers", "/opt/gpu-runtime", "serve"}, cyborgMain.Args[1:]) - require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "config" })) + require.Contains(t, cyborgMain.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) volumes := make(map[string]corev1.Volume, len(cyborg.Spec.PodSpec.Volumes)) @@ -1359,8 +1386,16 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin cyborgCommand []string cyborgArgs []string hybridOnly bool + configPath *string + configRole int + envFrom bool }{ {name: "image entrypoint and image command"}, + {name: "default startup without config mount", configPath: ptr.To(""), configRole: 1}, + {name: "default startup with misplaced config mount", configPath: ptr.To("/custom"), configRole: 1}, + {name: "default Agent without config mount", configPath: ptr.To("")}, + {name: "default Agent with misplaced config mount", configPath: ptr.To("/custom")}, + {name: "default Agent without config mount and unrelated EnvFrom", configPath: ptr.To(""), envFrom: true}, {name: "image entrypoint with user arguments", args: []string{"serve"}}, {name: "explicit executable with image arguments", command: []string{"/opt/custom-runtime"}}, {name: "explicit executable and arguments", command: []string{"/opt/custom-runtime"}, args: []string{"serve"}}, @@ -1386,143 +1421,129 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin }, } - t.Log("Exercise both merge algorithms; shared policy tests cover default selection") - strategies := []struct { - name string - componentStrategy v1beta1.ExtraPodSpecMergeStrategy - defaultStrategy v1alpha1.ExtraPodSpecMergeStrategy - }{ - { - name: "component override", - componentStrategy: v1beta1.ExtraPodSpecMergeStrategyOverride, - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, - }, - { - name: "component strategic", - componentStrategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, - }, - } - for _, mode := range modes { t.Run(mode.name, func(t *testing.T) { payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", mode.file)) require.NoError(t, err) - for _, strategy := range strategies { - t.Run(strategy.name, func(t *testing.T) { - for _, intent := range intents { - if intent.hybridOnly && !mode.hybrid { - continue + for _, intent := range intents { + if intent.hybridOnly && !mode.hybrid { + continue + } + t.Run(intent.name, func(t *testing.T) { + t.Log("Render the LPX LPU and GPU roles from the same command intent") + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + component := dgd.GetComponentByName("lpu") + require.NotNil(t, component) + agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) + templates := []*corev1.PodTemplateSpec{agent.PodTemplate, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate} + for roleIndex, template := range templates { + main := &template.Spec.Containers[0] + require.Equal(t, commonconsts.MainContainerName, main.Name) + command := intent.command + args := intent.args + if roleIndex == 1 && intent.cyborgCommand != nil { + command = intent.cyborgCommand + args = intent.cyborgArgs } - t.Run(intent.name, func(t *testing.T) { - t.Log("Render the LPX LPU and GPU roles from the same command intent") - dgd := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, dgd)) - component := dgd.GetComponentByName("lpu") - require.NotNil(t, component) - component.ExtraPodSpecMergeStrategy = strategy.componentStrategy - agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) - lpxMain := &agent.PodTemplate.Spec.Containers[0] - lpxMain.VolumeMounts = append(lpxMain.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) - templates := []*corev1.PodTemplateSpec{agent.PodTemplate} - if mode.hybrid { - templates = append(templates, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate) - } - for roleIndex, template := range templates { - main := &template.Spec.Containers[0] - require.Equal(t, commonconsts.MainContainerName, main.Name) - command := intent.command - args := intent.args - if roleIndex == 1 && intent.cyborgCommand != nil { - command = intent.cyborgCommand - args = intent.cyborgArgs - } - main.Command = slices.Clone(command) - main.Args = slices.Clone(args) + main.Command = slices.Clone(command) + main.Args = slices.Clone(args) + if intent.envFrom && roleIndex == 0 { + main.EnvFrom = []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated-env"}}}} + } + if intent.configPath != nil && roleIndex == intent.configRole { + configIndex := slices.IndexFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "config" }) + require.NotEqual(t, -1, configIndex) + require.Equal(t, "/configs", main.VolumeMounts[configIndex].MountPath) + if *intent.configPath == "" { + main.VolumeMounts = slices.Delete(main.VolumeMounts, configIndex, configIndex+1) + } else { + main.VolumeMounts[configIndex].MountPath = *intent.configPath } + } + } - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) - require.NoError(t, err) - pcs, _, err := renderPodCliqueSet( - t.Context(), - dgd, - &configv1alpha1.OperatorConfiguration{ - PodGeneration: configv1alpha1.PodGenerationConfiguration{ - DefaultExtraPodSpecMergeStrategy: strategy.defaultStrategy, - }, - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, - }, - &controller_common.RuntimeConfig{}, - kubeClient, - nil, - selected, - mustPlanSelectedLPX(t, dgd, selected), - newLPXRenderDeployment(t, dgd), - ) - if mode.single && len(intent.command) == 0 && len(intent.args) > 0 { - require.ErrorContains(t, err, "only supports --instance-model-name") - return + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) + require.NoError(t, err) + pcs, _, err := renderPodCliqueSet( + t.Context(), + dgd, + &configv1alpha1.OperatorConfiguration{ + MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, + }, + &controller_common.RuntimeConfig{}, + kubeClient, + nil, + selected, + mustPlanSelectedLPX(t, dgd, selected), + newLPXRenderDeployment(t, dgd), + ) + if intent.configPath != nil { + require.ErrorContains(t, err, `requires a volume mounted at "/configs"`) + return + } + if mode.single && len(intent.command) == 0 && len(intent.args) > 0 { + require.ErrorContains(t, err, "only supports --instance-model-name") + return + } + require.NoError(t, err) + + t.Log("Verify explicit commands are preserved and direct Agents receive their launch defaults") + var agents, conductors, cyborgs int + for _, clique := range pcs.Spec.Template.Cliques { + main := clique.Spec.PodSpec.Containers[0] + switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { + case lpxv1alpha1.PodRoleAgent: + agents++ + require.False(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) + if len(intent.command)+len(intent.args) == 0 { + require.Equal(t, []string{"/bin/bash"}, main.Command) + require.Len(t, main.Args, 2) + require.Equal(t, "-c", main.Args[0]) + require.Contains(t, main.Args[1], "GROVE_PCLQ_POD_INDEX") + } else { + wantCommand := intent.command + require.Equal(t, wantCommand, main.Command) + require.True(t, slices.Equal(intent.args, main.Args)) } - require.NoError(t, err) - - t.Log("Verify explicit commands are preserved and direct Agents receive their launch defaults") - var agents, conductors, cyborgs int - for _, clique := range pcs.Spec.Template.Cliques { - main := clique.Spec.PodSpec.Containers[0] - switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { - case lpxv1alpha1.PodRoleAgent: - agents++ - require.True(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) - if len(intent.command)+len(intent.args) == 0 { - require.Equal(t, []string{"/bin/bash"}, main.Command) - require.Len(t, main.Args, 2) - require.Equal(t, "-c", main.Args[0]) - require.Contains(t, main.Args[1], "GROVE_PCLQ_POD_INDEX") - } else { - wantCommand := intent.command - require.Equal(t, wantCommand, main.Command) - require.True(t, slices.Equal(intent.args, main.Args)) - } - case lpxv1alpha1.PodRoleConductor: - conductors++ - command := intent.command - if len(command) == 0 { - command = []string{"/bin/nova"} - } - require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) - require.Equal(t, append(slices.Clone(command), intent.args...), main.Args[4:4+len(command)+len(intent.args)]) - allocation := slices.Index(main.Args, "--allocation") - require.GreaterOrEqual(t, allocation, 0) - require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) - case lpxv1alpha1.PodRoleCyborgWorker: - cyborgs++ - wantCommand := intent.command - wantArgs := intent.args - if intent.cyborgCommand != nil { - wantCommand = intent.cyborgCommand - wantArgs = intent.cyborgArgs - } - if len(wantCommand) == 0 { - wantCommand = []string{"/usr/local/bin/cyborg"} - } - require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) - require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[4:]) - require.NotNil(t, main.LivenessProbe) - require.NotNil(t, main.ReadinessProbe) - require.NotNil(t, main.StartupProbe) - require.NotEmpty(t, main.Ports) - } + case lpxv1alpha1.PodRoleConductor: + conductors++ + command := intent.command + if len(command) == 0 { + command = []string{"/bin/nova"} } - require.Positive(t, agents) - if mode.hybrid { - require.Zero(t, conductors) - require.Equal(t, 1, cyborgs) - } else { - require.Equal(t, 1, conductors) - require.Zero(t, cyborgs) + require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) + require.Equal(t, append(slices.Clone(command), intent.args...), main.Args[4:4+len(command)+len(intent.args)]) + allocation := slices.Index(main.Args, "--allocation") + require.GreaterOrEqual(t, allocation, 0) + require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) + case lpxv1alpha1.PodRoleCyborgWorker: + cyborgs++ + wantCommand := intent.command + wantArgs := intent.args + if intent.cyborgCommand != nil { + wantCommand = intent.cyborgCommand + wantArgs = intent.cyborgArgs } - }) + if len(wantCommand) == 0 { + wantCommand = []string{"/usr/local/bin/cyborg"} + } + require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) + require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[4:]) + require.NotNil(t, main.LivenessProbe) + require.NotNil(t, main.ReadinessProbe) + require.NotNil(t, main.StartupProbe) + require.NotEmpty(t, main.Ports) + } + } + require.Positive(t, agents) + if mode.hybrid { + require.Zero(t, conductors) + require.Equal(t, 1, cyborgs) + } else { + require.Equal(t, 1, conductors) + require.Zero(t, cyborgs) } }) } @@ -1551,7 +1572,7 @@ func getResource[T any](t *testing.T, resources []client.Object, name string) T return resources[i].(T) } -func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { +func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { t.Log("Load an authored hybrid engine and override its independent leader template") payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) require.NoError(t, err) @@ -1561,6 +1582,7 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ {Name: "config", MountPath: "/custom-config", ReadOnly: true}, {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, + {Name: "model-storage", MountPath: "/nfs"}, } leader.Spec.Containers[0].Env = []corev1.EnvVar{ {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, @@ -1568,19 +1590,17 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, {Name: lpx.CyborgBatchSizeEnv, Value: "2"}, } - leader.Spec.Volumes = []corev1.Volume{{ + authoredConfig := corev1.Volume{ Name: "config", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }} + } + leader.Spec.Volumes = append(leader.Spec.Volumes, authoredConfig) - t.Log("Render the complete LPX workload with strategic merge enabled by operator default") + t.Log("Render the complete LPX workload with ordinary pod spec overrides") selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) pcs, _, err := renderPodCliqueSet(t.Context(), source, &configv1alpha1.OperatorConfiguration{ - PodGeneration: configv1alpha1.PodGenerationConfiguration{ - DefaultExtraPodSpecMergeStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, - }, MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, }, &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, @@ -1592,24 +1612,21 @@ func TestLPXRenderingPreservesStrategicCyborgOverrides(t *testing.T) { require.GreaterOrEqual(t, cliqueIndex, 0) podSpec := pcs.Spec.Template.Cliques[cliqueIndex].Spec.PodSpec - t.Log("Keep one authored config volume without the generated ConfigMap union member") - matches := 0 - var configVolume *corev1.Volume - for i := range podSpec.Volumes { - if podSpec.Volumes[i].Name == "config" { - matches++ - configVolume = &podSpec.Volumes[i] + t.Log("Keep one config volume alongside the authored InfiniBand volume") + require.Contains(t, podSpec.Volumes, leader.Spec.Volumes[0]) + var configVolumes []corev1.Volume + for _, volume := range podSpec.Volumes { + if volume.Name == authoredConfig.Name { + configVolumes = append(configVolumes, volume) } } - require.Equal(t, 1, matches) - require.NotNil(t, configVolume) - require.NotNil(t, configVolume.VolumeSource.EmptyDir) - require.Nil(t, configVolume.VolumeSource.ConfigMap) + require.Equal(t, []corev1.Volume{authoredConfig}, configVolumes) t.Log("Preserve authored mounts and runtime environment over the generated Cyborg defaults") main := podSpec.Containers[0] require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom-config", ReadOnly: true}) require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}) + require.False(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" })) env := make(map[string]string, len(main.Env)) for _, variable := range main.Env { env[variable.Name] = variable.Value diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 3501352aeceb..d4532b277bac 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -1669,15 +1669,6 @@ func generateBasePodSpecWithDefaults( componentDefaults ComponentDefaults, containerGPUs ContainerGPUCount, ) (*corev1.PodSpec, error) { - // Resolve the one merge policy shared by every PodTemplate overlay. - podTemplateMergeStrategy, err := v1alpha1.ResolveExtraPodSpecMergeStrategy( - v1alpha1.ExtraPodSpecMergeStrategy(component.ExtraPodSpecMergeStrategy), - operatorConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy, - ) - if err != nil { - return nil, fmt.Errorf("failed to resolve podTemplate merge strategy: %w", err) - } - // Start with base container generated per component type annotations := GetPodTemplateAnnotations(component) componentContext, err := generateComponentContext(component, parentGraphDeploymentName, namespace, numberOfNodes, NewDiscoveryContext(operatorConfig.Discovery.Backend, annotations)) @@ -1690,9 +1681,29 @@ func generateBasePodSpecWithDefaults( } if main := GetMainContainer(component); main != nil { - if err := mergeContainerByName(&container, main, podTemplateMergeStrategy); err != nil { + // Copy the authored container before merging fields and environment variables. + main = main.DeepCopy() + baseEnv := container.Env + if err := mergo.Merge(&container, *main, mergo.WithOverride); err != nil { return nil, fmt.Errorf("failed to merge podTemplate main container: %w", err) } + container.Env = MergeEnvs(baseEnv, main.Env) + + // An explicitly empty port list clears generated ports as well. + if main.Ports != nil { + container.Ports = main.Ports + } + + // Replace probes in full so authored handlers do not retain generated handlers. + if main.LivenessProbe != nil { + container.LivenessProbe = main.LivenessProbe + } + if main.ReadinessProbe != nil { + container.ReadinessProbe = main.ReadinessProbe + } + if main.StartupProbe != nil { + container.StartupProbe = main.StartupProbe + } } if err := applyCompilationCache(&container, component, backendFramework); err != nil { @@ -1732,7 +1743,7 @@ func generateBasePodSpecWithDefaults( sidecars := make([]corev1.Container, 0) if component.PodTemplate != nil { - podSpecOverride := *component.PodTemplate.Spec.DeepCopy() + podSpecOverride := component.PodTemplate.Spec.DeepCopy() for _, userContainer := range podSpecOverride.Containers { if userContainer.Name != commonconsts.MainContainerName { sidecars = append(sidecars, userContainer) @@ -1740,7 +1751,7 @@ func generateBasePodSpecWithDefaults( } podSpecOverride.Containers = nil - if err := mergePodSpecOverride(&podSpec, podSpecOverride, podTemplateMergeStrategy); err != nil { + if err := mergo.Merge(&podSpec, podSpecOverride, mergo.WithOverride); err != nil { return nil, fmt.Errorf("failed to merge podTemplate spec: %w", err) } } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 96aa53bbf539..f76c637d8752 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -296,7 +296,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) lpuDefaults := &imageEntrypointComponentDefaults{ComponentDefaults: &BaseComponentDefaults{}} lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, workload, lpuDefaults, nil) + p.discoveryContext, lpuDefaults, nil) if err != nil { return nil, nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) } @@ -313,7 +313,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m if conductor != nil && conductor.PodTemplate != nil { role := lpxRoleComponent(component, conductor.PodTemplate, p.dynamoDeployment, p.discoveryBackend) input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, workload, lpuDefaults, nil) + p.discoveryContext, lpuDefaults, nil) if err != nil { return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) } @@ -353,7 +353,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m gpu.componentName = component.ComponentName gpu.r = ServiceRole{Name: workload.CyborgTemplateName(), Role: RoleMain, Replicas: *role.Replicas} gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, workload, defaults, p.groveClusterTopologyDomains) + p.discoveryContext, defaults, p.groveClusterTopologyDomains) if err != nil { return nil, nil, fmt.Errorf("rendering %s.conductor: failed to generate podSpec for role %s: %w", component.ComponentName, gpu.r.Name, err) } @@ -431,15 +431,6 @@ func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentCont return container, nil } -func (d *selectedCyborgComponentDefaults) GetBasePodSpec(context ComponentContext) (corev1.PodSpec, error) { - podSpec, err := d.ComponentDefaults.GetBasePodSpec(context) - if err != nil { - return corev1.PodSpec{}, err - } - dynamolpx.ApplySelectedCyborgPodDefaults(&podSpec, d.configMapName) - return podSpec, nil -} - // renderSelectedLPXRole consumes a private component copy; other inputs are read-only. // alphaComponent may be nil; topologyDomains may be nil when no topology is consumed. func renderSelectedLPXRole( @@ -449,7 +440,6 @@ func renderSelectedLPXRole( operatorConfig *configv1alpha1.OperatorConfiguration, secretsRetriever SecretsRetriever, discoveryContext DiscoveryContext, - selectedWorkload *dynamolpx.SelectedWorkload, defaults ComponentDefaults, topologyDomains []v1beta1.TopologyDomain, ) (*corev1.PodTemplateSpec, error) { @@ -481,17 +471,12 @@ func renderSelectedLPXRole( basePodSpec.SecurityContext.FSGroupChangePolicy = ptr.To(corev1.FSGroupChangeOnRootMismatch) } - if component.IsLPX() && selectedWorkload.BuildFamily() == dynamolpx.BuildFamilyXT { - explicitVolumes := component.PodTemplate.Spec.Volumes - runtimeVolumes := []string{"config"} - if selectedWorkload.Pipeline() != dynamolpx.PipelineLPX { - runtimeVolumes = append(runtimeVolumes, "tmp", "hugepages", "host-dev", "host-sys") - } - basePodSpec.Volumes = slices.DeleteFunc(basePodSpec.Volumes, func(volume corev1.Volume) bool { - return slices.Contains(runtimeVolumes, volume.Name) && !slices.ContainsFunc(explicitVolumes, - func(explicit corev1.Volume) bool { return explicit.Name == volume.Name }) - }) - } + // LPX supplies the generated config volume after merging; discard only inferred PVCs. + explicitVolumes := component.PodTemplate.Spec.Volumes + basePodSpec.Volumes = slices.DeleteFunc(basePodSpec.Volumes, func(volume corev1.Volume) bool { + return volume.Name == "config" && !slices.ContainsFunc(explicitVolumes, + func(explicit corev1.Volume) bool { return explicit.Name == volume.Name }) + }) for _, annotationKey := range commonconsts.KubeTopologySourceAnnotationKeys() { delete(metadata.Annotations, annotationKey) } diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index d29363abed00..700878e47d4b 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -97,7 +97,7 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { t.Log("Render the role before runtime-specific lowering") template, err := renderSelectedLPXRole(component, source, nil, &configv1alpha1.OperatorConfiguration{}, &mockSecretsRetriever{}, DiscoveryContext{}, - &lpx.SelectedWorkload{}, &imageEntrypointComponentDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) + &imageEntrypointComponentDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) require.NoError(t, err) t.Log("Retain authored security context and default only an absent one") @@ -197,9 +197,6 @@ func TestLPXInputRevision(t *testing.T) { {"component/min-available", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).MinAvailable = ptr.To(int32(1)) }}, {"component/namespace", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).GlobalDynamoNamespace = true }}, {"component/runtime-version", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).RuntimeVersionOverride = "1.5.0" }}, - {"component/merge-strategy", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ExtraPodSpecMergeStrategy = v1beta1.ExtraPodSpecMergeStrategyOverride - }}, {"component/shared-memory", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).SharedMemorySize = ptr.To(resource.MustParse("16Gi")) }}, diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index fa0a4d380125..9c3a3b59d38f 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -6093,6 +6093,7 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { { name: "frontend with overriding env var", component: &v1alpha1.DynamoComponentDeploymentSharedSpec{ + ServiceName: "frontend", ComponentType: commonconsts.ComponentTypeFrontend, Envs: []corev1.EnvVar{ { @@ -6100,12 +6101,19 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { Value: "3000", }, }, + ExtraPodSpec: &v1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{ + Name: commonconsts.MainContainerName, + Ports: []corev1.ContainerPort{{Name: "metrics", ContainerPort: 9001, Protocol: corev1.ProtocolTCP}}, + Env: []corev1.EnvVar{{Name: "POD_NAME", Value: "static-pod-name"}}, + }}, }, backendFramework: BackendFrameworkVLLM, wantCommand: []string{"python3"}, wantArgs: []string{"-m", "dynamo.frontend"}, wantEnvVars: map[string]string{ - "DYN_HTTP_PORT": "3000", + "DYN_HTTP_PORT": "3000", + "POD_NAME": "static-pod-name", + "DYN_PARENT_DGD_K8S_NAME": "test-deployment", }, }, { @@ -6131,8 +6139,11 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Log("Convert and freeze the authored component, then render frontend defaults") + component := betaComponent(t, tt.component) + original := component.DeepCopy() podSpec, err := GenerateBasePodSpec( - betaComponent(t, tt.component), + component, tt.backendFramework, secretsRetriever, dynamoDeployment.Name, @@ -6153,8 +6164,14 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { if tt.wantErr { return } + t.Log("Preserve the authored input and replace authored ports without retaining generated ports") + require.Len(t, podSpec.Containers, 1) + require.Equal(t, original, component, "rendering must not mutate the authored pod template") + if main := GetMainContainer(original); main != nil && main.Ports != nil { + require.Equal(t, main.Ports, podSpec.Containers[0].Ports) + } - // Check command and args + t.Log("Preserve expected frontend command and arguments") if !reflect.DeepEqual(podSpec.Containers[0].Command, tt.wantCommand) { t.Errorf("GenerateBasePodSpec() command = %v, want %v", podSpec.Containers[0].Command, tt.wantCommand) @@ -6164,16 +6181,13 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { podSpec.Containers[0].Args, tt.wantArgs) } - // Check environment variables - envVars := make(map[string]string) + t.Log("Keep each expected environment value exactly once without a stale value source") + envVars := make(map[string][]corev1.EnvVar) for _, env := range podSpec.Containers[0].Env { - envVars[env.Name] = env.Value + envVars[env.Name] = append(envVars[env.Name], env) } for k, v := range tt.wantEnvVars { - if envVars[k] != v { - t.Errorf("GenerateBasePodSpec() env var %s = %v, want %v", - k, envVars[k], v) - } + require.Equal(t, []corev1.EnvVar{{Name: k, Value: v}}, envVars[k]) } }) } diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go index f8fdac7e4e13..481514eacd40 100644 --- a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go @@ -110,7 +110,6 @@ func updateLPUConductorContainer( ), }, ) - container.VolumeMounts = setVolumeMount(container.VolumeMounts, conductorSSHKeyVolumeMount()) // Default readiness to Nova's health endpoint while preserving an explicit probe. if container.ReadinessProbe == nil { diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index da4421bf0205..088c6eac5f39 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -6,6 +6,7 @@ package lpx import ( + "fmt" "slices" "strconv" @@ -14,6 +15,7 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" ) // configureHybridCyborg consumes a fresh hybrid clique from an admitted source @@ -30,9 +32,13 @@ func configureHybridCyborg( container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) // Apply manifest-aware runtime bindings to the selected Cyborg container. - if err := withLPUModelStorage(&cyborg.Spec.PodSpec, container, modelStorage); err != nil { + cyborgStorage, err := lpuModelStorageBinding(cyborg.Spec.PodSpec) + if err != nil { return err } + if !apiequality.Semantic.DeepEqual(cyborgStorage.volume, modelStorage.volume) || !apiequality.Semantic.DeepEqual(cyborgStorage.mount, modelStorage.mount) { + return fmt.Errorf("selected Cyborg podTemplate conflicts with model storage mount %q", modelStorage.mount.MountPath) + } cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, cyborg.Spec.Replicas) if err != nil { return err @@ -54,7 +60,7 @@ func configureHybridCyborg( return variable.Name == selectedCyborgServerHostsFileEnv && variable.Value == runtimeTemporaryStorageMountPath+"/lpu_servers" && variable.ValueFrom == nil }) { configFile = "lpu_servers" - if err := addRuntimeConfigStorage(&cyborg.Spec.PodSpec, container, configFile); err != nil { + if err := validateRuntimeConfigStorage(&cyborg.Spec.PodSpec, container, configFile); err != nil { return err } } @@ -68,6 +74,9 @@ func configureHybridCyborg( workloadDigest, ) if cyborgConfigMap != nil { + if err := withLPUConfigVolume(&cyborg.Spec.PodSpec, cyborgConfigMap.Name, true); err != nil { + return err + } cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) } if projection.configuredBuild.Family == BuildFamilyXT { diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 5cbe404d50bf..2482643bde8c 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -15,12 +15,9 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" ) const ( - selectedCyborgInfiniBandName = "infiniband" - selectedCyborgInfiniBandPath = "/dev/infiniband" selectedCyborgServerHostsFileEnv = "SERVER_HOSTS_FILE" selectedCyborgTokenizerDirEnv = "TOKENIZER_DIR" selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" @@ -39,11 +36,6 @@ func ApplySelectedCyborgContainerDefaults( ) error { projection := workload.modelProjections[0] - container.VolumeMounts = append( - container.VolumeMounts, - corev1.VolumeMount{Name: selectedCyborgInfiniBandName, MountPath: selectedCyborgInfiniBandPath}, - corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, - ) container.Env = append( container.Env, corev1.EnvVar{ @@ -83,28 +75,6 @@ func ApplySelectedCyborgContainerDefaults( return nil } -// ApplySelectedCyborgPodDefaults installs the operator-owned V2 Cyborg -// volumes before the user's PodSpec override is merged. podSpec must be non-nil -// and is mutated in place. -func ApplySelectedCyborgPodDefaults(podSpec *corev1.PodSpec, configMapName string) { - podSpec.Volumes = append( - podSpec.Volumes, - corev1.Volume{ - Name: lpuConfigVolumeName, - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, - }}, - }, - corev1.Volume{ - Name: selectedCyborgInfiniBandName, - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: selectedCyborgInfiniBandPath, - Type: ptr.To(corev1.HostPathDirectory), - }}, - }, - ) -} - // RenderCyborgConfigMap renders the XT hybrid configuration before Cyborg defaults are merged. // The workload must contain the selected hybrid model; agentPodSpec supplies its merged storage. func (w *SelectedWorkload) RenderCyborgConfigMap( diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index e271fb9521d7..866cce35618c 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -20,7 +20,6 @@ import ( controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/pelletier/go-toml/v2" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" @@ -161,26 +160,6 @@ func lpuModelStorageBinding(spec corev1.PodSpec) (lpuModelStorage, error) { return lpuModelStorage{volume: spec.Volumes[volumeIndex], mount: mount}, nil } -// withLPUModelStorage requires a nonnil spec and a unique container retained from its unchanged Containers slice. -func withLPUModelStorage(spec *corev1.PodSpec, container *corev1.Container, storage lpuModelStorage) error { - volumeIndex := slices.IndexFunc(spec.Volumes, func(existing corev1.Volume) bool { return existing.Name == storage.volume.Name }) - if volumeIndex < 0 { - spec.Volumes = append(spec.Volumes, storage.volume) - } else if !apiequality.Semantic.DeepEqual(spec.Volumes[volumeIndex], storage.volume) { - return fmt.Errorf("selected Cyborg podTemplate volume %q conflicts with model storage", storage.volume.Name) - } - - mountIndex := slices.IndexFunc(container.VolumeMounts, func(existing corev1.VolumeMount) bool { - return existing.Name == storage.mount.Name || existing.MountPath == storage.mount.MountPath - }) - if mountIndex < 0 { - container.VolumeMounts = append(container.VolumeMounts, storage.mount) - } else if !apiequality.Semantic.DeepEqual(container.VolumeMounts[mountIndex], storage.mount) { - return fmt.Errorf("selected Cyborg main container conflicts with model storage mount %q", storage.mount.MountPath) - } - return nil -} - func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (map[string]any, error) { if projections[0].pipeline != PipelineSpecDecode { config, err := nestedLPUModelConfig(projections[0], modelStoragePath) @@ -444,26 +423,14 @@ func withLPUConfigVolume(spec *corev1.PodSpec, configMapName string, allowOverri } container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) found = false + configMount := corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath} for _, mount := range container.VolumeMounts { if mount.Name == lpuConfigVolumeName || mount.MountPath == lpuConfigMountPath { - if !allowOverrides && (found || - mount.Name != lpuConfigVolumeName || - mount.MountPath != lpuConfigMountPath || - mount.ReadOnly || - mount.RecursiveReadOnly != nil || - mount.SubPath != "" || - mount.MountPropagation != nil || - mount.SubPathExpr != "") { + if !allowOverrides && (found || mount != configMount) { return fmt.Errorf("selected LPX main container reserves volume %q at %q", lpuConfigVolumeName, lpuConfigMountPath) } - found = found || mount.MountPath == lpuConfigMountPath + found = true } } - if !found { - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: lpuConfigVolumeName, - MountPath: lpuConfigMountPath, - }) - } return nil } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index 7118a57e8377..c5f74dc5a85e 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -19,39 +19,28 @@ const ( defaultLPUAgentCPUs int64 = 62 ) -func applyLPUHostDeviceVolumes(podSpec *corev1.PodSpec, replaceExisting bool) error { - apply := appendVolumeIfMissing - if replaceExisting { - apply = setVolumeByName - } - - // Keep authored storage only when it exposes the required host device directory. - for _, required := range []corev1.Volume{lpuHostDeviceVolume("host-dev", "/dev"), lpuHostDeviceVolume("host-sys", "/sys")} { - if !replaceExisting { - for _, existing := range podSpec.Volumes { - hostPath := existing.HostPath - if existing.Name == required.Name && (hostPath == nil || hostPath.Path != required.HostPath.Path || - hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && - *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate) { - return fmt.Errorf("selected LPX podTemplate volume %q must use directory hostPath %q", required.Name, required.HostPath.Path) - } - } +func validateLPUHostDeviceVolumes(podSpec *corev1.PodSpec) error { + // Keep the existing compatibility checks without supplying missing device volumes. + for _, volume := range podSpec.Volumes { + var path string + switch volume.Name { + case "host-dev": + path = "/dev" + case "host-sys": + path = "/sys" + default: + continue + } + hostPath := volume.HostPath + if hostPath == nil || hostPath.Path != path || + hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && + *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate { + return fmt.Errorf("selected LPX podTemplate volume %q must use directory hostPath %q", volume.Name, path) } - podSpec.Volumes = apply(podSpec.Volumes, required) } return nil } -func lpuHostDeviceVolume(name, path string) corev1.Volume { - return corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: path, - Type: ptr.To(corev1.HostPathDirectory), - }}, - } -} - func updateWorkerPodSpec(podSpec *corev1.PodSpec) { // TODO can we remove some of this when we get the device-plugin? podSpec.HostIPC = true @@ -80,10 +69,6 @@ func applyLPUWorkerContainerBase(container *corev1.Container, preserveCommand bo } container.TTY = true container.Stdin = true - container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ - Name: "hugepages", - MountPath: "/dev/hugepages", - }) // Agent scheduling already initialized device resources; add runtime CPU and hugepage requirements. cpuRequest := *resource.NewQuantity(defaultLPUAgentCPUs, resource.DecimalSI) @@ -94,14 +79,3 @@ func applyLPUWorkerContainerBase(container *corev1.Container, preserveCommand bo container.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") container.Resources.Limits[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") } - -func addLPUHostDeviceVolumeMounts(container *corev1.Container) { - container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ - Name: "host-sys", - MountPath: "/sys", - }) - container.VolumeMounts = setVolumeMount(container.VolumeMounts, corev1.VolumeMount{ - Name: "host-dev", - MountPath: "/dev", - }) -} diff --git a/deploy/operator/internal/dynamo/lpx/pod_mutation.go b/deploy/operator/internal/dynamo/lpx/pod_mutation.go deleted file mode 100644 index 004730f089f5..000000000000 --- a/deploy/operator/internal/dynamo/lpx/pod_mutation.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "slices" - - corev1 "k8s.io/api/core/v1" -) - -func appendVolumeIfMissing(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume { - for _, existing := range volumes { - if existing.Name == volume.Name { - return volumes - } - } - return append(volumes, volume) -} - -func setVolumeByName(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume { - return replaceConflicts(volumes, volume, func(existing corev1.Volume) bool { - return existing.Name == volume.Name - }) -} - -func setVolumeMount(volumeMounts []corev1.VolumeMount, volumeMount corev1.VolumeMount) []corev1.VolumeMount { - return replaceConflicts(volumeMounts, volumeMount, func(existing corev1.VolumeMount) bool { - return existing.MountPath == volumeMount.MountPath - }) -} - -func replaceConflicts[T any](items []T, replacement T, conflicts func(T) bool) []T { - // Preserve the first conflict's position while removing later conflicts, or append when none exists. - first := slices.IndexFunc(items, conflicts) - if first < 0 { - return append(items, replacement) - } - - out := append(items[:first], replacement) - for _, item := range items[first+1:] { - if !conflicts(item) { - out = append(out, item) - } - } - return out -} diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 81c5287a4017..0c2ea5c1a412 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -255,22 +255,43 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { }, }, {Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "read-only-tmp"}}}, + {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, + {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, + {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret", DefaultMode: ptr.To(int32(0644))}}}, }, } + base.Containers[0].VolumeMounts = append(base.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) conductor := *base.DeepCopy() agent := *base.DeepCopy() t.Log("Lower the image into the Nova conductor and privileged SSH worker roles") require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret"), "writable storage") + require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), "writable storage") conductor = *base.DeepCopy() + agent = *base.DeepCopy() conductor.Containers[0].VolumeMounts[0].ReadOnly = false conductor.Volumes[2].VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} + conductor.Volumes[5].Secret.DefaultMode = ptr.To(int32(0600)) + conductor.Volumes = append(conductor.Volumes, corev1.Volume{Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + conductor.Containers[0].VolumeMounts = append(conductor.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir, ReadOnly: true, + }) + agent.Containers[0].VolumeMounts[0].ReadOnly = false + agent.Volumes[2].VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} + agent.Containers[0].VolumeMounts = append(agent.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: "host-dev", MountPath: "/dev"}, + corev1.VolumeMount{Name: "host-sys", MountPath: "/sys"}, + corev1.VolumeMount{Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, + ) + conductorMounts := slices.Clone(conductor.Containers[0].VolumeMounts) + agentMounts := slices.Clone(agent.Containers[0].VolumeMounts) require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret")) require.NoError(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret")) t.Log("Verify the conductor starts Nova with immutable placement and source behavior") require.Len(t, conductor.Containers, 1) conductorContainer := conductor.Containers[0] + require.Equal(t, conductorMounts, conductorContainer.VolumeMounts) require.Equal(t, "conductor", conductorContainer.Name) require.Equal(t, []string{"/bin/sh", "-ec"}, conductorContainer.Command) require.Equal(t, []string{runtimeConfigExpansion + "exec \"$@\"\n", "--", "/configs/datacenter.toml", "/tmp/datacenter.toml", "/bin/nova"}, conductorContainer.Args[:5]) @@ -302,6 +323,7 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { t.Log("Verify the Agent hosts SSHD with local devices and the source resource profile") require.Len(t, agent.Containers, 1) agentContainer := agent.Containers[0] + require.Equal(t, agentMounts, agentContainer.VolumeMounts) require.Equal(t, lpuAgentContainerName, agentContainer.Name) require.Equal(t, []string{"/bin/bash"}, agentContainer.Command) require.Equal(t, []string{"-c", lpuWorkerRunCommand}, agentContainer.Args) @@ -349,10 +371,10 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { func TestConfigureNodeLocalXTConductorSSHInitUsesMainImage(t *testing.T) { t.Log("Place an unrelated sidecar before the XT conductor main container") - podSpec := corev1.PodSpec{Containers: []corev1.Container{ - {Name: "metrics", Image: "metrics-sidecar", ImagePullPolicy: corev1.PullAlways}, - {Name: commonconsts.MainContainerName, Image: "lpu-runtime", ImagePullPolicy: corev1.PullIfNotPresent}, - }} + podSpec := renderTestPodSpec() + podSpec.Containers[0].Image = "lpu-runtime" + podSpec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent + podSpec.Containers = append([]corev1.Container{{Name: "metrics", Image: "metrics-sidecar", ImagePullPolicy: corev1.PullAlways}}, podSpec.Containers...) t.Log("Configure the conductor without deriving its runtime identity from list position") require.NoError(t, configureNodeLocalConductorRuntime(&podSpec, BuildFamilyXT, "lpu-wkr-m-0", "ssh-secret")) @@ -370,10 +392,11 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { const sshSecretName = "mpi.ssh" t.Log("Configure each XT runtime role with a valid dotted SSH Secret name") - base := corev1.PodSpec{Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}} + base := renderTestPodSpec() + testVolumeByName(t, base.Volumes, runtimeSSHVolumeName).Secret.SecretName = sshSecretName configureAgentScheduling(&base, BuildFamilyXT) direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() - require.NoError(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", sshSecretName)) + require.NoError(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", sshSecretName, false)) require.NoError(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, sshSecretName)) require.NoError(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", sshSecretName)) @@ -396,6 +419,110 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { } } +func TestGeneratedAgentSSHMountRequirements(t *testing.T) { + for _, runtime := range []struct { + name string + family BuildFamily + direct, multiNode bool + }{ + {"XT", BuildFamilyXT, false, false}, + {"HX", BuildFamilyHX, false, false}, + {"direct single-node", BuildFamilyXT, true, false}, + {"direct multi-node", BuildFamilyXT, true, true}, + } { + for _, test := range []struct { + name string + mount corev1.VolumeMount + command []string + args []string + wantErr bool + }{ + {name: "missing", wantErr: true}, + {name: "wrong volume", mount: corev1.VolumeMount{Name: "other", MountPath: "/ssh-pk"}, wantErr: true}, + {name: "misplaced", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/elsewhere"}, wantErr: true}, + {name: "subPath", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", SubPath: "private.key"}, wantErr: true}, + {name: "subPathExpr", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", SubPathExpr: "$(KEY)"}, wantErr: true}, + {name: "writable", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk"}}, + {name: "read-only", mount: corev1.VolumeMount{Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}}, + {name: "custom command", command: []string{"/custom-agent"}}, + {name: "custom args", args: []string{"custom-agent"}}, + } { + custom := len(test.command) != 0 || len(test.args) != 0 + if runtime.family == BuildFamilyHX && custom { + continue + } + t.Run(runtime.name+"/"+test.name, func(t *testing.T) { + t.Log("Author the Agent startup and selected key mount on a fresh runtime fixture") + pod := renderTestPodSpec() + main := &pod.Containers[0] + main.Command, main.Args = test.command, test.args + main.Resources = corev1.ResourceRequirements{} + main.SecurityContext = &corev1.SecurityContext{ReadOnlyRootFilesystem: ptr.To(true)} + main.VolumeMounts = nil + if test.mount.Name != "" { + main.VolumeMounts = append(main.VolumeMounts, test.mount) + } + if runtime.family == BuildFamilyXT && !custom { + main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) + } + before := pod.DeepCopy() + configureAgentScheduling(&pod, runtime.family) + + t.Log("Require SSH only for generated startup that consumes the key directory") + var err error + if runtime.direct { + err = configureDirectHybridAgentRuntime(&pod, "graph-lpu", "ssh-secret", runtime.multiNode) + } else { + err = configureNodeLocalAgentRuntime(&pod, runtime.family, false, "ssh-secret") + } + if test.wantErr && (!runtime.direct || runtime.multiNode) { + require.ErrorContains(t, err, `requires volume "ssh-secret" mounted at "/ssh-pk" without subPath`) + } else { + require.NoError(t, err) + require.False(t, ptr.Deref(pod.Containers[0].SecurityContext.ReadOnlyRootFilesystem, false)) + } + require.Equal(t, before.Containers[0].VolumeMounts, pod.Containers[0].VolumeMounts) + require.Equal(t, before.Volumes, pod.Volumes) + if custom { + require.Equal(t, test.command, pod.Containers[0].Command) + require.Equal(t, test.args, pod.Containers[0].Args) + } + }) + } + } +} + +func TestGeneratedAgentConfigMountOverrides(t *testing.T) { + for _, test := range []struct { + name string + env []corev1.EnvVar + wantErr bool + }{ + {name: "default", wantErr: true}, + {name: "empty", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR"}}, wantErr: true}, + {name: "canonical", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", Value: "/configs"}}, wantErr: true}, + {name: "custom", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", Value: "/custom"}}}, + {name: "ValueFrom", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "custom"}, Key: "config-dir"}, + }}}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Require /configs unless an explicit variable overrides it; EnvFrom alone is insufficient") + container := corev1.Container{Env: test.env, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated"}, + }}}} + before := container.DeepCopy() + err := validateGeneratedAgentMounts(&container, true, false) + if test.wantErr { + require.ErrorContains(t, err, `requires a volume mounted at "/configs"`) + } else { + require.NoError(t, err) + } + require.Equal(t, *before, container) + }) + } +} + func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { for _, test := range []struct { name string @@ -428,14 +555,21 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { {Name: commonconsts.MainContainerName, Image: "lpu-runtime"}, {Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: test.volume.Name, MountPath: "/application"}}}, }, - Volumes: []corev1.Volume{test.volume}, + Volumes: []corev1.Volume{ + test.volume, + {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, + }, + } + base.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, } configureAgentScheduling(&base, BuildFamilyXT) direct, agent := base.DeepCopy(), base.DeepCopy() t.Log("Reject incompatible bindings through both XT runtime entry points") errors := []error{ - configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret"), + configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret", false), configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), } for _, err := range errors { @@ -448,6 +582,7 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { t.Log("Never rebind the authored volume or its sidecar mount") for _, pod := range []*corev1.PodSpec{direct, agent} { + require.Equal(t, base.Volumes, pod.Volumes) require.Equal(t, test.volume.VolumeSource, testVolumeByName(t, pod.Volumes, test.volume.Name)) require.Equal(t, base.Containers[1], pod.Containers[1]) } @@ -491,20 +626,22 @@ func TestXTRuntimesValidateTemporaryStorage(t *testing.T) { t.Log("Use either a named tmp volume or an existing /tmp mount") base := corev1.PodSpec{ Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, - Volumes: []corev1.Volume{{Name: name, VolumeSource: test.source}}, - } - if name != "tmp" || test.readOnly { - base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: name, MountPath: "/tmp", ReadOnly: test.readOnly}} + Volumes: []corev1.Volume{ + {Name: name, VolumeSource: test.source}, + {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, + }, } + base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: name, MountPath: "/tmp", ReadOnly: test.readOnly}, {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}} if name != "tmp" { base.Volumes = append(base.Volumes, corev1.Volume{Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "unused"}}}) } + base.Containers[0].VolumeMounts = append(base.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) configureAgentScheduling(&base, BuildFamilyXT) direct, agent := base.DeepCopy(), base.DeepCopy() t.Log("Require writable temporary storage through both XT runtime entry points") for _, err := range []error{ - configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret"), + configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret", false), configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), } { if test.wantErr { @@ -516,9 +653,8 @@ func TestXTRuntimesValidateTemporaryStorage(t *testing.T) { t.Log("Preserve authored storage and the selected mount") for _, pod := range []*corev1.PodSpec{direct, agent} { - for _, volume := range base.Volumes { - require.Equal(t, volume.VolumeSource, testVolumeByName(t, pod.Volumes, volume.Name)) - } + require.Equal(t, base.Containers[0].VolumeMounts, pod.Containers[0].VolumeMounts) + require.Equal(t, base.Volumes, pod.Volumes) if !test.wantErr { require.Contains(t, pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: name, MountPath: "/tmp"}) } @@ -528,6 +664,31 @@ func TestXTRuntimesValidateTemporaryStorage(t *testing.T) { } } +func TestRuntimeTemporaryStorageBindings(t *testing.T) { + for _, test := range []struct { + name string + mounts []corev1.VolumeMount + err string + }{ + {name: "container-local storage"}, + {name: "missing tmp source", mounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, err: `references missing volume "tmp"`}, + {name: "missing custom source", mounts: []corev1.VolumeMount{{Name: "scratch", MountPath: "/tmp"}}, err: `references missing volume "scratch"`}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Use container-local storage or validate the authored mount without inserting storage") + pod := corev1.PodSpec{Containers: []corev1.Container{{VolumeMounts: test.mounts}}} + before := pod.DeepCopy() + err := validateRuntimeTemporaryStorage(&pod, &pod.Containers[0]) + if test.err != "" { + require.ErrorContains(t, err, test.err) + } else { + require.NoError(t, err) + } + require.Equal(t, *before, pod) + }) + } +} + func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { t.Log("Define custom entrypoint, probe and Pod security preservation cases") customStartup := testExecProbe("custom-startup") @@ -587,9 +748,15 @@ func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { podSpec := corev1.PodSpec{ Containers: []corev1.Container{*test.container.DeepCopy()}, SecurityContext: test.podSecurity.DeepCopy(), + Volumes: []corev1.Volume{ + {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, + }, } + podSpec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", SubPath: "workspace"}} + authoredVolumes := slices.Clone(podSpec.Volumes) configureAgentScheduling(&podSpec, BuildFamilyXT) - err := configureDirectHybridAgentRuntime(&podSpec, "graph-lpu", "ssh-secret") + err := configureDirectHybridAgentRuntime(&podSpec, "graph-lpu", "ssh-secret", false) require.NoError(t, err) t.Log("Preserve the entrypoint and user probes while supplying defaults when absent") @@ -598,6 +765,8 @@ func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { require.Equal(t, test.wantArgs, agent.Args) require.Equal(t, test.wantStartup, agent.StartupProbe) require.Equal(t, test.wantReadiness, agent.ReadinessProbe) + require.Equal(t, []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", SubPath: "workspace"}}, agent.VolumeMounts) + require.Equal(t, authoredVolumes, podSpec.Volumes) t.Log("Preserve orthogonal Pod security settings while using the root runtime identity") wantSecurity := test.podSecurity.DeepCopy() @@ -627,7 +796,7 @@ func TestLPXInitContainerNames(t *testing.T) { for _, list := range []string{"containers", "initContainers"} { t.Run(test.name+"/"+name+"/"+list, func(t *testing.T) { t.Log("Author an independent container with an image and command to preserve") - spec := corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "runtime"}}} + spec := renderTestPodSpec() authored := corev1.Container{Name: name, Image: "custom", Command: []string{"/custom-init"}} if list == "containers" { spec.Containers = append(spec.Containers, authored) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 4cd2a8dc1925..f87544033c8a 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -206,6 +206,14 @@ func RenderSelectedNodeLocal( hybrid := projections[0].pipeline == PipelineLPX if hybrid { + // HX Cyborg may inherit its Agent's configuration mount. + container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) + if cyborgConfigMap == nil && slices.ContainsFunc(container.VolumeMounts, + func(mount corev1.VolumeMount) bool { return mount.Name == lpuConfigVolumeName }) { + if err := withLPUConfigVolume(&cyborg.Spec.PodSpec, configMap.Name, true); err != nil { + return nil, err + } + } if err := configureHybridCyborg( cyborg, projections[0], @@ -258,10 +266,17 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo } if workload.Pipeline() == PipelineLPX { + // Extra Agents beyond one per runtime partition imply multi-node SSH setup. + projection := workload.modelProjections[0] + partitionCount := len(projection.partitions) + if workload.BuildFamily() == BuildFamilyXT { + partitionCount = len(projection.configuredBuild.Partitions) + } if err := configureDirectHybridAgentRuntime( agentPodSpec, configMapName, sshSecretName, + projection.agentReplicas > partitionCount, ); err != nil { return err } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index f20080945b5b..35ffeba87f71 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -7,6 +7,7 @@ package lpx import ( "encoding/json" + "slices" "strings" "testing" @@ -28,6 +29,44 @@ const ( testTargetStageName = "target" ) +func TestHybridSSHMountFollowsRuntimePartitionSize(t *testing.T) { + for _, test := range []struct { + name string + family BuildFamily + physical int + runtime int + agents int + wantErr bool + }{ + {"XT independent single-node", BuildFamilyXT, 2, 2, 2, false}, + {"XT multi-node", BuildFamilyXT, 1, 1, 2, true}, + {"XT collapsed chain", BuildFamilyXT, 2, 1, 2, true}, + {"HX independent single-node", BuildFamilyHX, 2, 0, 2, false}, + {"HX multi-node", BuildFamilyHX, 1, 0, 2, true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Supply resolved runtime counts and an Agent without an SSH mount") + workload := &SelectedWorkload{modelProjections: []*ModelProjection{{ + pipeline: PipelineLPX, agentReplicas: test.agents, + partitions: make([]BuildPartition, test.physical), + configuredBuild: Build{Family: test.family, Partitions: make([]BuildPartition, test.runtime)}, + }}} + pod := renderTestPodSpec() + pod.Containers[0].VolumeMounts = slices.DeleteFunc(pod.Containers[0].VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == runtimeSSHVolumeName + }) + + t.Log("Require SSH only when a runtime partition spans multiple Agents") + err := configureLPURolePods(&pod, nil, workload, "config", "allocation", "ssh-secret") + if test.wantErr { + require.ErrorContains(t, err, `requires volume "ssh-secret" mounted at "/ssh-pk"`) + } else { + require.NoError(t, err) + } + }) + } +} + func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*ModelProjection, input RenderInput) (*grovev1alpha1.PodCliqueSet, error) { // Attach test projections to their authored stage without inventing templates. for _, projection := range projections { @@ -102,7 +141,8 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { template.Spec.Containers[0].VolumeMounts = append(template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) } else { - template.Spec.Containers[0].VolumeMounts[0].MountPath = "/tmp" + template.Spec.Containers[0].VolumeMounts[0].MountPath = "/model-cache" + pcs.Spec.Template.Cliques[0].Spec.PodSpec.Containers[0].VolumeMounts[1].MountPath = "/model-cache" } t.Log("Render into the fresh PCS without retaining stale runtime identity") @@ -126,13 +166,15 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { if test.family == lpxv1alpha1.TargetFamilyXt8888 { conductor := namedClique(t, rendered, "lpu-ldr") require.Equal(t, "kept", conductor.Annotations["user"]) + require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) } else { cyborg := namedClique(t, rendered, "lpu-engine-gpu").Spec.PodSpec.Containers[0] - require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/tmp"}) + require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/model-cache"}) require.Contains(t, cyborg.Env, corev1.EnvVar{ - Name: "GBUILD_MANIFEST_PATH", Value: "/tmp/model-build/manifest.v2.capnp.bin", + Name: "GBUILD_MANIFEST_PATH", Value: "/model-cache/model-build/manifest.v2.capnp.bin", }) } }) @@ -329,8 +371,14 @@ func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { RoleName: "lpu-engine-gpu", Replicas: 1, MinAvailable: &one, PodSpec: corev1.PodSpec{ SchedulerName: corev1.DefaultSchedulerName, + Volumes: []corev1.Volume{ + {Name: "model-storage", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}}}, + }, Containers: []corev1.Container{{Name: "main", Image: "cyborg", Env: []corev1.EnvVar{ {Name: selectedCyborgServerHostsFileEnv, Value: "/tmp/lpu_servers"}, + }, VolumeMounts: []corev1.VolumeMount{ + {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + {Name: "model-storage", MountPath: "/models"}, }}}, ResourceClaims: []corev1.PodResourceClaim{{ Name: "candidate", @@ -349,10 +397,20 @@ func renderTestPodSpec() corev1.PodSpec { VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: "model-storage", }}, - }}, + }, + {Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "ssh-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, + {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, + {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, + }, Containers: []corev1.Container{{ Name: "main", Image: "runtime", - VolumeMounts: []corev1.VolumeMount{{Name: "model-storage", MountPath: "/models"}}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "model-storage", MountPath: "/models"}, + {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + {Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}, + {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, + }, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{ corev1.ResourceName("lpu.nvidia.com/devices"): resource.MustParse("1"), }}, diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index a5f5fc233dd4..46ce0abd42ad 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -19,10 +19,9 @@ import ( // Preserve rendered workload identities and image-owned executable paths. const ( - runtimeTemporaryStorageVolumeName = "tmp" - runtimeTemporaryStorageMountPath = "/tmp" - lp30InitContainerName = "prepare-lp30" - lp30InitPath = "/bin/lp30-agent-v2-init" + runtimeTemporaryStorageMountPath = "/tmp" + lp30InitContainerName = "prepare-lp30" + lp30InitPath = "/bin/lp30-agent-v2-init" ) var lp30WorkerEnvironment = []corev1.EnvVar{ @@ -111,6 +110,7 @@ func stripNovaOnlyArgs(args []string) ([]string, bool) { func configureDirectHybridAgentRuntime( agentPodSpec *corev1.PodSpec, lpuConfigMapName, sshSecretName string, + multiNode bool, ) error { if strings.TrimSpace(sshSecretName) == "" { return fmt.Errorf("direct hybrid agent runtime requires an MPI SSH secret name") @@ -123,6 +123,9 @@ func configureDirectHybridAgentRuntime( agent.Name = lpuAgentContainerName applyLPUWorkerContainerBase(agent, hasCustomStartup) if len(agent.Args) == 0 && !hasCustomStartup { + if err := validateGeneratedAgentMounts(agent, true, multiNode); err != nil { + return err + } agent.Args = []string{"-c", lpuPartitionRunCommand} } @@ -141,8 +144,6 @@ func configureDirectHybridAgentRuntime( corev1.EnvVar{Name: "READINESS_PORT", Value: fmt.Sprintf("%d", LPUReadinessPort)}, ) - addLPUHostDeviceVolumeMounts(agent) - agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount()) retargetMainContainerReferences(agentPodSpec, agent) agent.Env = append(agent.Env, @@ -162,7 +163,7 @@ func configureDirectHybridAgentRuntime( }, ) setNodeLocalPodIPEnv(agent, true) - if err := addRuntimeTemporaryStorage(agentPodSpec, agent, false); err != nil { + if err := validateRuntimeTemporaryStorage(agentPodSpec, agent); err != nil { return err } updateWorkerPodSpec(agentPodSpec) @@ -175,19 +176,10 @@ func configureDirectHybridAgentRuntime( agentPodSpec.SecurityContext.RunAsGroup = ptr.To(int64(0)) agentPodSpec.SecurityContext.RunAsNonRoot = ptr.To(false) - if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { - return err - } - if err := applyLPUHostDeviceVolumes(agentPodSpec, false); err != nil { + if err := validateSSHVolume(agentPodSpec, sshSecretName); err != nil { return err } - agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ - Name: "hugepages", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumHugePages, - }}, - }) - return nil + return validateLPUHostDeviceVolumes(agentPodSpec) } // configureNodeLocalConductorRuntime consumes a nonnil conductor PodSpec. @@ -211,7 +203,7 @@ func configureNodeLocalConductorRuntime( // Materialize the GPC conductor around the image's Nova binary. setNodeLocalPodIPEnv(conductor, isXT) - if err := addRuntimeConfigStorage(conductorPodSpec, conductor, "datacenter.toml"); err != nil { + if err := validateRuntimeConfigStorage(conductorPodSpec, conductor, "datacenter.toml"); err != nil { return err } updateLPUConductorContainer(conductor, allocation) @@ -251,31 +243,30 @@ func configureNodeLocalAgentRuntime( agent.Name = lpuAgentContainerName retargetMainContainerReferences(agentPodSpec, agent) configureNodeLocalAgentWorkerContainer(agent, isXT, preserveAgentEntrypoint) - if err := addRuntimeTemporaryStorage(agentPodSpec, agent, !isXT); err != nil { + if err := validateRuntimeTemporaryStorage(agentPodSpec, agent); err != nil { return err } agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) - if err := applyLPUHostDeviceVolumes(agentPodSpec, !isXT); err != nil { + if err := validateLPUHostDeviceVolumes(agentPodSpec); err != nil { return err } - if err := addSSHVolume(agentPodSpec, sshSecretName, 0644); err != nil { + if err := validateSSHVolume(agentPodSpec, sshSecretName); err != nil { return err } - // Expose node-local devices and hugetlbfs through host /dev without an unaccounted HugePages volume. - addLPUHostDeviceVolumeMounts(agent) - agent.VolumeMounts = setVolumeMount(agent.VolumeMounts, sshVolumeMount()) + // Check only the paths consumed by the selected generated worker script. + if !preserveAgentEntrypoint { + if err := validateGeneratedAgentMounts(agent, isXT, true); err != nil { + return err + } + } + + // Apply the family-specific worker resource and initialization requirements. if isXT { applyLPUWorkerContainerBase(agent, preserveAgentEntrypoint) agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) agent.SecurityContext.RunAsNonRoot = ptr.To(false) - agentPodSpec.Volumes = appendVolumeIfMissing(agentPodSpec.Volumes, corev1.Volume{ - Name: "hugepages", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumHugePages, - }}, - }) } else { return addLP30InitContainer(agentPodSpec, agent) } @@ -340,6 +331,12 @@ func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) erro if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), lp30InitContainerName).ToAggregate(); err != nil { return err } + // The generated init mounts fixed names; their host volumes must be authored. + for _, name := range []string{"host-dev", "host-sys"} { + if !slices.ContainsFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }) { + return fmt.Errorf("LP30 initialization requires podTemplate volume %q", name) + } + } initContainer := corev1.Container{ Name: lp30InitContainerName, @@ -358,8 +355,11 @@ func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) erro RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "host-sys", MountPath: "/sys"}, + {Name: "host-dev", MountPath: "/dev"}, + }, } - addLPUHostDeviceVolumeMounts(&initContainer) podSpec.InitContainers = append(podSpec.InitContainers, initContainer) return nil } @@ -439,38 +439,48 @@ func retargetResourceFieldReferencesInDownwardAPI(items []corev1.DownwardAPIVolu } } -func addRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container, replaceExisting bool) error { - // Give conductor expansion and Agent setup a pod-lifetime writable workspace. - volume := corev1.Volume{ - Name: runtimeTemporaryStorageVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, +func validateGeneratedAgentMounts(container *corev1.Container, requireConfig, requireSSH bool) error { + // Only an explicit configuration override can waive the default mount. + usesDefaultConfigDir := true + for _, variable := range container.Env { + if variable.Name == "LPU_CONFIG_DIR" { + usesDefaultConfigDir = variable.ValueFrom == nil && (variable.Value == "" || variable.Value == lpuConfigMountPath) + } } - mount := corev1.VolumeMount{Name: runtimeTemporaryStorageVolumeName, MountPath: runtimeTemporaryStorageMountPath} - if replaceExisting { - podSpec.Volumes = setVolumeByName(podSpec.Volumes, volume) - container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) - return nil + if requireConfig && usesDefaultConfigDir && !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == lpuConfigMountPath }) { + return fmt.Errorf("LPX runtime configuration requires a volume mounted at %q", lpuConfigMountPath) } - // Validate the selected binding without rebinding authored storage used by other containers. - for _, existing := range container.VolumeMounts { - if existing.MountPath == mount.MountPath { - mount = existing - } + // Generated SSH setup reads the Secret's keys directly from this directory. + if requireSSH && !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == runtimeSSHVolumeName && mount.MountPath == runtimeSSHSecretMountPath && mount.SubPath == "" && mount.SubPathExpr == "" + }) { + return fmt.Errorf("LPX agent requires volume %q mounted at %q without subPath", runtimeSSHVolumeName, runtimeSSHSecretMountPath) } - readOnly := mount.ReadOnly - for _, existing := range podSpec.Volumes { - if existing.Name == mount.Name { - readOnly = readOnly || volumeIsReadOnly(existing.VolumeSource) - } + return nil +} + +func validateRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container) error { + // Agents use writable root filesystems; validate only explicitly mounted workspace storage. + mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.MountPath == runtimeTemporaryStorageMountPath + }) + if mountIndex < 0 { + return nil + } + mount := container.VolumeMounts[mountIndex] + if mount.ReadOnly { + return fmt.Errorf("LPX runtime requires writable storage at %q", mount.MountPath) + } + + // Validate the authored source without creating or replacing it. + volumeIndex := slices.IndexFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) + if volumeIndex < 0 { + return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) } - if readOnly { + if volumeIsReadOnly(podSpec.Volumes[volumeIndex].VolumeSource) { return fmt.Errorf("LPX runtime requires writable storage at %q", mount.MountPath) } - container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) - podSpec.Volumes = appendVolumeIfMissing(podSpec.Volumes, volume) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index 5ae01039a282..ce3a251bbe94 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -7,6 +7,7 @@ package lpx import ( "fmt" + "slices" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" @@ -25,35 +26,17 @@ const ( conductorSSHPrivateKeyPath + "; chmod 600 " + conductorSSHPrivateKeyPath ) -func addSSHVolume(podSpec *corev1.PodSpec, secretName string, defaultMode int32) error { +func validateSSHVolume(podSpec *corev1.PodSpec, secretName string) error { // Preserve authored storage and key projections instead of replacing their contents. for _, volume := range podSpec.Volumes { - if volume.Name == runtimeSSHVolumeName && (volume.Secret == nil || - volume.Secret.SecretName != secretName || len(volume.Secret.Items) != 0) { - return fmt.Errorf("selected LPX podTemplate volume %q is reserved for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) + if volume.Name == runtimeSSHVolumeName { + if volume.Secret == nil || volume.Secret.SecretName != secretName || len(volume.Secret.Items) != 0 { + return fmt.Errorf("selected LPX podTemplate volume %q is reserved for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) + } + return nil } } - - // Apply the runtime's key permissions to its dedicated Secret volume. - sshVolume := corev1.Volume{ - Name: runtimeSSHVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secretName, - DefaultMode: ptr.To(defaultMode), - }, - }, - } - podSpec.Volumes = setVolumeByName(podSpec.Volumes, sshVolume) - return nil -} - -func sshVolumeMount() corev1.VolumeMount { - return corev1.VolumeMount{ - Name: runtimeSSHVolumeName, - MountPath: runtimeSSHSecretMountPath, - ReadOnly: true, - } + return fmt.Errorf("selected LPX podTemplate requires volume %q for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) } func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName string) error { @@ -62,25 +45,22 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se return err } - // Reject authored destination storage before changing the source Secret volume. - for _, volume := range podSpec.Volumes { - if volume.Name == conductorSSHKeyVolumeName { - return fmt.Errorf("selected LPX podTemplate volume %q is reserved for the conductor SSH key", conductorSSHKeyVolumeName) - } - } - - // Mount the source Secret and the writable destination used by OpenMPI. - if err := addSSHVolume(podSpec, secretName, 0600); err != nil { + // Require the authored source Secret used by the generated init container. + if err := validateSSHVolume(podSpec, secretName); err != nil { return err } - // Keep the copied private key in a Pod-local writable volume. - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: conductorSSHKeyVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) + // The init container and Nova must share the authored Pod-local key directory. + if !slices.ContainsFunc(podSpec.Volumes, func(volume corev1.Volume) bool { + return volume.Name == conductorSSHKeyVolumeName && volume.EmptyDir != nil + }) { + return fmt.Errorf("LPX conductor requires emptyDir volume %q", conductorSSHKeyVolumeName) + } + if !slices.ContainsFunc(conductor.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == conductorSSHKeyVolumeName && mount.MountPath == conductorSSHKeyDir && mount.SubPath == "" && mount.SubPathExpr == "" + }) { + return fmt.Errorf("LPX conductor requires volume %q mounted at %q without subPath", conductorSSHKeyVolumeName, conductorSSHKeyDir) + } // Copy the key with OpenSSH's required mode before the conductor starts. initContainer := corev1.Container{ @@ -94,12 +74,11 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), }, - VolumeMounts: []corev1.VolumeMount{conductorSSHKeyVolumeMount(), sshVolumeMount()}, + VolumeMounts: []corev1.VolumeMount{ + {Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}, + {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, + }, } podSpec.InitContainers = append(podSpec.InitContainers, initContainer) return nil } - -func conductorSSHKeyVolumeMount() corev1.VolumeMount { - return corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir} -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go index 417282840ec1..b0d95e5d62b1 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go @@ -14,7 +14,7 @@ import ( "k8s.io/utils/ptr" ) -func TestAddSSHVolumePreservesAuthoredStorage(t *testing.T) { +func TestValidateSSHVolumePreservesAuthoredStorage(t *testing.T) { for _, test := range []struct { name string source corev1.VolumeSource @@ -32,37 +32,41 @@ func TestAddSSHVolumePreservesAuthoredStorage(t *testing.T) { pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: test.source}}} before := pod.DeepCopy() - t.Log("Install the MPI key only when the existing Secret binding is compatible") - err := addSSHVolume(&pod, "mpi.ssh", 0644) + t.Log("Require a compatible MPI key binding without changing the authored source") + err := validateSSHVolume(&pod, "mpi.ssh") if test.wantErr { require.ErrorContains(t, err, `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"`) require.Equal(t, *before, pod) return } require.NoError(t, err) - require.Len(t, pod.Volumes, 1) - require.Equal(t, &corev1.SecretVolumeSource{SecretName: "mpi.ssh", DefaultMode: ptr.To(int32(0644))}, pod.Volumes[0].Secret) + require.Equal(t, *before, pod) }) } } -func TestAddConductorSSHKeyRejectsAuthoredKeyVolume(t *testing.T) { +func TestAddConductorSSHKeyValidatesAuthoredStorage(t *testing.T) { t.Parallel() for _, test := range []struct { name string source corev1.VolumeSource + mount corev1.VolumeMount + err string }{ - {"Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}}, - {"PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, - {"EmptyDir", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {name: "Secret", source: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}, err: "requires emptyDir"}, + {name: "PVC", source: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}, err: "requires emptyDir"}, + {name: "missing source", err: "requires emptyDir"}, + {name: "missing mount", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, err: "without subPath"}, + {name: "subPath", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, mount: corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir, SubPath: "keys"}, err: "without subPath"}, + {name: "EmptyDir", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, mount: corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}}, } { t.Run(test.name, func(t *testing.T) { - t.Log("Mount authored storage using the reserved conductor key-volume name") + t.Log("Author the conductor key source, destination and runtime mount") pod := corev1.PodSpec{ Containers: []corev1.Container{{ Name: commonconsts.MainContainerName, Image: "lpu-runtime", - VolumeMounts: []corev1.VolumeMount{{Name: conductorSSHKeyVolumeName, MountPath: "/application-data"}}, + VolumeMounts: []corev1.VolumeMount{test.mount}, }}, Volumes: []corev1.Volume{ {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ @@ -73,10 +77,17 @@ func TestAddConductorSSHKeyRejectsAuthoredKeyVolume(t *testing.T) { } before := pod.DeepCopy() - t.Log("Reject the collision before changing storage, mounts or source Secret permissions") + t.Log("Require a shared Pod-local directory without changing authored storage") err := addConductorSSHKey(&pod, &pod.Containers[0], "mpi.ssh") - require.ErrorContains(t, err, `volume "single-v2-ssh-key" is reserved for the conductor SSH key`) - require.Equal(t, *before, pod) + if test.err != "" { + require.ErrorContains(t, err, test.err) + require.Equal(t, *before, pod) + return + } + require.NoError(t, err) + require.Equal(t, before.Volumes, pod.Volumes) + require.Equal(t, before.Containers, pod.Containers) + require.Len(t, pod.InitContainers, 1) }) } } @@ -87,12 +98,13 @@ func TestXTRuntimesRejectSSHVolumeCollision(t *testing.T) { Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, } + base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}} configureAgentScheduling(&base, BuildFamilyXT) direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() t.Log("Propagate the collision through every XT runtime configuration path") const wantError = `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"` - require.ErrorContains(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", "mpi.ssh"), wantError) + require.ErrorContains(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", "mpi.ssh", false), wantError) require.ErrorContains(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "mpi.ssh"), wantError) require.ErrorContains(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", "mpi.ssh"), wantError) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup.go b/deploy/operator/internal/dynamo/lpx/runtime_startup.go index adca8f2a9aeb..45b617b549c8 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup.go @@ -9,6 +9,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" ) const runtimeConfigExpansion = `: "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" @@ -34,18 +35,29 @@ func wrapRuntimeStartup(container *corev1.Container, defaultCommand, configFile, container.Command, container.Args = []string{"/bin/sh", "-ec"}, args } -// addRuntimeConfigStorage keeps expanded configuration writable and local to this Pod. -func addRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, configFile string) error { +// validateRuntimeConfigStorage requires writable, Pod-local storage for expanded configuration. +func validateRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, configFile string) error { // Reject incompatible authored bindings instead of replacing their volumes or mounts. - mount := corev1.VolumeMount{Name: runtimeTemporaryStorageVolumeName, MountPath: runtimeTemporaryStorageMountPath} + hasConfigMount := false + var mount corev1.VolumeMount for _, existing := range container.VolumeMounts { + hasConfigMount = hasConfigMount || existing.MountPath == lpuConfigMountPath if existing.MountPath == runtimeTemporaryStorageMountPath+"/"+configFile { return fmt.Errorf("LPX runtime configuration cannot overwrite a volume mounted at %q", existing.MountPath) } - if existing.MountPath == mount.MountPath { + if existing.MountPath == runtimeTemporaryStorageMountPath { mount = existing } } + if !hasConfigMount { + return fmt.Errorf("LPX runtime configuration requires a volume mounted at %q", lpuConfigMountPath) + } + if mount.Name == "" { + if container.SecurityContext != nil && ptr.Deref(container.SecurityContext.ReadOnlyRootFilesystem, false) { + return fmt.Errorf("LPX runtime configuration requires writable storage at %q", runtimeTemporaryStorageMountPath) + } + return nil + } if mount.ReadOnly { return fmt.Errorf("LPX runtime configuration requires writable storage at %q", mount.MountPath) } @@ -54,12 +66,8 @@ func addRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Containe if volume.EmptyDir == nil { return fmt.Errorf("LPX runtime configuration requires Pod-local emptyDir storage at %q", mount.MountPath) } - container.VolumeMounts = setVolumeMount(container.VolumeMounts, mount) return nil } } - if mount.Name != runtimeTemporaryStorageVolumeName { - return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) - } - return addRuntimeTemporaryStorage(podSpec, container, false) + return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index 307f5090ce05..a10dc935ea61 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" ) func TestRuntimeStartup(t *testing.T) { @@ -47,6 +48,21 @@ func TestRuntimeStartup(t *testing.T) { } func TestRuntimeConfigStorage(t *testing.T) { + for _, security := range []*corev1.SecurityContext{nil, {ReadOnlyRootFilesystem: ptr.To(false)}, {ReadOnlyRootFilesystem: ptr.To(true)}} { + t.Log("Allow container-local expansion unless the retained root filesystem is read-only") + pod := corev1.PodSpec{} + container := corev1.Container{SecurityContext: security, VolumeMounts: []corev1.VolumeMount{{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}}} + before := container.DeepCopy() + err := validateRuntimeConfigStorage(&pod, &container, "lpu_servers") + if security != nil && ptr.Deref(security.ReadOnlyRootFilesystem, false) { + require.ErrorContains(t, err, "writable storage") + } else { + require.NoError(t, err) + } + require.Empty(t, pod.Volumes) + require.Equal(t, *before, container) + } + for _, source := range []corev1.VolumeSource{ {EmptyDir: &corev1.EmptyDirVolumeSource{}}, {PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "shared"}}, @@ -54,15 +70,21 @@ func TestRuntimeConfigStorage(t *testing.T) { } { t.Log("Reuse only a writable Pod-local workspace for expanded runtime configuration") pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "scratch", VolumeSource: source}}} - container := corev1.Container{VolumeMounts: []corev1.VolumeMount{{Name: "scratch", MountPath: "/tmp"}}} - err := addRuntimeConfigStorage(&pod, &container, "lpu_servers") + container := corev1.Container{SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: ptr.To(true)}, VolumeMounts: []corev1.VolumeMount{ + {Name: "scratch", MountPath: "/tmp"}, + {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + }} + authoredPod, authoredContainer := pod.DeepCopy(), container.DeepCopy() + err := validateRuntimeConfigStorage(&pod, &container, "lpu_servers") if source.EmptyDir == nil { require.ErrorContains(t, err, "Pod-local emptyDir") } else { require.NoError(t, err) require.Len(t, pod.Volumes, 1) } + require.Equal(t, *authoredPod, pod) + require.Equal(t, *authoredContainer, container) container.VolumeMounts[0].ReadOnly = true - require.ErrorContains(t, addRuntimeConfigStorage(&pod, &container, "lpu_servers"), "writable storage") + require.ErrorContains(t, validateRuntimeConfigStorage(&pod, &container, "lpu_servers"), "writable storage") } } diff --git a/deploy/operator/internal/dynamo/patch.go b/deploy/operator/internal/dynamo/patch.go deleted file mode 100644 index e07e18b64b67..000000000000 --- a/deploy/operator/internal/dynamo/patch.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package dynamo - -import ( - "fmt" - "slices" - - v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/imdario/mergo" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - -func mergeObjectOverride(obj any, patch any) error { - if err := mergo.Merge(obj, patch, mergo.WithOverride); err != nil { - return fmt.Errorf("failed to apply override merge: %w", err) - } - return nil -} - -// Strategic merge field-merges matching list items, which breaks union-style -// objects when a user needs to replace an existing entry. -func removeNamed[T any](items, overrides []T, nameOf func(T) string) []T { - if len(items) == 0 || len(overrides) == 0 { - return items - } - - // Index every non-empty override name. - overrideNames := make(map[string]struct{}, len(overrides)) - for _, override := range overrides { - if name := nameOf(override); name != "" { - overrideNames[name] = struct{}{} - } - } - if len(overrideNames) == 0 { - return items - } - - // Remove every overridden base item while preserving survivor order. - return slices.DeleteFunc(items, func(item T) bool { - _, exists := overrideNames[nameOf(item)] - return exists - }) -} - -// mergeContainerByName merges override into base; both must be non-nil. -// mergeStrategy must be resolved by ResolveExtraPodSpecMergeStrategy. -// It preserves override by copying it once, then transferring owned fields to base. -func mergeContainerByName(base *corev1.Container, override *corev1.Container, mergeStrategy v1alpha1.ExtraPodSpecMergeStrategy) error { - user := override.DeepCopy() - user.Name = commonconsts.MainContainerName - - if mergeStrategy == v1alpha1.ExtraPodSpecMergeStrategyOverride { - baseEnv := base.Env - if err := mergeObjectOverride(base, *user); err != nil { - return err - } - base.Env = MergeEnvs(baseEnv, user.Env) - if user.Ports != nil { - base.Ports = user.Ports - } - } else { - clearPorts := user.Ports != nil && len(user.Ports) == 0 - base.Env = removeNamed(base.Env, user.Env, func(env corev1.EnvVar) string { return env.Name }) - if !clearPorts { - base.Ports = removeNamed(base.Ports, user.Ports, func(port corev1.ContainerPort) string { return port.Name }) - } - if err := patchObjectStrategic(base, user); err != nil { - return err - } - if clearPorts { - base.Ports = user.Ports - } - } - if user.LivenessProbe != nil { - base.LivenessProbe = user.LivenessProbe - } - if user.ReadinessProbe != nil { - base.ReadinessProbe = user.ReadinessProbe - } - if user.StartupProbe != nil { - base.StartupProbe = user.StartupProbe - } - base.Name = commonconsts.MainContainerName - return nil -} - -// mergePodSpecOverride applies the caller-owned override to podSpec, which must be non-nil. -// mergeStrategy must be resolved by ResolveExtraPodSpecMergeStrategy. -func mergePodSpecOverride(podSpec *corev1.PodSpec, podSpecOverride corev1.PodSpec, mergeStrategy v1alpha1.ExtraPodSpecMergeStrategy) error { - if mergeStrategy == v1alpha1.ExtraPodSpecMergeStrategyOverride { - return mergeObjectOverride(podSpec, &podSpecOverride) - } - - // Replace named volume unions before strategic field merging. - podSpec.Volumes = removeNamed(podSpec.Volumes, podSpecOverride.Volumes, func(volume corev1.Volume) string { return volume.Name }) - return patchObjectStrategic(podSpec, &podSpecOverride) -} - -func patchObjectStrategic[T any](obj T, patch T) error { - objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) - if err != nil { - return fmt.Errorf("failed to convert object to unstructured: %w", err) - } - - patchMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(patch) - if err != nil { - return fmt.Errorf("failed to convert patch to unstructured: %w", err) - } - - var dataStruct T - patchedObj, err := strategicpatch.StrategicMergeMapPatch(objMap, patchMap, dataStruct) - if err != nil { - return fmt.Errorf("failed to apply strategic merge patch: %w", err) - } - - err = runtime.DefaultUnstructuredConverter.FromUnstructured(patchedObj, obj) - if err != nil { - return fmt.Errorf("failed to deserialize patched object: %w", err) - } - - return nil -} diff --git a/deploy/operator/internal/dynamo/patch_test.go b/deploy/operator/internal/dynamo/patch_test.go deleted file mode 100644 index 07a6e63d434f..000000000000 --- a/deploy/operator/internal/dynamo/patch_test.go +++ /dev/null @@ -1,294 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package dynamo - -import ( - "testing" - - configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" -) - -func TestGenerateBasePodSpec_StrategicMergeReplacesExistingUnionStyleEntries(t *testing.T) { - secretsRetriever := &mockSecretsRetriever{} - controllerConfig := &configv1alpha1.OperatorConfiguration{} - controllerConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy = v1alpha1.ExtraPodSpecMergeStrategyStrategic - - t.Run("container ports with same name", func(t *testing.T) { - component := &v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "decode", - ComponentType: v1beta1.ComponentTypeDecode, - PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: commonconsts.MainContainerName, - Ports: []corev1.ContainerPort{ - { - Name: commonconsts.DynamoSystemPortName, - ContainerPort: 28090, - Protocol: corev1.ProtocolTCP, - }, - }, - }, - }, - }, - }, - } - - podSpec, err := GenerateBasePodSpec( - component, - BackendFrameworkVLLM, - secretsRetriever, - "test-deployment", - "default", - RoleMain, - 1, - controllerConfig, - commonconsts.MultinodeDeploymentTypeGrove, - "decode", - nil, - staticContainerGPUCount(0), - ) - - require.NoError(t, err) - require.Len(t, podSpec.Containers, 1) - - systemPorts := make([]int32, 0) - for _, port := range podSpec.Containers[0].Ports { - if port.Name == commonconsts.DynamoSystemPortName { - systemPorts = append(systemPorts, port.ContainerPort) - } - } - assert.Equal(t, []int32{28090}, systemPorts) - assert.NotContains(t, podSpec.Containers[0].Ports, corev1.ContainerPort{ - Name: commonconsts.DynamoSystemPortName, - ContainerPort: int32(commonconsts.DynamoSystemPort), - Protocol: corev1.ProtocolTCP, - }) - }) - - t.Run("env vars", func(t *testing.T) { - component := &v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "frontend", - ComponentType: v1beta1.ComponentTypeFrontend, - PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: commonconsts.MainContainerName, - Env: []corev1.EnvVar{ - {Name: "POD_NAME", Value: "static-pod-name"}, - }, - }, - }, - }, - }, - } - - podSpec, err := GenerateBasePodSpec( - component, - BackendFrameworkVLLM, - secretsRetriever, - "test-deployment", - "default", - RoleMain, - 1, - controllerConfig, - commonconsts.MultinodeDeploymentTypeGrove, - "frontend", - nil, - staticContainerGPUCount(0), - ) - - require.NoError(t, err) - require.Len(t, podSpec.Containers, 1) - - matches := 0 - var podNameEnv *corev1.EnvVar - for i := range podSpec.Containers[0].Env { - if podSpec.Containers[0].Env[i].Name == "POD_NAME" { - matches++ - podNameEnv = &podSpec.Containers[0].Env[i] - } - } - - require.Equal(t, 1, matches) - require.NotNil(t, podNameEnv) - assert.Equal(t, "static-pod-name", podNameEnv.Value) - assert.Nil(t, podNameEnv.ValueFrom) - }) -} - -func TestGenerateDynamoComponentsDeployments_ExtraPodSpecMergeStrategyPropagation(t *testing.T) { - tests := []struct { - name string - strategy v1beta1.ExtraPodSpecMergeStrategy - wantStrategy v1beta1.ExtraPodSpecMergeStrategy - }{ - { - name: "explicit strategy propagates to generated DCD spec", - strategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, - wantStrategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, - }, - { - name: "unset strategy remains unset on generated DCD spec", - strategy: "", - wantStrategy: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dgd := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd", - Namespace: "default", - }, - Spec: v1beta1.DynamoGraphDeploymentSpec{ - BackendFramework: "vllm", - Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ - { - ComponentName: "frontend", - ComponentType: v1beta1.ComponentTypeFrontend, - Replicas: ptr.To(int32(1)), - ExtraPodSpecMergeStrategy: tt.strategy, - }, - }, - }, - } - - dcds, err := GenerateDynamoComponentsDeployments(dgd, nil, nil, RollingUpdateContext{}) - require.NoError(t, err) - - dcd := dcds["frontend"] - require.NotNil(t, dcd) - assert.Equal(t, tt.wantStrategy, dcd.Spec.ExtraPodSpecMergeStrategy) - }) - } -} - -func TestGenerateBasePodSpec_InvalidExtraPodSpecMergeStrategy(t *testing.T) { - secretsRetriever := &mockSecretsRetriever{} - controllerConfig := &configv1alpha1.OperatorConfiguration{} - component := &v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "frontend", - ComponentType: v1beta1.ComponentTypeFrontend, - ExtraPodSpecMergeStrategy: v1beta1.ExtraPodSpecMergeStrategy("invalid"), - } - - _, err := GenerateBasePodSpec( - component, - BackendFrameworkVLLM, - secretsRetriever, - "test-deployment", - "default", - RoleMain, - 1, - controllerConfig, - commonconsts.MultinodeDeploymentTypeGrove, - "frontend", - nil, - staticContainerGPUCount(0), - ) - - require.Error(t, err) - assert.Contains(t, err.Error(), "invalid extraPodSpec merge strategy") -} - -func TestGenerateBasePodSpecForController_UsesOperatorConfigDefaultExtraPodSpecMergeStrategy(t *testing.T) { - tests := []struct { - name string - strategy v1beta1.ExtraPodSpecMergeStrategy - defaultStrategy v1alpha1.ExtraPodSpecMergeStrategy - wantPortNames []string - }{ - { - name: "falls back to override when field is absent", - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, - wantPortNames: []string{"metrics"}, - }, - { - name: "falls back to runtime default when field is absent", - strategy: "", - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, - wantPortNames: []string{commonconsts.DynamoContainerPortName, "metrics"}, - }, - { - name: "explicit field overrides runtime default", - strategy: v1beta1.ExtraPodSpecMergeStrategyOverride, - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyStrategic, - wantPortNames: []string{"metrics"}, - }, - { - name: "explicit strategic field overrides runtime override", - strategy: v1beta1.ExtraPodSpecMergeStrategyStrategic, - defaultStrategy: v1alpha1.ExtraPodSpecMergeStrategyOverride, - wantPortNames: []string{commonconsts.DynamoContainerPortName, "metrics"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - controllerConfig := &configv1alpha1.OperatorConfiguration{} - controllerConfig.PodGeneration.DefaultExtraPodSpecMergeStrategy = tt.defaultStrategy - - podSpec, err := GenerateBasePodSpecForController( - &v1beta1.DynamoComponentDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "frontend", - Namespace: "default", - }, - Spec: v1beta1.DynamoComponentDeploymentSpec{ - BackendFramework: "vllm", - DynamoComponentDeploymentSharedSpec: v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "frontend", - ComponentType: v1beta1.ComponentTypeFrontend, - ExtraPodSpecMergeStrategy: tt.strategy, - PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: commonconsts.MainContainerName, - Ports: []corev1.ContainerPort{ - { - Name: "metrics", - ContainerPort: 9001, - Protocol: corev1.ProtocolTCP, - }, - }, - }, - }, - }, - }, - }, - }, - }, - &mockSecretsRetriever{}, - controllerConfig, - RoleMain, - commonconsts.MultinodeDeploymentTypeGrove, - staticContainerGPUCount(0), - GenerateBasePodSpecForControllerOptions{}, - ) - require.NoError(t, err) - require.Len(t, podSpec.Containers, 1) - - var gotPortNames []string - for _, port := range podSpec.Containers[0].Ports { - gotPortNames = append(gotPortNames, port.Name) - } - assert.ElementsMatch(t, tt.wantPortNames, gotPortNames) - }) - } -} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index af6952134982..440855854fc3 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -39,7 +39,6 @@ spec: name: model-storage - name: lpu type: lpx - extraPodSpecMergeStrategy: strategic replicas: 2 lpx: buildId: llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c @@ -77,10 +76,14 @@ spec: limits: nvidia.com/gpu: '1' volumeMounts: + - name: config + mountPath: /configs - mountPath: /etc/cyborg/model-card.json name: cyborg-model-card readOnly: true subPath: model-card.json + - name: infiniband + mountPath: /dev/infiniband - mountPath: /nfs name: model-storage tolerations: @@ -90,6 +93,11 @@ spec: - configMap: name: dgd-lpx-v2-vllm-cyborg-model-card name: cyborg-model-card + - name: infiniband + hostPath: + path: /dev/infiniband + type: Directory + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - name: agent podTemplate: spec: @@ -139,6 +147,18 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: hugepages, mountPath: /dev/hugepages} + - {name: host-sys, mountPath: /sys} + - {name: host-dev, mountPath: /dev} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} - name: prefill type: prefill replicas: 1 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 6734957ab721..d30e505bfcf6 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -360,20 +360,18 @@ spec: periodSeconds: 10 timeoutSeconds: 5 volumeMounts: + - mountPath: /configs + name: config - mountPath: /etc/cyborg/model-card.json name: cyborg-model-card readOnly: true subPath: model-card.json - - mountPath: /nfs - name: model-storage - mountPath: /dev/infiniband name: infiniband - - mountPath: /configs - name: config + - mountPath: /nfs + name: model-storage - mountPath: /dev/shm name: shared-memory - - mountPath: /tmp - name: tmp restartPolicy: Always schedulerName: default-scheduler securityContext: @@ -387,22 +385,20 @@ spec: - configMap: name: dgd-lpx-v2-vllm-cyborg-model-card name: cyborg-model-card - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - hostPath: path: /dev/infiniband type: Directory name: infiniband - - configMap: - name: lstg3y7d-decode-cd1df68308eaf508 - name: config + - name: model-storage + persistentVolumeClaim: + claimName: model-storage - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - emptyDir: {} - name: tmp + - configMap: + name: lstg3y7d-decode-cd1df68308eaf508 + name: config replicas: 1 roleName: lpu-engine-gpu startsAfter: @@ -573,8 +569,6 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - mountPath: /dev/hugepages @@ -586,8 +580,8 @@ spec: - mountPath: /ssh-pk name: ssh-secret readOnly: true - - mountPath: /tmp - name: tmp + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -611,6 +605,21 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: + medium: HugePages + name: hugepages + - hostPath: + path: /sys + type: Directory + name: host-sys + - hostPath: + path: /dev + type: Directory + name: host-dev + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret - emptyDir: medium: Memory sizeLimit: 8Gi @@ -618,23 +627,6 @@ spec: - configMap: name: lstg3y7d-lpu-70f8ece55681fe1b name: config - - emptyDir: {} - name: tmp - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - emptyDir: - medium: HugePages - name: hugepages replicas: 17 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index 26b02660fe76..dc7f74d22934 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -37,9 +37,16 @@ spec: mountPath: /configs - name: infiniband mountPath: /dev/infiniband + - {name: model-storage, mountPath: /nfs} resources: claims: - name: gpu + volumes: + - name: infiniband + hostPath: + path: /dev/infiniband + type: Directory + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - name: agent podTemplate: spec: @@ -56,7 +63,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: hugepages, mountPath: /dev/hugepages} + - {name: host-sys, mountPath: /sys} + - {name: host-dev, mountPath: /dev} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index c3e8a6b3256d..6bc48beefeb1 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -209,12 +209,10 @@ spec: name: config - mountPath: /dev/infiniband name: infiniband - - mountPath: /dev/shm - name: shared-memory - mountPath: /nfs name: model-storage - - mountPath: /tmp - name: tmp + - mountPath: /dev/shm + name: shared-memory resourceClaims: - name: gpu resourceClaimTemplateName: decode-gpu @@ -225,22 +223,20 @@ spec: fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 volumes: - - configMap: - name: ltudmjcv-decode-97936407dc305636 - name: config - hostPath: path: /dev/infiniband type: Directory name: infiniband + - name: model-storage + persistentVolumeClaim: + claimName: model-storage - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - emptyDir: {} - name: tmp + - configMap: + name: ltudmjcv-decode-97936407dc305636 + name: config replicas: 1 roleName: lpu-engine-gpu startsAfter: @@ -362,8 +358,6 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - mountPath: /dev/hugepages @@ -375,8 +369,8 @@ spec: - mountPath: /ssh-pk name: ssh-secret readOnly: true - - mountPath: /tmp - name: tmp + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -400,6 +394,21 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: + medium: HugePages + name: hugepages + - hostPath: + path: /sys + type: Directory + name: host-sys + - hostPath: + path: /dev + type: Directory + name: host-dev + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret - emptyDir: medium: Memory sizeLimit: 8Gi @@ -407,23 +416,6 @@ spec: - configMap: name: ltudmjcv-lpu-6bebe5759af346e9 name: config - - emptyDir: {} - name: tmp - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - emptyDir: - medium: HugePages - name: hugepages replicas: 4 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 6395ae8aa487..77033300c345 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -18,6 +18,24 @@ spec: lpx: buildId: node-local-v2-connected roles: + - name: conductor + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + command: + - /opt/dynamo-lpu + args: + - serve + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -34,7 +52,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 4459148b4d01..45615b0fe402 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -209,14 +209,12 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory initContainers: - args: - -c @@ -247,21 +245,19 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - configMap: - name: lgoyrwtj-lpu-62ffb8651b497e5f - name: config - - emptyDir: {} - name: tmp - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret - - emptyDir: {} - name: single-v2-ssh-key + - configMap: + name: lgoyrwtj-lpu-62ffb8651b497e5f + name: config replicas: 1 roleName: lpu-ldr startsAfter: @@ -355,21 +351,19 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true - mountPath: /dev/hugepages name: hugepages + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -390,15 +384,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lgoyrwtj-lpu-62ffb8651b497e5f - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -414,6 +399,13 @@ spec: - emptyDir: medium: HugePages name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lgoyrwtj-lpu-62ffb8651b497e5f + name: config replicas: 4 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index 96cae27b567c..54851b187fb2 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -30,10 +30,20 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} - name: lpu type: lpx replicas: 1 @@ -41,6 +51,19 @@ spec: buildId: node-local-v2-connected roles: - name: conductor + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -53,7 +76,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 92ba3c6a06e5..3af8e2dcb5ef 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -277,14 +277,12 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory initContainers: - args: - -c @@ -315,21 +313,19 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - configMap: - name: lmppuelh-lpu-29156c984c942fc4 - name: config - - emptyDir: {} - name: tmp - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret - - emptyDir: {} - name: single-v2-ssh-key + - configMap: + name: lmppuelh-lpu-29156c984c942fc4 + name: config replicas: 1 roleName: lpu-ldr startsAfter: @@ -507,21 +503,19 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true - mountPath: /dev/hugepages name: hugepages + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -542,15 +536,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lmppuelh-lpu-29156c984c942fc4 - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -566,6 +551,13 @@ spec: - emptyDir: medium: HugePages name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lmppuelh-lpu-29156c984c942fc4 + name: config replicas: 4 roleName: lpu-wkr-m-0 - annotations: @@ -739,21 +731,19 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true - mountPath: /dev/hugepages name: hugepages + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -774,15 +764,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lmppuelh-lpu-29156c984c942fc4 - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -798,6 +779,13 @@ spec: - emptyDir: medium: HugePages name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lmppuelh-lpu-29156c984c942fc4 + name: config replicas: 4 roleName: lpu-wkr-m-1 - annotations: @@ -971,21 +959,19 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true - mountPath: /dev/hugepages name: hugepages + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -1006,15 +992,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lmppuelh-lpu-29156c984c942fc4 - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -1030,6 +1007,13 @@ spec: - emptyDir: medium: HugePages name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lmppuelh-lpu-29156c984c942fc4 + name: config replicas: 4 roleName: lpu-wkr-m-2 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index 94f9d94e4a01..b3574c680630 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -32,9 +32,13 @@ spec: - /opt/gpu-runtime args: - serve + volumeMounts: + - {name: model-storage, mountPath: /nfs} resources: claims: - name: gpu + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - name: agent podTemplate: spec: @@ -50,7 +54,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: hugepages, mountPath: /dev/hugepages} + - {name: host-sys, mountPath: /sys} + - {name: host-dev, mountPath: /dev} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 1f929dab902b..c2bdff1e529d 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -180,10 +180,10 @@ spec: periodSeconds: 10 timeoutSeconds: 5 volumeMounts: - - mountPath: /dev/shm - name: shared-memory - mountPath: /nfs name: model-storage + - mountPath: /dev/shm + name: shared-memory resourceClaims: - name: gpu resourceClaimTemplateName: decode-gpu @@ -194,13 +194,13 @@ spec: fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - name: model-storage - persistentVolumeClaim: - claimName: model-storage replicas: 1 roleName: lpu-engine-gpu startsAfter: @@ -516,8 +516,6 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - mountPath: /dev/hugepages @@ -529,8 +527,8 @@ spec: - mountPath: /ssh-pk name: ssh-secret readOnly: true - - mountPath: /tmp - name: tmp + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -554,6 +552,21 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: + medium: HugePages + name: hugepages + - hostPath: + path: /sys + type: Directory + name: host-sys + - hostPath: + path: /dev + type: Directory + name: host-dev + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret - emptyDir: medium: Memory sizeLimit: 8Gi @@ -561,23 +574,6 @@ spec: - configMap: name: lxymegpm-lpu-40e022e2d45905fb name: config - - emptyDir: {} - name: tmp - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - emptyDir: - medium: HugePages - name: hugepages replicas: 1 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index 8593af620cdc..c470c2eaf7eb 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -18,6 +18,23 @@ spec: lpx: buildId: node-local-v3-hx-connected roles: + - name: conductor + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/llama32-1b-lp30 + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -33,7 +50,15 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 156e82ce76d5..b29e0ab30324 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -240,14 +240,12 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory initContainers: - args: - -c @@ -278,21 +276,19 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - configMap: - name: l2q6k2r6-lpu-68b42744ed35823d - name: config - - emptyDir: {} - name: tmp - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret - - emptyDir: {} - name: single-v2-ssh-key + - configMap: + name: l2q6k2r6-lpu-68b42744ed35823d + name: config replicas: 1 roleName: lpu-ldr startsAfter: @@ -451,19 +447,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -505,15 +499,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: l2q6k2r6-lpu-68b42744ed35823d - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -526,6 +511,13 @@ spec: secret: defaultMode: 420 secretName: ssh-secret + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: l2q6k2r6-lpu-68b42744ed35823d + name: config replicas: 1 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 7b3e4306421b..b0dcd7136e32 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -33,10 +33,18 @@ spec: volumeMounts: - name: model-storage mountPath: /nfs + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} - name: lpu type: lpx replicas: 1 @@ -44,6 +52,22 @@ spec: buildId: node-local-v3-hx-sd-target roles: - name: conductor + podTemplate: + spec: + containers: + - name: main + image: example.invalid/lpu-runtime:test + args: + - --instance-model-name + - test/v3-specdecode + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -59,7 +83,15 @@ spec: volumeMounts: - name: model-storage mountPath: /nfs + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index cb1081964b50..2814dca51585 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -296,14 +296,12 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory initContainers: - args: - -c @@ -334,21 +332,19 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc - name: config - - emptyDir: {} - name: tmp - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret - - emptyDir: {} - name: single-v2-ssh-key + - configMap: + name: ld55picp-lpu-3c4a4d4af295f6cc + name: config replicas: 1 roleName: lpu-ldr startsAfter: @@ -511,19 +507,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -565,15 +559,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -586,6 +571,13 @@ spec: secret: defaultMode: 420 secretName: ssh-secret + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: ld55picp-lpu-3c4a4d4af295f6cc + name: config replicas: 1 roleName: lpu-wkr-m-0 - annotations: @@ -744,19 +736,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -798,15 +788,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -819,6 +800,13 @@ spec: secret: defaultMode: 420 secretName: ssh-secret + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: ld55picp-lpu-3c4a4d4af295f6cc + name: config replicas: 1 roleName: lpu-wkr-m-1 - annotations: @@ -977,19 +965,17 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -1031,15 +1017,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -1052,6 +1029,13 @@ spec: secret: defaultMode: 420 secretName: ssh-secret + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: ld55picp-lpu-3c4a4d4af295f6cc + name: config replicas: 1 roleName: lpu-wkr-m-2 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml index f33a56b3dfda..5fa55fbc25cc 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml @@ -14,7 +14,6 @@ spec: components: - name: lpu type: lpx - extraPodSpecMergeStrategy: strategic replicas: 1 lpx: buildId: llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c @@ -31,6 +30,27 @@ spec: vocab_size: 128256 roles: - name: conductor + podTemplate: + spec: + containers: + - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: main + args: + - --agent-env-vars + - EXTRA_RUNTIME_KNOB + env: + - name: MPI_BTL_TCP_IF_INCLUDE + value: 7.241.144.0/24 + - name: OMPI_MCA_btl_tcp_if_include + value: 7.241.144.0/24 + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -48,3 +68,15 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index e9620c4336d0..538174d8a3b4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -290,14 +290,12 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory initContainers: - args: - -c @@ -328,21 +326,19 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - configMap: - name: lhkmp4xg-lpu-1575485f621480a4 - name: config - - emptyDir: {} - name: tmp - name: ssh-secret secret: defaultMode: 384 secretName: ssh-secret - - emptyDir: {} - name: single-v2-ssh-key + - configMap: + name: lhkmp4xg-lpu-1575485f621480a4 + name: config replicas: 1 roleName: lpu-ldr startsAfter: @@ -520,21 +516,19 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage - - mountPath: /dev/shm - name: shared-memory - mountPath: /configs name: config - - mountPath: /tmp - name: tmp - - mountPath: /sys - name: host-sys - mountPath: /dev name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true - mountPath: /dev/hugepages name: hugepages + - mountPath: /dev/shm + name: shared-memory dnsPolicy: ClusterFirstWithHostNet hostIPC: true hostNetwork: true @@ -555,15 +549,6 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lhkmp4xg-lpu-1575485f621480a4 - name: config - - emptyDir: {} - name: tmp - hostPath: path: /dev type: Directory @@ -579,6 +564,13 @@ spec: - emptyDir: medium: HugePages name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lhkmp4xg-lpu-1575485f621480a4 + name: config replicas: 17 roleName: lpu-wkr-m-0 headlessServiceConfig: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml index 601e82652695..7efbfe2940fb 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml @@ -14,12 +14,24 @@ spec: components: - name: lpx type: lpx - extraPodSpecMergeStrategy: strategic replicas: 1 lpx: buildId: single-v2-manifest-defaults/build_manifest_defaults roles: - name: conductor + podTemplate: + spec: + containers: + - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: main + volumeMounts: + - {name: model-storage, mountPath: /nfs} + - {name: config, mountPath: /configs} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} + - {name: single-v2-ssh-key, emptyDir: {}} - name: agent podTemplate: spec: @@ -29,3 +41,15 @@ spec: volumeMounts: - mountPath: /nfs name: model-storage + - name: config + mountPath: /configs + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} + volumes: + - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index fbe070a209d2..abe8b7b8a3c8 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -257,7 +257,7 @@ vLLM and SGLang take tensor/pipeline/data parallelism as CLI flags (covered in t You normally do not need to set the top-level `spec.backendFramework` field — the operator infers the backend from the worker command. Set it explicitly (`vllm`, `sglang`, or `trtllm`) only when a feature needs the framework known up front, such as GMS failover or multinode TensorRT-LLM. -For LPX on LPU hardware, declare the build and runtime roles in a `type: lpx` component: +For an XT LPU-only deployment, declare separate Agent and Nova conductor templates in a `type: lpx` component: ```yaml spec: @@ -274,15 +274,68 @@ spec: - name: main image: volumeMounts: + - name: config + mountPath: /configs + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: hugepages + mountPath: /dev/hugepages + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true - name: model-storage mountPath: /models volumes: + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: mpi-run-ssh-secret + defaultMode: 0644 + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: conductor + podTemplate: + spec: + containers: + - name: main + image: + volumeMounts: + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh + - name: model-storage + mountPath: /models + volumes: + - name: ssh-secret + secret: + secretName: mpi-run-ssh-secret + defaultMode: 0600 + - name: single-v2-ssh-key + emptyDir: {} - name: model-storage persistentVolumeClaim: claimName: model-storage ``` -Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with a build available through the configured LPX model registry and `` with its compatible runtime image. The `agent` role configures Agent Pods; the optional `conductor` role configures the conductor. With one LPX component, the conductor reuses the agent template unless the conductor supplies a separate `podTemplate`. For a hybrid engine, configure the hybrid runtime template on the conductor role. +Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with an XT LPU-only build available through the configured LPX model registry and `` with its compatible runtime image. + +Declare static volumes and every LPX runtime container mount in its role's `podTemplate`, including the conductor's SSH source and destination volumes. Use the configured MPI SSH Secret name; `mpi-run-ssh-secret` is the chart default. The operator generates the `config` volume with its content-based ConfigMap name and init containers with their own mounts. Keep the `hugepages` volume and mount for XT Agents and HX hybrid Agents; omit both for HX LPU-only and speculative-decoding Agents. For a hybrid engine, configure the hybrid runtime's template, volumes, and mounts on the conductor role. + +Runtime configuration and scratch files use the container's writable `/tmp`; a separate volume is optional. For a read-only root filesystem, mount a writable `emptyDir` at `/tmp`. The conductor's separate `/tmp/dynamo-lpu-ssh` mount shares its prepared key with the init container and is still required. For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only an `agent` role; the target has `conductor` and `agent` roles. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 6f13b531cded..20548120435a 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -333,7 +333,6 @@ _Appears in:_ | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| | `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| -| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the
operator-generated pod and container defaults. | | Enum: [override strategic]
Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| @@ -381,7 +380,6 @@ _Appears in:_ | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| | `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| -| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the
operator-generated pod and container defaults. | | Enum: [override strategic]
Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| @@ -779,26 +777,6 @@ _Appears in:_ | `mainContainer` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#container-v1-core)_ | | | | -#### ExtraPodSpecMergeStrategy - -_Underlying type:_ _string_ - - - -_Validation:_ -- Enum: [override strategic] - -_Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec) -- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) -- [PodGenerationConfiguration](#podgenerationconfiguration) - -| Field | Description | -| --- | --- | -| `override` | | -| `strategic` | | - - #### FailoverSpec @@ -1944,10 +1922,9 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides according to -`extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and -pod-level configuration directly in `podTemplate` without any -`extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides. Users can add sidecars, +init containers, and pod-level configuration directly in `podTemplate` +without any `extraPodSpec`-style escape hatch. @@ -1962,7 +1939,6 @@ _Appears in:_ | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| -| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | extraPodSpecMergeStrategy controls how podTemplate is merged with the
operator-generated pod and container defaults. When omitted, the operator
uses its configured defaultExtraPodSpecMergeStrategy, which defaults to
"override". | | Enum: [override strategic]
Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| @@ -1997,7 +1973,6 @@ _Appears in:_ | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| -| `extraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | extraPodSpecMergeStrategy controls how podTemplate is merged with the
operator-generated pod and container defaults. When omitted, the operator
uses its configured defaultExtraPodSpecMergeStrategy, which defaults to
"override". | | Enum: [override strategic]
Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| @@ -2340,25 +2315,6 @@ _Appears in:_ | `checkpoint` _[ComponentCheckpointConfig](#componentcheckpointconfig)_ | checkpoint configures container-image snapshotting and restore for
this component. Set `checkpoint.enabled: true` to opt in. Without
checkpointRef, the DGD controller creates a DGD-scoped SnapshotJob and
later restores pods in the same DGD generation from its PodSnapshot.
With checkpointRef, the DGD restores from the named
PodSnapshot in the same namespace. The user-facing shape of this field is
still settling, which is why it lives under `experimental` in v1beta1
instead of at the top level. | | Optional: \{\}
| -#### ExtraPodSpecMergeStrategy - -_Underlying type:_ _string_ - - - -_Validation:_ -- Enum: [override strategic] - -_Appears in:_ -- [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec) -- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec) - -| Field | Description | -| --- | --- | -| `override` | | -| `strategic` | | - - #### FailoverSpec @@ -3593,7 +3549,6 @@ OperatorConfiguration is the Schema for the operator configuration. | `rbac` _[RBACConfiguration](#rbacconfiguration)_ | RBAC configuration for cross-namespace resource management (cluster-wide mode) | | | | `mpi` _[MPIConfiguration](#mpiconfiguration)_ | MPI SSH secret configuration | | | | `checkpoint` _[CheckpointConfiguration](#checkpointconfiguration)_ | Checkpoint/restore configuration | | | -| `podGeneration` _[PodGenerationConfiguration](#podgenerationconfiguration)_ | Pod generation defaults | | | | `discovery` _[DiscoveryConfiguration](#discoveryconfiguration)_ | Discovery backend configuration | | | | `gpu` _[GPUConfiguration](#gpuconfiguration)_ | GPU discovery configuration | | | | `logging` _[LoggingConfiguration](#loggingconfiguration)_ | Logging configuration | | | @@ -3620,22 +3575,6 @@ _Appears in:_ | `volcanoScheduler` _[VolcanoSchedulerConfiguration](#volcanoschedulerconfiguration)_ | VolcanoScheduler configuration | | | -#### PodGenerationConfiguration - - - -PodGenerationConfiguration holds pod generation defaults applied by the operator. - - - -_Appears in:_ -- [OperatorConfiguration](#operatorconfiguration) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `defaultExtraPodSpecMergeStrategy` _[ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy)_ | DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a
component leaves spec.extraPodSpecMergeStrategy unset. | override | Enum: [override strategic]
| - - #### RBACConfiguration diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 6c7c06da01d9..2e3474a9518c 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,12 +10,12 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 3 Kubernetes API packages with 178 typed CRD and config sections. +Dynamo publishes 3 Kubernetes API packages with 175 typed CRD and config sections. -72 types -76 types -30 types +71 types +75 types +29 types
@@ -445,12 +445,6 @@ See [ExtraPodSpec](#extrapodspec). **Validation:** Optional: \{\} - -ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the operator-generated pod and container defaults. -See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). -**Validation:** Enum: [override strategic] Optional: \{\} - - LivenessProbe to detect and restart unhealthy containers. See [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core). @@ -624,12 +618,6 @@ See [ExtraPodSpec](#extrapodspec). **Validation:** Optional: \{\} - -ExtraPodSpecMergeStrategy controls how extraPodSpec is merged with the operator-generated pod and container defaults. -See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). -**Validation:** Enum: [override strategic] Optional: \{\} - - LivenessProbe to detect and restart unhealthy containers. See [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core). @@ -1283,19 +1271,6 @@ See [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1 - - -**Kind:** `enum` -**Underlying type:** `string` -**Validation:** Enum: [override strategic] -**Appears in:** [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec), [PodGenerationConfiguration](#podgenerationconfiguration) - -**Allowed values** - -- override -- strategic - - FailoverSpec configures active-passive failover for a worker component. For intraPod mode: requires gpuMemoryService.enabled; the main container is cloned @@ -2711,10 +2686,9 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides according to -`extraPodSpecMergeStrategy`. Users can add sidecars, init containers, and -pod-level configuration directly in `podTemplate` without any -`extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides. Users can add sidecars, +init containers, and pod-level configuration directly in `podTemplate` +without any `extraPodSpec`-style escape hatch. **Kind:** `type` @@ -2753,12 +2727,6 @@ See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes- **Validation:** Optional: \{\} - -extraPodSpecMergeStrategy controls how podTemplate is merged with the operator-generated pod and container defaults. When omitted, the operator uses its configured defaultExtraPodSpecMergeStrategy, which defaults to "override". -See [ExtraPodSpecMergeStrategy](#v1beta1-extrapodspecmergestrategy). -**Validation:** Enum: [override strategic] Optional: \{\} - - replicas is the desired number of Pods for this component. When `scalingAdapter` is set on this component, this field is managed by the DynamoGraphDeploymentScalingAdapter and should not be modified directly. **Validation:** Minimum: 0 Optional: \{\} @@ -2874,12 +2842,6 @@ See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes- **Validation:** Optional: \{\} - -extraPodSpecMergeStrategy controls how podTemplate is merged with the operator-generated pod and container defaults. When omitted, the operator uses its configured defaultExtraPodSpecMergeStrategy, which defaults to "override". -See [ExtraPodSpecMergeStrategy](#v1beta1-extrapodspecmergestrategy). -**Validation:** Enum: [override strategic] Optional: \{\} - - replicas is the desired number of Pods for this component. When `scalingAdapter` is set on this component, this field is managed by the DynamoGraphDeploymentScalingAdapter and should not be modified directly. **Validation:** Minimum: 0 Optional: \{\} @@ -3489,19 +3451,6 @@ See [ComponentCheckpointConfig](#componentcheckpointconfig). - - -**Kind:** `enum` -**Underlying type:** `string` -**Validation:** Enum: [override strategic] -**Appears in:** [DynamoComponentDeploymentSharedSpec](#v1beta1-dynamocomponentdeploymentsharedspec), [DynamoComponentDeploymentSpec](#v1beta1-dynamocomponentdeploymentspec) - -**Allowed values** - -- override -- strategic - - FailoverSpec configures active-passive failover for a worker component. The main container is cloned into two engine containers (active + standby) @@ -4873,11 +4822,6 @@ Checkpoint/restore configuration See [CheckpointConfiguration](#checkpointconfiguration). - -Pod generation defaults -See [PodGenerationConfiguration](#podgenerationconfiguration). - - Discovery backend configuration See [DiscoveryConfiguration](#discoveryconfiguration). @@ -4932,20 +4876,6 @@ See [VolcanoSchedulerConfiguration](#volcanoschedulerconfiguration). - -PodGenerationConfiguration holds pod generation defaults applied by the operator. - - -**Kind:** `type` -**Appears in:** [OperatorConfiguration](#operatorconfiguration) - - -DefaultExtraPodSpecMergeStrategy is the operator-wide fallback when a component leaves spec.extraPodSpecMergeStrategy unset. -See [ExtraPodSpecMergeStrategy](#extrapodspecmergestrategy). -**Validation:** Enum: [override strategic] - - - RBACConfiguration holds RBAC settings for cluster-wide mode. From fa8077dd0db5cafaa9929fb56564ba25741ea2b4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 10:35:49 +0000 Subject: [PATCH 053/196] docs(helm): clarify LPX integration enablement Signed-off-by: Andrew Paprotsky (cherry picked from commit 85f25a606e75605c78a6dc2017cf7b3f2705efe6) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 2 +- .../helm/charts/platform/components/operator/values.yaml | 5 ++--- deploy/helm/charts/platform/values.yaml | 9 ++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 9e9c58262bd2..e66df58c8186 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -170,7 +170,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | -| dynamo-operator.lpx.enabled | bool | `false` | Enable LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs. | +| dynamo-operator.lpx.enabled | bool | `false` | Enable the Dynamo operator's LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs. | | dynamo-operator.lpx.modelRegistryURL | string | `""` | Configure the LPU model registry used by the LPX integration. | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. | diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 498d7c411eb4..4e2abe80c203 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -22,10 +22,9 @@ # the cluster-wide operator. upgradeCRD: true -# LPX is an externally installed scheduler integration. Enabling it only -# configures the Dynamo operator; this chart never installs LPX resources. +# The LPX scheduler and its CRDs must be installed separately. lpx: - # -- Enable node-local LpuPipelineRequest production. Startup requires the + # -- Enable the Dynamo operator's LPX integration. Startup requires the # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest # and scheduler.grove.io/v1alpha1 PodGang APIs. enabled: false diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 747e89bfae4a..52c4f312606e 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -91,12 +91,11 @@ dynamo-operator: # -- URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). modelExpressURL: "" - # LPX is installed and operated separately. These values only opt the - # Dynamo operator into producing node-local scheduling requests. + # The LPX scheduler and its CRDs must be installed separately. lpx: - # -- Enable LPX integration. Startup requires the externally installed - # scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and - # scheduler.grove.io/v1alpha1 PodGang APIs. + # -- Enable the Dynamo operator's LPX integration. Startup requires the + # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest + # and scheduler.grove.io/v1alpha1 PodGang APIs. enabled: false # -- Configure the LPU model registry used by the LPX integration. modelRegistryURL: "" From 57e2b36f7ce6364e4e667d364a225db4f6ca4ce5 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 10:35:19 +0000 Subject: [PATCH 054/196] chore(helm): remove redundant test exclusion comment Signed-off-by: Andrew Paprotsky (cherry picked from commit 2b64a9c5a85b744e89e9036c6bc10c2c328b4578) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/components/operator/.helmignore | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/helm/charts/platform/components/operator/.helmignore b/deploy/helm/charts/platform/components/operator/.helmignore index fea6ca5f742d..faeb926b1a07 100644 --- a/deploy/helm/charts/platform/components/operator/.helmignore +++ b/deploy/helm/charts/platform/components/operator/.helmignore @@ -21,5 +21,4 @@ .idea/ *.tmproj .vscode/ -# test files tests/ From 89be03dd471022da00ce9088e45d45dc155f9520 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 10:57:20 +0000 Subject: [PATCH 055/196] refactor(operator): align role PodTemplate with upstream Signed-off-by: Andrew Paprotsky (cherry picked from commit 518b2454432807085f2f2464603a6f1b2e85a60a) Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1beta1/common.go | 11 ++++++----- deploy/operator/api/v1beta1/zz_generated.deepcopy.go | 10 +++++----- .../bases/nvidia.com_dynamocomponentdeployments.yaml | 5 +++-- .../crd/bases/nvidia.com_dynamographdeployments.yaml | 5 +++-- .../additional-resources/api-reference-k8s.md | 2 +- .../reference/kubernetes-api/full-api-reference.mdx | 12 ++++++------ 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index 4a3294e48ef6..46d9d8090cf6 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -134,15 +134,16 @@ type ComponentRoleSpec struct { // +kubebuilder:validation:Minimum=1 Replicas *int32 `json:"replicas,omitempty"` - // podTemplate configures Pods for this role. The enclosing component - // defines whether the template is required or may be inherited. - // +optional - PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` - // providerOverride configures the provider workload unit generated for this // role. It is supported only for components embedded in a DGD. // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` + + // podTemplate defines the Pod configuration for this role. Admission permits + // it only when the enclosing component type explicitly supports role-specific + // Pod templates. + // +optional + PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } // ComponentRole returns the authored role, or nil when the component does not declare it. diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 51edcf67ce3c..81e3a38f05d9 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -184,16 +184,16 @@ func (in *ComponentRoleSpec) DeepCopyInto(out *ComponentRoleSpec) { *out = new(int32) **out = **in } - if in.PodTemplate != nil { - in, out := &in.PodTemplate, &out.PodTemplate - *out = new(v1.PodTemplateSpec) - (*in).DeepCopyInto(*out) - } if in.ProviderOverride != nil { in, out := &in.ProviderOverride, &out.ProviderOverride *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(v1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentRoleSpec. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 959abf0c6d87..92a328b0a75f 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -28676,8 +28676,9 @@ spec: type: string podTemplate: description: |- - podTemplate configures Pods for this role. The enclosing component - defines whether the template is required or may be inherited. + podTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. properties: metadata: description: |- diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 01f87244d468..676cf89f8b1c 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -29441,8 +29441,9 @@ spec: type: string podTemplate: description: |- - podTemplate configures Pods for this role. The enclosing component - defines whether the template is required or may be inherited. + podTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. properties: metadata: description: |- diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 20548120435a..e38fa4769775 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -1772,8 +1772,8 @@ _Appears in:_ | --- | --- | --- | --- | | `name` _string_ | name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
multinode.nodeCount; leader must be 1 and worker must be
multinode.nodeCount minus 1. | | Minimum: 1
Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate configures Pods for this role. The enclosing component
defines whether the template is required or may be inherited. | | Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. | | Optional: \{\}
| #### ComponentType diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 2e3474a9518c..4f325af77f93 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -2507,17 +2507,17 @@ replicas is the logical cardinality of this role in one complete component insta **Validation:** Minimum: 1 Optional: \{\} - -podTemplate configures Pods for this role. The enclosing component defines whether the template is required or may be inherited. -See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). -**Validation:** Optional: \{\} - - providerOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. See [ProviderOverride](#v1beta1-provideroverride). **Validation:** Optional: \{\} + + +podTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. +See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). +**Validation:** Optional: \{\} + From 6f712fd755298b16a6ef756011f533a09650522b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 19:28:45 +0000 Subject: [PATCH 056/196] refactor(operator)!: clean up legacy LPU API compatibility Cherry-pick the merged PR #490 delta into the private upstream LPX staging branch. (cherry picked from commit e924d8eb493ad1a6ec853c20e5a81bd712905a45) Signed-off-by: Andrew Paprotsky --- deploy/operator/api/roundtrip_fuzz_test.go | 11 ++++- deploy/operator/api/v1alpha1/common.go | 15 ++++--- .../dynamocomponentdeployment_types.go | 1 - .../v1alpha1/dynamographdeployment_types.go | 8 ++-- .../api/v1alpha1/zz_generated.deepcopy.go | 20 ++++----- deploy/operator/api/v1beta1/common.go | 4 +- .../dynamocomponentdeployment_types.go | 2 +- .../v1beta1/dynamographdeployment_types.go | 10 ++--- .../api/v1beta1/zz_generated.deepcopy.go | 10 ++--- ...nvidia.com_dynamocomponentdeployments.yaml | 15 +++---- .../nvidia.com_dynamographdeployments.yaml | 13 +++--- ...onentdeployment_validation_envtest_test.go | 33 +++++++++++--- .../additional-resources/api-reference-k8s.md | 10 ++--- .../dynamo-component-deployment.mdx | 2 +- .../kubernetes-api/full-api-reference.mdx | 44 +++++++++---------- 15 files changed, 110 insertions(+), 88 deletions(-) diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index fa1050ecab67..c096459fe8c5 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -182,7 +182,7 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { fuzzBetaDGDRSpec, fuzzBetaDGDRStatus, fuzzBetaDGDROverrides, - // Shared PlacementStatus: pick admissible values so the + // PlacementStatus (v1alpha1 + v1beta1): pick admissible values so the // round-trip fuzzer exercises Placement without producing shapes the CRD // schema rejects. State draws from the enum; Score draws either nil or a // value inside the [0, 1] bounds enforced by the CRD. @@ -195,6 +195,15 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { ) p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) }, + func(p *v1alpha1.PlacementStatus, c randfill.Continue) { + p.State = oneOf(c, + v1alpha1.PlacementScoreStateReported, + v1alpha1.PlacementScoreStatePartial, + v1alpha1.PlacementScoreStateUnsupported, + v1alpha1.PlacementScoreStateUnknown, + ) + p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) + }, // v1beta1 Components: the listMapKey marker requires name // to be non-empty and unique; MaxItems caps the length at 25. // Enforce both so the input is admissible. diff --git a/deploy/operator/api/v1alpha1/common.go b/deploy/operator/api/v1alpha1/common.go index 143f2e6f49fa..c8d31e911e62 100644 --- a/deploy/operator/api/v1alpha1/common.go +++ b/deploy/operator/api/v1alpha1/common.go @@ -89,15 +89,16 @@ type ComponentRoleSpec struct { // +kubebuilder:validation:Minimum=1 Replicas *int32 `json:"replicas,omitempty"` - // PodTemplate configures Pods for this role. The enclosing component - // defines whether the template is required or may be inherited. - // +optional - PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` - // ProviderOverride configures the provider workload unit generated for this // role. It is supported only for components embedded in a DGD. // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` + + // PodTemplate defines the Pod configuration for this role. Admission permits + // it only when the enclosing component type explicitly supports role-specific + // Pod templates. + // +optional + PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } // +kubebuilder:validation:XValidation:rule="!has(self.create) || self.create == false || (has(self.size) && has(self.storageClass) && has(self.volumeAccessMode))",message="When create is true, size, storageClass, and volumeAccessMode are required" @@ -475,10 +476,10 @@ type DynamoCheckpointIdentity struct { // +kubebuilder:validation:Required Model string `json:"model"` - // BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). + // BackendFramework is the runtime framework (vllm, sglang, trtllm). // Deprecated: legacy identity only. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=vllm;sglang;trtllm;lpu + // +kubebuilder:validation:Enum=vllm;sglang;trtllm BackendFramework string `json:"backendFramework"` // DynamoVersion is the Dynamo platform version. diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 04c3c9fc46eb..6dde61ec9dbb 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -54,7 +54,6 @@ type DynamoComponentDeploymentSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)",message="componentType is immutable after it is set" -// +kubebuilder:validation:XValidation:rule="!has(self.componentType) || self.componentType != 'lpu'",message="componentType lpu is not supported; use lpx" // +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.componentType) && self.componentType == 'lpx')",message="lpx may only be set when componentType is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx') || has(self.lpx)",message="lpx is required when componentType is lpx" // +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx' && has(self.replicas) && self.replicas < 1)",message="replicas must be positive when componentType is lpx" diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 669d8dbe1f90..4f228b00aafb 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -228,6 +228,10 @@ type DynamoGraphDeploymentStatus struct { // The map key is the service name from spec.services. // +optional Checkpoints map[string]ServiceCheckpointStatus `json:"checkpoints,omitempty"` + // RollingUpdate tracks the progress of operator manged rolling updates. + // Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). + // +optional + RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` // Placement groups DGD-level scheduler placement signals (score, reporting // state, and any future placement fields). // +optional @@ -235,10 +239,6 @@ type DynamoGraphDeploymentStatus struct { // LPX contains the status of the graph's LPX workload, when present. // +optional LPX *v1beta1.DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` - // RollingUpdate tracks the progress of operator manged rolling updates. - // Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). - // +optional - RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` } // ServiceCheckpointStatus contains checkpoint information for a single service. diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 86ffabd6e81a..51f864cbeed6 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -124,16 +124,16 @@ func (in *ComponentRoleSpec) DeepCopyInto(out *ComponentRoleSpec) { *out = new(int32) **out = **in } - if in.PodTemplate != nil { - in, out := &in.PodTemplate, &out.PodTemplate - *out = new(v1.PodTemplateSpec) - (*in).DeepCopyInto(*out) - } if in.ProviderOverride != nil { in, out := &in.ProviderOverride, &out.ProviderOverride *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(v1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentRoleSpec. @@ -962,6 +962,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateStatus) + (*in).DeepCopyInto(*out) + } if in.Placement != nil { in, out := &in.Placement, &out.Placement *out = new(PlacementStatus) @@ -972,11 +977,6 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(v1beta1.DynamoGraphDeploymentLPXStatus) (*in).DeepCopyInto(*out) } - if in.RollingUpdate != nil { - in, out := &in.RollingUpdate, &out.RollingUpdate - *out = new(RollingUpdateStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index 46d9d8090cf6..ce98036c71b6 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -526,10 +526,10 @@ type DynamoCheckpointIdentity struct { // +kubebuilder:validation:MinLength=1 Model string `json:"model"` - // backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). + // backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). // Deprecated: legacy identity only. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=vllm;sglang;trtllm;lpu + // +kubebuilder:validation:Enum=vllm;sglang;trtllm BackendFramework string `json:"backendFramework"` // dynamoVersion is the Dynamo platform version. diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index b66520edf29c..b97458fb7eca 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -288,7 +288,7 @@ type DynamoComponentDeploymentStatus struct { // +kubebuilder:storageversion // +kubebuilder:resource:shortName=dcd // +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.conditions[?(@.type=='Available')].status",description="Available" -// +kubebuilder:printcolumn:name="Backend",type="string",JSONPath=`.spec.backendFramework`,description="GPU backend framework" +// +kubebuilder:printcolumn:name="Backend",type="string",JSONPath=`.spec.backendFramework`,description="Backend framework (sglang, vllm, trtllm)" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // DynamoComponentDeployment is the Schema for the dynamocomponentdeployments API. diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 5ada213f9a08..8053b05fd1a8 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -133,6 +133,11 @@ type DynamoGraphDeploymentStatus struct { // +optional Checkpoints map[string]ComponentCheckpointStatus `json:"checkpoints,omitempty"` + // rollingUpdate tracks the progress of operator-managed rolling updates. + // Currently only supported for single-node, non-Grove deployments (DCD/Deployment). + // +optional + RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` + // placement groups DGD-level scheduler placement signals (score, reporting // state, and any future placement fields). // +optional @@ -141,11 +146,6 @@ type DynamoGraphDeploymentStatus struct { // lpx contains the status of the graph's LPX workload, when present. // +optional LPX *DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` - - // rollingUpdate tracks the progress of operator-managed rolling updates. - // Currently only supported for single-node, non-Grove deployments (DCD/Deployment). - // +optional - RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` } // DGD Ready condition reasons used to classify Grove-backed not-ready diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 81e3a38f05d9..d2fc2c869b67 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -898,6 +898,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt (*out)[key] = val } } + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateStatus) + (*in).DeepCopyInto(*out) + } if in.Placement != nil { in, out := &in.Placement, &out.Placement *out = new(PlacementStatus) @@ -908,11 +913,6 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(DynamoGraphDeploymentLPXStatus) (*in).DeepCopyInto(*out) } - if in.RollingUpdate != nil { - in, out := &in.RollingUpdate, &out.RollingUpdate - *out = new(RollingUpdateStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 92a328b0a75f..faac5c4e2e58 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -713,13 +713,12 @@ spec: properties: backendFramework: description: |- - BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). + BackendFramework is the runtime framework (vllm, sglang, trtllm). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm - - lpu type: string dtype: description: |- @@ -11484,8 +11483,9 @@ spec: type: string podTemplate: description: |- - PodTemplate configures Pods for this role. The enclosing component - defines whether the template is required or may be inherited. + PodTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. properties: metadata: description: |- @@ -19717,8 +19717,6 @@ spec: rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set rule: '!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)' - - message: componentType lpu is not supported; use lpx - rule: '!has(self.componentType) || self.componentType != ''lpu''' - message: replicas must be positive when componentType is lpx rule: '!(has(self.componentType) && self.componentType == ''lpx'' && has(self.replicas) && self.replicas < 1)' - message: scalingAdapter is not supported when componentType is lpx @@ -19913,7 +19911,7 @@ spec: jsonPath: .status.conditions[?(@.type=='Available')].status name: Available type: string - - description: GPU backend framework + - description: Backend framework (sglang, vllm, trtllm) jsonPath: .spec.backendFramework name: Backend type: string @@ -20198,13 +20196,12 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). + backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm - - lpu type: string dtype: description: |- diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 676cf89f8b1c..4017c102bdfd 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -1054,13 +1054,12 @@ spec: properties: backendFramework: description: |- - BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). + BackendFramework is the runtime framework (vllm, sglang, trtllm). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm - - lpu type: string dtype: description: |- @@ -11874,8 +11873,9 @@ spec: type: string podTemplate: description: |- - PodTemplate configures Pods for this role. The enclosing component - defines whether the template is required or may be inherited. + PodTemplate defines the Pod configuration for this role. Admission permits + it only when the enclosing component type explicitly supports role-specific + Pod templates. properties: metadata: description: |- @@ -20134,8 +20134,6 @@ spec: rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set rule: '!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)' - - message: componentType lpu is not supported; use lpx - rule: '!has(self.componentType) || self.componentType != ''lpu''' - message: lpx may only be set when componentType is lpx rule: '!has(self.lpx) || (has(self.componentType) && self.componentType == ''lpx'')' - message: lpx is required when componentType is lpx @@ -20914,13 +20912,12 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). + backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). Deprecated: legacy identity only. enum: - vllm - sglang - trtllm - - lpu type: string dtype: description: |- diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go index e804d9c819b2..1ca72a8d3b68 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go @@ -166,13 +166,6 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { }), wantWebhookErrs: []string{"spec.roles[0].podTemplate: Forbidden: is not supported for this component role"}, }, - { - name: "v1alpha1 rejects the legacy lpu component type", - deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { - dcd.Spec.ComponentType = "lpu" - }), - wantCELErr: "spec: Invalid value: componentType lpu is not supported; use lpx", - }, { name: "standalone v1alpha1 canonical lpx component is rejected", deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { @@ -730,6 +723,32 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { }, }), }, + { + name: "v1alpha1 checkpoint identity rejects lpu backend", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.Checkpoint = &nvidiacomv1alpha1.ServiceCheckpointConfig{ + Identity: &nvidiacomv1alpha1.DynamoCheckpointIdentity{ + Model: "model", + BackendFramework: "lpu", + }, + } + }), + wantSchemaErr: `spec.checkpoint.identity.backendFramework: Unsupported value: "lpu": supported values: "vllm", "sglang", "trtllm"`, + }, + { + name: "v1beta1 checkpoint identity rejects lpu backend", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.Experimental = &nvidiacomv1beta1.ExperimentalSpec{ + Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{ + Identity: &nvidiacomv1beta1.DynamoCheckpointIdentity{ + Model: "model", + BackendFramework: "lpu", + }, + }, + } + }), + wantSchemaErr: `spec.experimental.checkpoint.identity.backendFramework: Unsupported value: "lpu": supported values: "vllm", "sglang", "trtllm"`, + }, { name: "checkpoint target container name is validated by the source schema", deployment: alphaDCDWithSharedSpec(nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index e38fa4769775..7e52c8b15b0f 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -151,8 +151,8 @@ _Appears in:_ | --- | --- | --- | --- | | `name` _string_ | Name identifies the role within the enclosing component independently of
generated provider resource names. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
Required: \{\}
| | `replicas` _integer_ | Replicas is the logical cardinality of this role in one complete component
instance. The enclosing component type defines the cardinality. For
multinode components, admission defaults and persists omitted values from
Multinode.NodeCount; leader must be 1 and worker must be
Multinode.NodeCount minus 1. | | Minimum: 1
Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate configures Pods for this role. The enclosing component
defines whether the template is required or may be inherited. | | Optional: \{\}
| | `providerOverride` _[ProviderOverride](#provideroverride)_ | ProviderOverride configures the provider workload unit generated for this
role. It is supported only for components embedded in a DGD. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | PodTemplate defines the Pod configuration for this role. Admission permits
it only when the enclosing component type explicitly supports role-specific
Pod templates. | | Optional: \{\}
| #### ConfigMapKeySelector @@ -275,7 +275,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `model` _string_ | Model is the model identifier (e.g., "meta-llama/Llama-3-70B").
Deprecated: legacy identity only. | | Required: \{\}
| -| `backendFramework` _string_ | BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm lpu]
Required: \{\}
| +| `backendFramework` _string_ | BackendFramework is the runtime framework (vllm, sglang, trtllm).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| | `dynamoVersion` _string_ | DynamoVersion is the Dynamo platform version.
Deprecated: legacy identity only. | | Optional: \{\}
| | `tensorParallelSize` _integer_ | TensorParallelSize is the tensor parallel configuration.
Deprecated: automatic capture derives compatibility from the worker. | 1 | Minimum: 1
Optional: \{\}
| | `pipelineParallelSize` _integer_ | PipelineParallelSize is the pipeline parallel configuration.
Deprecated: automatic capture derives compatibility from the worker. | 1 | Minimum: 1
Optional: \{\}
| @@ -639,9 +639,9 @@ _Appears in:_ | `services` _object (keys:string, values:[ServiceReplicaStatus](#servicereplicastatus))_ | Services contains per-service replica status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| +| `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| | `placement` _[PlacementStatus](#placementstatus)_ | Placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| | `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | LPX contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| -| `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| #### DynamoModel @@ -1879,7 +1879,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `model` _string_ | model is the model identifier (e.g. "meta-llama/Llama-3-70B").
Deprecated: legacy identity only. | | MinLength: 1
Required: \{\}
| -| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm lpu]
Required: \{\}
| +| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| | `dynamoVersion` _string_ | dynamoVersion is the Dynamo platform version.
Deprecated: legacy identity only. | | Optional: \{\}
| | `tensorParallelSize` _integer_ | tensorParallelSize is the tensor parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| | `pipelineParallelSize` _integer_ | pipelineParallelSize is the pipeline parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| @@ -2261,9 +2261,9 @@ _Appears in:_ | `components` _object (keys:string, values:[ComponentReplicaStatus](#componentreplicastatus))_ | components contains per-component replica status information, keyed by component name. | | Optional: \{\}
| | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| +| `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| | `placement` _[PlacementStatus](#placementstatus)_ | placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| | `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | lpx contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| -| `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| #### EPPConfig diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index a1344273cd1d..1eeb9c07e82e 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -324,7 +324,7 @@ The upgrade does not delete the legacy `DynamoCheckpoint` CRD, instances, `PodSn Runtime framework. - Allowed values: vllm sglang trtllm lpu + Allowed values: vllm sglang trtllm Dynamo platform version. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 4f325af77f93..68703dcbfafc 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -154,17 +154,17 @@ Replicas is the logical cardinality of this role in one complete component insta **Validation:** Minimum: 1 Optional: \{\} - -PodTemplate configures Pods for this role. The enclosing component defines whether the template is required or may be inherited. -See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). -**Validation:** Optional: \{\} - - ProviderOverride configures the provider workload unit generated for this role. It is supported only for components embedded in a DGD. See [ProviderOverride](#provideroverride). **Validation:** Optional: \{\} + + +PodTemplate defines the Pod configuration for this role. Admission permits it only when the enclosing component type explicitly supports role-specific Pod templates. +See [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core). +**Validation:** Optional: \{\} + @@ -293,8 +293,8 @@ Model is the model identifier (e.g., "meta-llama/Llama-3-70B"). Deprecated: lega -BackendFramework is the runtime framework (vllm, sglang, trtllm, lpu). Deprecated: legacy identity only. -**Validation:** Enum: [vllm sglang trtllm lpu] Required: \{\} +BackendFramework is the runtime framework (vllm, sglang, trtllm). Deprecated: legacy identity only. +**Validation:** Enum: [vllm sglang trtllm] Required: \{\} @@ -1092,6 +1092,12 @@ See [ServiceCheckpointStatus](#servicecheckpointstatus). **Validation:** Optional: \{\} + +RollingUpdate tracks the progress of operator manged rolling updates. Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). +See [RollingUpdateStatus](#rollingupdatestatus). +**Validation:** Optional: \{\} + + Placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). See [PlacementStatus](#placementstatus). @@ -1103,12 +1109,6 @@ LPX contains the status of the graph's LPX workload, when present. See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). **Validation:** Optional: \{\} - - -RollingUpdate tracks the progress of operator manged rolling updates. Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). -See [RollingUpdateStatus](#rollingupdatestatus). -**Validation:** Optional: \{\} - @@ -2613,8 +2613,8 @@ model is the model identifier (e.g. "meta-llama/Llama-3-70B"). Deprecated: legac -backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`, or `lpu`). Deprecated: legacy identity only. -**Validation:** Enum: [vllm sglang trtllm lpu] Required: \{\} +backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). Deprecated: legacy identity only. +**Validation:** Enum: [vllm sglang trtllm] Required: \{\} @@ -3370,6 +3370,12 @@ See [ComponentCheckpointStatus](#componentcheckpointstatus). **Validation:** Optional: \{\} + +rollingUpdate tracks the progress of operator-managed rolling updates. Currently only supported for single-node, non-Grove deployments (DCD/Deployment). +See [RollingUpdateStatus](#v1beta1-rollingupdatestatus). +**Validation:** Optional: \{\} + + placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). See [PlacementStatus](#v1beta1-placementstatus). @@ -3381,12 +3387,6 @@ lpx contains the status of the graph's LPX workload, when present. See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). **Validation:** Optional: \{\} - - -rollingUpdate tracks the progress of operator-managed rolling updates. Currently only supported for single-node, non-Grove deployments (DCD/Deployment). -See [RollingUpdateStatus](#v1beta1-rollingupdatestatus). -**Validation:** Optional: \{\} - From df87346cec2db353dce8429d90430da97cbefc8d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 17:33:59 +0000 Subject: [PATCH 057/196] fix(operator): refresh Grove scheduling diagnostics Signed-off-by: Andrew Paprotsky (cherry picked from commit 065ea89b956788cca6c20aa935e7a07477e61532) --- .../controller/dynamographdeployment_controller_test.go | 6 ++++-- .../internal/controller_common/workload_predicates.go | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index ac8b785e76ed..33f78bd0330c 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -3868,6 +3868,7 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { }) } + t.Log("Wake reconciliation when only the PodClique scheduling message changes") oldPodClique := base() oldPodClique.Status.Conditions = []metav1.Condition{{ Type: groveconstants.ConditionTypePodCliqueScheduled, @@ -3877,7 +3878,7 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { }} newPodClique := oldPodClique.DeepCopy() newPodClique.Status.Conditions[0].Message = "two nodes unavailable" - assert.False(t, controller_common.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique)) + assert.True(t, controller_common.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique)) } func TestPCSGStatusChangeIsSignificant(t *testing.T) { @@ -3978,6 +3979,7 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { }) } + t.Log("Wake reconciliation when only the scaling group scheduling message changes") oldScalingGroup := base() oldScalingGroup.Status.Conditions = []metav1.Condition{{ Type: groveconstants.ConditionTypeMinAvailableBreached, @@ -3987,7 +3989,7 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { }} newScalingGroup := oldScalingGroup.DeepCopy() newScalingGroup.Status.Conditions[0].Message = "two replicas unavailable" - assert.False(t, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup)) + assert.True(t, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup)) } func TestGroveChildEventPredicates(t *testing.T) { diff --git a/deploy/operator/internal/controller_common/workload_predicates.go b/deploy/operator/internal/controller_common/workload_predicates.go index c772d637ba91..9a3b29f0a071 100644 --- a/deploy/operator/internal/controller_common/workload_predicates.go +++ b/deploy/operator/internal/controller_common/workload_predicates.go @@ -44,7 +44,8 @@ func groveScheduledConditionChanged(oldConditions, newConditions []metav1.Condit if oldCondition != nil && newCondition != nil && (oldCondition.Status != newCondition.Status || - oldCondition.Reason != newCondition.Reason) { + oldCondition.Reason != newCondition.Reason || + oldCondition.Message != newCondition.Message) { return true } } From 27f6148440c7a9e4c98b0060b1b9456ec32f57da Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 17:44:57 +0000 Subject: [PATCH 058/196] fix(operator): retry incomplete LPX model refreshes Signed-off-by: Andrew Paprotsky (cherry picked from commit 7fef40e9dc6eec18a275dff2be9226eb783323f4) --- .../operator/internal/controller/lpx/controller.go | 2 +- .../internal/controller/lpx/modeldownload.go | 2 +- .../internal/controller/lpx/modeldownload_test.go | 13 +++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 3472530b579c..b9b87e81ab07 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -181,7 +181,7 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } } if ready == metav1.ConditionTrue && deployment.Status.ModelDownload != nil && deployment.Status.ModelDownload.LastCheckedAt != nil && result.RequeueAfter == 0 { - result.RequeueAfter = max(time.Second, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) + result.RequeueAfter = max(modelDownloadRequeueAfter, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) } result, err = completeLPXDeadline(ctx, deadlineAt, result, err) }() diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index f6727eda8056..3388634c567c 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -86,7 +86,7 @@ func (r *graphReconciler) reconcileModelDownloads( } else if previous == nil { deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{} } - if running || checkSucceeded { + if checkSucceeded { checkedAt := metav1.Now() deployment.Status.ModelDownload.LastCheckedAt = &checkedAt } diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 0d97f6fdd640..79416cea01f7 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -177,12 +177,12 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { name: "stale check remains ready when ModelExpress is down", checkedAt: time.Now().Add(-modelDownloadRefreshInterval), err: fmt.Errorf("ModelExpress unavailable"), - wantCalls: 1, + wantCalls: 2, }, { name: "stale check remains ready while an evicted model redownloads", checkedAt: time.Now().Add(-modelDownloadRefreshInterval), - wantCalls: 1, + wantCalls: 2, }, } @@ -210,9 +210,14 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { if len(registry.calls) != tt.wantCalls { t.Fatalf("ModelExpress calls = %d, want %d", len(registry.calls), tt.wantCalls) } - if tt.wantCalls > 0 && !child.Status.ModelDownload.LastCheckedAt.After(tt.checkedAt) { - t.Fatalf("last check = %s, want after %s", child.Status.ModelDownload.LastCheckedAt, tt.checkedAt) + + t.Log("Only a successful refresh starts another 24-hour cache window") + if tt.wantCalls > 0 && tt.ready { + require.True(t, child.Status.ModelDownload.LastCheckedAt.After(tt.checkedAt)) + } else { + require.Equal(t, tt.checkedAt, child.Status.ModelDownload.LastCheckedAt.Time) } + if !slices.Equal(child.Status.ModelDownload.Builds, []string{modelDownloadTestBuildID}) { t.Fatalf("downloaded builds = %v, want preserved", child.Status.ModelDownload.Builds) } From 54303729948b334d737161ed175045976eee365a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 18:09:08 +0000 Subject: [PATCH 059/196] fix(operator): preserve LPX PodCliqueSet list order Signed-off-by: Andrew Paprotsky (cherry picked from commit a322a0d0cf4b83511b7b5bc2bae15c49c7ad4261) --- .../operator/internal/controller/lpx/grove.go | 2 +- .../internal/controller/lpx/lifecycle_test.go | 49 +++++++++++++++++-- .../internal/controller_common/resource.go | 31 ++++++++---- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index 2e3aa520cd2a..d87bf2f4d4ce 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -42,6 +42,6 @@ func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( } // Match normal PCS synchronization; Grove owns rollout and immutable-field validation. - modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired) + modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired, commoncontroller.WithPreservedListOrder()) return synced, modified, err } diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index acfc674f15de..bc5d75338117 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -662,7 +662,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { cached.UID = "cached-pcs-uid" cached.ResourceVersion = "7" cached.Generation = 4 - hash, err := commoncontroller.GetSpecHash(cached) + hash, err := commoncontroller.GetSpecHash(cached, commoncontroller.WithPreservedListOrder()) require.NoError(t, err) metav1.SetMetaDataAnnotation(&cached.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) delete(cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) @@ -718,7 +718,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.Equal(t, 1, updates) want.Spec = desired.Spec want.ResourceVersion = synced.ResourceVersion - want.Annotations[commoncontroller.NvidiaAnnotationHashKey], err = commoncontroller.GetSpecHash(desired) + want.Annotations[commoncontroller.NvidiaAnnotationHashKey], err = commoncontroller.GetSpecHash(desired, commoncontroller.WithPreservedListOrder()) require.NoError(t, err) want.Annotations[commoncontroller.NvidiaAnnotationGenerationKey] = "5" require.True(t, apiequality.Semantic.DeepEqual(want, synced)) @@ -741,6 +741,49 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.Equal(t, concurrent, stored) } +func TestLPXPodCliqueSetListOrder(t *testing.T) { + for _, reorder := range []bool{false, true} { + t.Run(fmt.Sprintf("reorder=%t", reorder), func(t *testing.T) { + t.Log("Create an LPX PCS with two ordered init containers") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + desired := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, selected) + desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers = []corev1.Container{ + {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, + } + observed, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + require.NoError(t, err) + require.True(t, modified) + + t.Log("Model API-defaulted live fields without changing the applied generation") + observed.Generation = 1 + require.Nil(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.EnableServiceLinks) + observed.Spec.Template.Cliques[0].Spec.PodSpec.EnableServiceLinks = ptr.To(true) + require.NoError(t, reconciler.Update(ctx, observed)) + before := observed.DeepCopy() + if reorder { + slices.Reverse(desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) + } + + t.Log("Apply authored ordering changes but ignore unchanged desired specs") + synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, observed, desired) + require.NoError(t, err) + require.Equal(t, reorder, modified) + require.Equal(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers, synced.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) + require.Equal(t, before, observed, "synchronization must not mutate its observation") + if !reorder { + require.Equal(t, before, synced, "API defaults must not cause a spec rewrite") + } + + t.Log("The next reconciliation is a no-op") + _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + require.NoError(t, err) + require.False(t, modified) + }) + } +} + func TestLPXPodCliqueSetPublicationFence(t *testing.T) { t.Log("Prepare a render for publication and observe writes across the authoritative fence") ctx := t.Context() @@ -795,7 +838,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.True(t, metav1.IsControlledBy(synced, dgd)) require.Equal(t, desired.Labels, synced.Labels) require.Equal(t, desired.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) - hash, hashErr := commoncontroller.GetSpecHash(desired) + hash, hashErr := commoncontroller.GetSpecHash(desired, commoncontroller.WithPreservedListOrder()) require.NoError(t, hashErr) require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) require.Equal(t, "1", synced.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) diff --git a/deploy/operator/internal/controller_common/resource.go b/deploy/operator/internal/controller_common/resource.go index 31a4102367d4..0f47ca8b7fcd 100644 --- a/deploy/operator/internal/controller_common/resource.go +++ b/deploy/operator/internal/controller_common/resource.go @@ -76,11 +76,19 @@ type Reconciler interface { // if the resource should be deleted, the returned resource must contain the necessary information to delete it (name and namespace) type ResourceGenerator[T client.Object] func(ctx context.Context) (T, bool, error) -// SyncOption configures an exceptional SyncResource ownership policy. +// SyncOption configures resource synchronization. type SyncOption func(*syncOptions) type syncOptions struct { - sharedOwnership bool + sharedOwnership bool + preserveListOrder bool +} + +// WithPreservedListOrder makes authored list ordering significant in spec hashes. +func WithPreservedListOrder() SyncOption { + return func(options *syncOptions) { + options.preserveListOrder = true + } } // WithSharedOwnership permits a caller to reconcile a resource whose controller @@ -255,7 +263,7 @@ func SyncObservedResource[T client.Object]( logs.Info("No parent resource provided, creating resource without owner reference (independent lifecycle)") } - hash, err := GetSpecHash(desired) + hash, err := GetSpecHash(desired, opts...) if err != nil { logs.Error(err, "Failed to get spec hash.") recordResourceEvent(r, desired, corev1.EventTypeWarning, "GetSpecHash", "Get", "Failed to get spec hash for %s %s: %s", resourceType, resourceNamespace, err) @@ -283,7 +291,7 @@ func SyncObservedResource[T client.Object]( } } - changeResult, err := GetSpecChangeResult(observed, desired) + changeResult, err := GetSpecChangeResult(observed, desired, opts...) if err != nil { recordResourceEvent(r, desired, corev1.EventTypeWarning, fmt.Sprintf("CalculatePatch%s", resourceType), "Update", "Failed to calculate patch for %s %s: %s", resourceType, resourceNamespace, err) return false, desired, fmt.Errorf("failed to check if spec has changed: %w", err) @@ -468,12 +476,12 @@ type SpecChangeResult struct { // Returns: // - SpecChangeResult with update information // - error if hash computation fails -func GetSpecChangeResult(current client.Object, desired client.Object) (SpecChangeResult, error) { +func GetSpecChangeResult(current client.Object, desired client.Object, opts ...SyncOption) (SpecChangeResult, error) { desiredSpec, err := getSpec(desired) if err != nil { return SpecChangeResult{}, err } - desiredHash, err := GetResourceHash(desiredSpec) + desiredHash, err := GetResourceHash(desiredSpec, opts...) if err != nil { return SpecChangeResult{}, err } @@ -588,12 +596,12 @@ func generateSpecDiff(oldResource, newResource client.Object) (string, error) { return diff, nil } -func GetSpecHash(obj client.Object) (string, error) { +func GetSpecHash(obj client.Object, opts ...SyncOption) (string, error) { spec, err := getSpec(obj) if err != nil { return "", err } - return GetResourceHash(spec) + return GetResourceHash(spec, opts...) } // updateAnnotations sets both hash and generation annotations on an object @@ -608,13 +616,18 @@ func updateAnnotations(obj client.Object, hash string, generation int64) { } // GetResourceHash returns a consistent hash for the given object spec -func GetResourceHash(obj any) (string, error) { +func GetResourceHash(obj any, opts ...SyncOption) (string, error) { // Convert obj to a map[string]interface{} objMap, err := json.Marshal(obj) if err != nil { return "", err } + // JSON already orders map keys deterministically without reordering lists. + if resolveSyncOptions(opts).preserveListOrder { + return fmt.Sprintf("%x", sha256.Sum256(objMap)), nil + } + var objData map[string]interface{} if err := json.Unmarshal(objMap, &objData); err != nil { return "", err From f52b41126bbf82306c4f2353773d6df75b997774 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 18:30:28 +0000 Subject: [PATCH 060/196] fix(operator): scope device volume checks to LP30 init Signed-off-by: Andrew Paprotsky (cherry picked from commit aa447e3fa1f1d47dfdc683cc2d2157259f182d2e) --- .../operator/internal/dynamo/lpx/lpu_pod.go | 23 ----------- .../internal/dynamo/lpx/pod_roles_test.go | 40 +++++++++++++++---- .../internal/dynamo/lpx/runtime_agent.go | 28 +++++++------ 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index c5f74dc5a85e..12d3f1f63dd0 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -6,7 +6,6 @@ package lpx import ( - "fmt" "slices" corev1 "k8s.io/api/core/v1" @@ -19,28 +18,6 @@ const ( defaultLPUAgentCPUs int64 = 62 ) -func validateLPUHostDeviceVolumes(podSpec *corev1.PodSpec) error { - // Keep the existing compatibility checks without supplying missing device volumes. - for _, volume := range podSpec.Volumes { - var path string - switch volume.Name { - case "host-dev": - path = "/dev" - case "host-sys": - path = "/sys" - default: - continue - } - hostPath := volume.HostPath - if hostPath == nil || hostPath.Path != path || - hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && - *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate { - return fmt.Errorf("selected LPX podTemplate volume %q must use directory hostPath %q", volume.Name, path) - } - } - return nil -} - func updateWorkerPodSpec(podSpec *corev1.PodSpec) { // TODO can we remove some of this when we get the device-plugin? podSpec.HostIPC = true diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 0c2ea5c1a412..95c6442fbb15 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -523,7 +523,7 @@ func TestGeneratedAgentConfigMountOverrides(t *testing.T) { } } -func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { +func TestHostDeviceVolumeValidationIsLimitedToLP30Init(t *testing.T) { for _, test := range []struct { name string volume corev1.Volume @@ -549,7 +549,7 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { }}, false}, } { t.Run(test.name, func(t *testing.T) { - t.Log("Give an authored sidecar a volume name reserved by the Agent runtime") + t.Log("Give an authored sidecar a volume name used by LP30 initialization") base := corev1.PodSpec{ Containers: []corev1.Container{ {Name: commonconsts.MainContainerName, Image: "lpu-runtime"}, @@ -567,17 +567,13 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { configureAgentScheduling(&base, BuildFamilyXT) direct, agent := base.DeepCopy(), base.DeepCopy() - t.Log("Reject incompatible bindings through both XT runtime entry points") + t.Log("Leave authored device bindings to the DGD creator in both XT runtimes") errors := []error{ configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret", false), configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), } for _, err := range errors { - if test.wantErr { - require.ErrorContains(t, err, "must use directory hostPath") - } else { - require.NoError(t, err) - } + require.NoError(t, err) } t.Log("Never rebind the authored volume or its sidecar mount") @@ -586,6 +582,34 @@ func TestXTRuntimesValidateHostDeviceVolumes(t *testing.T) { require.Equal(t, test.volume.VolumeSource, testVolumeByName(t, pod.Volumes, test.volume.Name)) require.Equal(t, base.Containers[1], pod.Containers[1]) } + + t.Log("Validate only the host volumes mounted by the generated LP30 init") + lp30 := renderTestPodSpec() + index := slices.IndexFunc(lp30.Volumes, func(volume corev1.Volume) bool { return volume.Name == test.volume.Name }) + lp30.Volumes[index] = test.volume + before := lp30.DeepCopy() + err := addLP30InitContainer(&lp30, &lp30.Containers[0]) + if test.wantErr { + require.ErrorContains(t, err, "must use directory hostPath") + require.Equal(t, *before, lp30) + } else { + require.NoError(t, err) + require.Equal(t, before.Volumes, lp30.Volumes) + require.Equal(t, before.Containers, lp30.Containers) + } + }) + } + + for _, name := range []string{"host-dev", "host-sys"} { + t.Run("LP30 missing "+name, func(t *testing.T) { + t.Log("Reject a missing generated init binding without changing the Pod") + pod := renderTestPodSpec() + pod.Volumes = slices.DeleteFunc(pod.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }) + before := pod.DeepCopy() + err := addLP30InitContainer(&pod, &pod.Containers[0]) + require.ErrorContains(t, err, "requires podTemplate volume") + require.ErrorContains(t, err, name) + require.Equal(t, *before, pod) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 46ce0abd42ad..7a1513b3e6a2 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -176,10 +176,7 @@ func configureDirectHybridAgentRuntime( agentPodSpec.SecurityContext.RunAsGroup = ptr.To(int64(0)) agentPodSpec.SecurityContext.RunAsNonRoot = ptr.To(false) - if err := validateSSHVolume(agentPodSpec, sshSecretName); err != nil { - return err - } - return validateLPUHostDeviceVolumes(agentPodSpec) + return validateSSHVolume(agentPodSpec, sshSecretName) } // configureNodeLocalConductorRuntime consumes a nonnil conductor PodSpec. @@ -248,9 +245,6 @@ func configureNodeLocalAgentRuntime( } agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) - if err := validateLPUHostDeviceVolumes(agentPodSpec); err != nil { - return err - } if err := validateSSHVolume(agentPodSpec, sshSecretName); err != nil { return err } @@ -331,12 +325,6 @@ func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) erro if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), lp30InitContainerName).ToAggregate(); err != nil { return err } - // The generated init mounts fixed names; their host volumes must be authored. - for _, name := range []string{"host-dev", "host-sys"} { - if !slices.ContainsFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }) { - return fmt.Errorf("LP30 initialization requires podTemplate volume %q", name) - } - } initContainer := corev1.Container{ Name: lp30InitContainerName, @@ -360,6 +348,20 @@ func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) erro {Name: "host-dev", MountPath: "/dev"}, }, } + + // Validate only host volumes consumed by the operator-generated init container. + for _, mount := range initContainer.VolumeMounts { + index := slices.IndexFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) + if index < 0 { + return fmt.Errorf("LP30 initialization requires podTemplate volume %q", mount.Name) + } + hostPath := podSpec.Volumes[index].HostPath + if hostPath == nil || hostPath.Path != mount.MountPath || + hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && + *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate { + return fmt.Errorf("LP30 initialization volume %q must use directory hostPath %q", mount.Name, mount.MountPath) + } + } podSpec.InitContainers = append(podSpec.InitContainers, initContainer) return nil } From 1de6c885f12ee2601431b8c2ba9e4416e614bbef Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 14 Sep 2026 20:46:18 +0000 Subject: [PATCH 061/196] docs(operator): restore upstream BackendFramework comment Signed-off-by: Andrew Paprotsky (cherry picked from commit b485242ac736fbd90f8e7b72f1c52fc3b22f1399) Signed-off-by: Andrew Paprotsky --- .../operator/api/v1alpha1/dynamocomponentdeployment_types.go | 3 +-- .../crd/bases/nvidia.com_dynamocomponentdeployments.yaml | 4 +--- .../kubernetes-api/additional-resources/api-reference-k8s.md | 2 +- .../pages/reference/kubernetes-api/full-api-reference.mdx | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 6dde61ec9dbb..8e4448d59451 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -41,8 +41,7 @@ const ( // DynamoComponentDeploymentSpec defines the desired state of DynamoComponentDeployment // +kubebuilder:validation:XValidation:rule="!(has(self.componentType) && self.componentType == 'lpx')",message="standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment" type DynamoComponentDeploymentSpec struct { - // BackendFramework specifies the GPU backend framework (for example, - // "sglang", "vllm", or "trtllm"). + // BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") // +kubebuilder:validation:Enum=sglang;vllm;trtllm BackendFramework string `json:"backendFramework,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index faac5c4e2e58..2ec8318b9374 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -667,9 +667,7 @@ spec: type: integer type: object backendFramework: - description: |- - BackendFramework specifies the GPU backend framework (for example, - "sglang", "vllm", or "trtllm"). + description: BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") enum: - sglang - vllm diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 7e52c8b15b0f..b53b2f4e8f77 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -362,7 +362,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `backendFramework` _string_ | BackendFramework specifies the GPU backend framework (for example,
"sglang", "vllm", or "trtllm"). | | Enum: [sglang vllm trtllm]
| +| `backendFramework` _string_ | BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") | | Enum: [sglang vllm trtllm]
| | `annotations` _object (keys:string, values:string)_ | Annotations to add to generated Kubernetes resources for this component
(such as Pod, Service, and Ingress when applicable). | | | | `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | | | `serviceName` _string_ | The name of the component | | | diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 68703dcbfafc..9e72ed82f334 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -533,7 +533,7 @@ DynamoComponentDeploymentSpec defines the desired state of DynamoComponentDeploy **Appears in:** [DynamoComponentDeployment](#dynamocomponentdeployment) -BackendFramework specifies the GPU backend framework (for example, "sglang", "vllm", or "trtllm"). +BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") **Validation:** Enum: [sglang vllm trtllm] From 18953342e9a108be47ed721a22c714f8c6cfcbb9 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:27:54 +0300 Subject: [PATCH 062/196] fix(operator): preserve omitted LPX replicas through admission (#495) Signed-off-by: Andrew Paprotsky (cherry picked from commit 770cb178d97717629aad300aa554eb08fd2fd147) Signed-off-by: Andrew Paprotsky --- .../dynamographdeployment_handler.go | 8 +++--- ...hdeployment_lpx_validation_envtest_test.go | 28 +++++++++++++++++-- ...graphdeployment_validation_envtest_test.go | 15 +++++++++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go index 9e8fb69c69e0..e1cf541baa3c 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go @@ -55,7 +55,7 @@ func NewDGDDefaulter(operatorVersion string) *DGDDefaulter { } // Default implements admission.CustomDefaulter. -// On every operation: defaults nil component Replicas to 1 and persists the +// On every operation: defaults nil non-LPX component Replicas to 1 and persists the // replica counts implied by explicit multinode roles. // On CREATE: sets the controller-owned workload provider from routing intent before provider-specific defaults. // On the Grove pathway: defaults nil MinAvailable to 1. Scaling to replicas=0 @@ -88,12 +88,12 @@ func (d *DGDDefaulter) Default(ctx context.Context, obj runtime.Object) error { provideroverride.DefaultTarget(dgd.Spec.ProviderOverride, provider, provideroverride.ScopeRoot, nil) } - // Default nil replicas on every operation so newly added components remain safe to expand. + // Apply component defaults on every operation, including newly added components. for i := range dgd.Spec.Components { component := &dgd.Spec.Components[i] - // Default omitted replica counts before the controller expands component roles. - if component.Replicas == nil { + // Preserve omitted LPX replicas so Grove can retain its native scale. + if !component.IsLPX() && component.Replicas == nil { component.Replicas = ptr.To(int32(1)) } defaultMultinodeRoleReplicas(component) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 0374453a6ad0..6cd824e91cf1 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -19,9 +19,10 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { // Keep LPX inputs and oracles together without a separate admission execution path. return []dgdAdmissionTestCase{ { - name: "valid singleton lpx deployment with implicit conductor", + name: "singleton LPX with implicit conductor preserves omitted replicas", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { component := &dgd.Spec.Components[0] + component.Replicas = nil component.Roles = component.Roles[:1] component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ @@ -36,10 +37,30 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }, }} }), + wantReplicas: map[string]*int32{"lpx": nil}, }, { - name: "valid canonical v1alpha1 lpx deployment", - deployment: alphaLPXDGDForAdmission(nil), + name: "v1alpha1 LPX preserves omitted replicas on CREATE", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Replicas = nil + }), + wantReplicas: map[string]*int32{"lpx": nil}, + }, + { + name: "LPX preserves omitted replicas on UPDATE", + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Replicas = nil + }), + wantReplicas: map[string]*int32{"lpx": nil}, + }, + { + name: "v1alpha1 LPX preserves omitted replicas on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Replicas = nil + }), + wantReplicas: map[string]*int32{"lpx": nil}, }, { name: "LPX admits nine hybrid replicas with lowered conductor volumes", @@ -54,6 +75,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }}}, }} }), + wantReplicas: map[string]*int32{"lpx": k8sptr.To(int32(9))}, }, { name: "LPX native template schema rejects duplicate main container names", diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index dba265aaa378..10a6ce169fbf 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -69,6 +69,7 @@ type dgdAdmissionTestCase struct { wantPodAnnotations map[string]string wantProvider string wantRoleReplicas map[string]int32 + wantReplicas map[string]*int32 } func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { @@ -3074,9 +3075,21 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { } } actual := runAdmissionTest(t, test) - if tt.wantPodAnnotations != nil || tt.wantProvider != "" || tt.wantRoleReplicas != nil { + if tt.wantPodAnnotations != nil || tt.wantProvider != "" || tt.wantRoleReplicas != nil || tt.wantReplicas != nil { t.Log("Convert the admitted DGD for result assertions") actualDGD := admittedBetaDGD(t, actual) + + t.Log("Verify admission preserved component replica intent") + for name, want := range tt.wantReplicas { + component := actualDGD.GetComponentByName(name) + if component == nil { + t.Fatalf("admitted DGD has no component %q", name) + } + if !k8sptr.Equal(component.Replicas, want) { + t.Fatalf("component %q: replicas = %v, want %v", name, component.Replicas, want) + } + } + if tt.wantProvider != "" { t.Log("Verify creation-time routing intent determined the admitted workload provider") if got := actualDGD.Annotations[consts.KubeAnnotationWorkloadProvider]; got != tt.wantProvider { From eba9c3b8013843c5e950861dca465111c410299e Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:43:23 +0300 Subject: [PATCH 063/196] fix(operator): use Grove scale subresource for LPX replicas (#496) Signed-off-by: Andrew Paprotsky (cherry picked from commit ab6d55250db894004d0848b217fa2a13f606813e) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 8 ++- .../controller/lpx/controller_test.go | 56 +++++++++++++++++++ .../internal/controller/lpx/lifecycle_test.go | 12 ++++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index b9b87e81ab07..12644f10b04f 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -20,6 +20,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -284,8 +285,11 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a } else if !metav1.IsControlledBy(group, synced) || !group.DeletionTimestamp.IsZero() { return state, ctrl.Result{}, fmt.Errorf("LPX scaling group lacks the current PCS owner") } else if group.Spec.Replicas != *replicas { - group.Spec.Replicas = *replicas - if err := r.Update(ctx, group); err != nil { + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, + } + if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { return state, ctrl.Result{}, err } } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 1dad78df7aa4..23212505c271 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -20,6 +20,7 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -32,6 +33,61 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) +func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { + conflict := apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), "group", errors.New("stale version")) + for _, tc := range []struct { + name string + replicas *int32 + err error + wantUpdates int + }{ + {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, + {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, + {name: "unchanged", replicas: ptr.To(int32(9))}, + {name: "omitted"}, + {name: "conflict", replicas: ptr.To(int32(12)), err: conflict, wantUpdates: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe a Grove group with nine replicas") + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = tc.replicas + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) + group.Spec.Replicas = 9 + createLPXTestObjects(t, t.Context(), r.Client, objects...) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + updates := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "scale", subresource) + require.Equal(t, client.ObjectKeyFromObject(group), client.ObjectKeyFromObject(object)) + options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) + scale := options.SubResourceBody.(*autoscalingv1.Scale) + require.Equal(t, group.ResourceVersion, scale.ResourceVersion) + require.Equal(t, *tc.replicas, scale.Spec.Replicas) + updates++ + if tc.err != nil { + return tc.err + } + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + }) + + t.Log("Only explicit changes write scale, preserving the observed resource-version precondition") + _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) + require.ErrorIs(t, err, tc.err) + require.Equal(t, tc.wantUpdates, updates) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + want := int32(9) + if tc.replicas != nil && tc.err == nil { + want = *tc.replicas + } + require.Equal(t, want, group.Spec.Replicas) + }) + } +} + func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { t.Log("Publish a speculative engine using its frozen child identity") child, source, registry := newLPXSpecDecodeTestDGD(t) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index bc5d75338117..41295513b298 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -35,6 +35,7 @@ import ( grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/require" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -2397,6 +2398,17 @@ func newLPXTestReconciler( Build() nextLPRUID := 0 wrapped := interceptor.NewClient(base, interceptor.Funcs{ + // The fake client does not implement scale for Grove custom resources. + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && subresource == "scale" { + options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) + scale := options.SubResourceBody.(*autoscalingv1.Scale) + group.ResourceVersion = scale.ResourceVersion + group.Spec.Replicas = scale.Spec.Replicas + return delegated.Update(ctx, group) + } + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { if request.UID == "" { From 5e59774c1a1f4d10974478724428a08f735e40cf Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:43:23 +0300 Subject: [PATCH 064/196] docs(operator): align remaining backend comments with upstream (#498) Signed-off-by: Andrew Paprotsky (cherry picked from commit 1dde1c7ad21337b0f84cc6732891c4db37c6a853) Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1beta1/common.go | 2 +- .../operator/api/v1beta1/dynamocomponentdeployment_types.go | 3 +-- .../crd/bases/nvidia.com_dynamocomponentdeployments.yaml | 6 ++---- .../config/crd/bases/nvidia.com_dynamographdeployments.yaml | 2 +- .../additional-resources/api-reference-k8s.md | 4 ++-- .../pages/reference/kubernetes-api/full-api-reference.mdx | 4 ++-- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index ce98036c71b6..a26d03cc80ca 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -526,7 +526,7 @@ type DynamoCheckpointIdentity struct { // +kubebuilder:validation:MinLength=1 Model string `json:"model"` - // backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). + // backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). // Deprecated: legacy identity only. // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=vllm;sglang;trtllm diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index b97458fb7eca..6296d9c59840 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -49,8 +49,7 @@ const ( // DynamoComponentDeploymentSpec defines the desired state of a DynamoComponentDeployment. // +kubebuilder:validation:XValidation:rule="!(has(self.type) && self.type == 'lpx')",message="standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment" type DynamoComponentDeploymentSpec struct { - // backendFramework specifies the GPU backend framework (for example, - // "sglang", "vllm", or "trtllm"). + // backendFramework specifies the backend framework. // +kubebuilder:validation:Enum=sglang;vllm;trtllm BackendFramework string `json:"backendFramework,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 2ec8318b9374..02c73dadbe95 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -19947,9 +19947,7 @@ spec: description: spec defines the desired state for this Dynamo component deployment. properties: backendFramework: - description: |- - backendFramework specifies the GPU backend framework (for example, - "sglang", "vllm", or "trtllm"). + description: backendFramework specifies the backend framework. enum: - sglang - vllm @@ -20194,7 +20192,7 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). + backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). Deprecated: legacy identity only. enum: - vllm diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 4017c102bdfd..3c3c43245827 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20912,7 +20912,7 @@ spec: properties: backendFramework: description: |- - backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). + backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). Deprecated: legacy identity only. enum: - vllm diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index b53b2f4e8f77..b258d2985a64 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -1879,7 +1879,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `model` _string_ | model is the model identifier (e.g. "meta-llama/Llama-3-70B").
Deprecated: legacy identity only. | | MinLength: 1
Required: \{\}
| -| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| +| `backendFramework` _string_ | backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`).
Deprecated: legacy identity only. | | Enum: [vllm sglang trtllm]
Required: \{\}
| | `dynamoVersion` _string_ | dynamoVersion is the Dynamo platform version.
Deprecated: legacy identity only. | | Optional: \{\}
| | `tensorParallelSize` _integer_ | tensorParallelSize is the tensor parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| | `pipelineParallelSize` _integer_ | pipelineParallelSize is the pipeline parallel configuration.
Deprecated: checkpoint launch uses the pod template instead. | 1 | Minimum: 1
Optional: \{\}
| @@ -1967,7 +1967,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `backendFramework` _string_ | backendFramework specifies the GPU backend framework (for example,
"sglang", "vllm", or "trtllm"). | | Enum: [sglang vllm trtllm]
| +| `backendFramework` _string_ | backendFramework specifies the backend framework. | | Enum: [sglang vllm trtllm]
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| | `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 9e72ed82f334..dea9e5cc8c38 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -2613,7 +2613,7 @@ model is the model identifier (e.g. "meta-llama/Llama-3-70B"). Deprecated: legac -backendFramework is the runtime framework (`vllm`, `sglang`, or `trtllm`). Deprecated: legacy identity only. +backendFramework is the runtime framework (`vllm`, `sglang`, `trtllm`). Deprecated: legacy identity only. **Validation:** Enum: [vllm sglang trtllm] Required: \{\} @@ -2811,7 +2811,7 @@ DynamoComponentDeploymentSpec defines the desired state of a DynamoComponentDepl **Appears in:** [DynamoComponentDeployment](#v1beta1-dynamocomponentdeployment) -backendFramework specifies the GPU backend framework (for example, "sglang", "vllm", or "trtllm"). +backendFramework specifies the backend framework. **Validation:** Enum: [sglang vllm trtllm] From a9b2e328a446da27442d78406742ac41ecef42bc Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:59:35 +0300 Subject: [PATCH 065/196] test(docs): update Kubernetes API type counts (#500) Signed-off-by: Andrew Paprotsky (cherry picked from commit d31a4f2492ab73c774ae67c3dae08645bb4f8210) --- docs/fern/scripts/tests/test_gen_kubernetes_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index 61972681cdfd..efe9a8d7be1f 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -48,9 +48,9 @@ # (PodReference, PodSnapshotSource/Spec/Status, # PodSnapshotContentSource/Spec/Status, PodSnapshotReference), which are # owned by github.com/ai-dynamo/snapshot. - "nvidia.com/v1alpha1": 72, - "nvidia.com/v1beta1": 76, - "operator.config.dynamo.nvidia.com/v1alpha1": 30, + "nvidia.com/v1alpha1": 71, + "nvidia.com/v1beta1": 75, + "operator.config.dynamo.nvidia.com/v1alpha1": 29, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( "Pod Specification Defaults", From a9607ff831443a2ebfcde0f6664ddc45ef94626d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:53:50 +0300 Subject: [PATCH 066/196] fix(operator): retire removed LPX before ordinary reconciliation (#501) Signed-off-by: Andrew Paprotsky (cherry picked from commit 86a74e9bd44a64df1c1a0c15edacb9b8bccdc589) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_status.go | 3 +- .../internal/controller/dgd_grove_program.go | 32 ++++++- .../controller/dgd_lpx_handoff_test.go | 83 ++++++++++++++++++- 3 files changed, 111 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 52b7b9b1ae08..beefd7c9da21 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -84,7 +84,6 @@ func mergeLPXChildStatus( // but mirrors current LPX placement and clears stale mirrors. func lpxPlacementProjection( source *v1beta1.DynamoGraphDeployment, - child *v1alpha1.LPXGraphDeployment, current *v1beta1.PlacementStatus, previousLPX *v1beta1.DynamoGraphDeploymentLPXStatus, nextLPX *v1beta1.DynamoGraphDeploymentLPXStatus, @@ -92,7 +91,7 @@ func lpxPlacementProjection( if nextLPX != nil { return nextLPX.Placement } - if len(lpx.Components(source)) > 0 || child != nil { + if len(lpx.Components(source)) > 0 { return nil } if previousLPX != nil && apiequality.Semantic.DeepEqual(current, previousLPX.Placement) { diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index d9a4d5aa613a..a7516ca1b072 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -114,6 +115,25 @@ func (p *groveProgram) Reconcile( "hasMultinode", req.DGD.HasAnyMultinodeComponent(), ) + // Removing LPX must not depend on ordinary workloads reconciling successfully. + var child *nvidiacomv1alpha1.LPXGraphDeployment + var err error + if !req.DGD.HasLPXComponent() { + child, err = p.lpx.Reconcile(ctx, req.DGD) + if err != nil { + return programResult, fmt.Errorf("reconcile LPX child: %w", err) + } + + // Do not keep reporting retired LPX capacity when ordinary reconciliation fails. + programResult.Status.Placement = lpxPlacementProjection(req.DGD, programResult.Status.Placement, programResult.Status.LPX, nil) + programResult.Status.LPX = nil + for name := range programResult.Status.Components { + if req.DGD.GetComponentByName(name) == nil { + delete(programResult.Status.Components, name) + } + } + } + if err := p.rollout.migrateCurrentWorkerHashIfNeeded(ctx, req.DGD); err != nil { log.FromContext(ctx).Error(err, "Failed to migrate worker hash") return programResult, failWorkloadProgram(reasonFailedToMigrateWorkerHash, err) @@ -166,15 +186,19 @@ func (p *groveProgram) Reconcile( } return programResult, fmt.Errorf("failed to reconcile Grove workloads: %w", err) } - child, err := p.lpx.Reconcile(ctx, req.DGD) - if err != nil { - return programResult, fmt.Errorf("reconcile LPX child: %w", err) + + // Keep LPX creation and updates after ordinary reconciliation and restart selection. + if req.DGD.HasLPXComponent() { + child, err = p.lpx.Reconcile(ctx, req.DGD) + if err != nil { + return programResult, fmt.Errorf("reconcile LPX child: %w", err) + } } + previousLPX := programResult.Status.LPX result, programResult.Status.LPX = mergeLPXChildStatus(req.DGD, child, result) programResult.Status.Placement = lpxPlacementProjection( req.DGD, - child, programResult.Status.Placement, previousLPX, programResult.Status.LPX, diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 517ca9b47a12..2fee66eaa25f 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -75,7 +75,7 @@ func projectLPXChildStatus( ) { previousLPX := status.LPX *result, status.LPX = mergeLPXChildStatus(source, child, *result) - status.Placement = lpxPlacementProjection(source, child, status.Placement, previousLPX, status.LPX) + status.Placement = lpxPlacementProjection(source, status.Placement, previousLPX, status.LPX) } func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { @@ -636,6 +636,87 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { } } +func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { + for _, test := range []struct { + name string + retainLPX bool + childGone bool + independentPlacement bool + }{ + {name: "remove"}, + {name: "retain", retainLPX: true}, + {name: "independent placement", independentPlacement: true}, + {name: "pending child already gone", childGone: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Edit a mixed DGD while ordinary shared-resource reconciliation will fail") + child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + if test.childGone { + require.NoError(t, kube.Delete(t.Context(), child)) + } else { + child.Finalizers = []string{"test.example/child-cleanup"} + require.NoError(t, kube.Update(t.Context(), child)) + } + if !test.retainLPX { + source.Spec.Components = nil + } else { + source.Spec.Components[0].Replicas = ptr.To(int32(2)) + } + source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, + }) + source.Generation++ + require.NoError(t, kube.Update(t.Context(), source)) + source.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ + Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92)}, + } + source.Status.Placement = source.Status.LPX.Placement.DeepCopy() + source.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ + "lpx": {Replicas: 1}, "prefill": {Replicas: 2}, + } + if test.independentPlacement { + source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.5)} + } + if test.childGone { + source.Status.LPX, source.Status.Placement = nil, nil + source.Status.Components["lpx"] = v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup} + } + previousStatus := source.Status.DeepCopy() + program := (&DynamoGraphDeploymentReconciler{ + Client: kube, Config: &configv1alpha1.OperatorConfiguration{}, Recorder: events.NewFakeRecorder(10), + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, + }).newGroveProgram() + + t.Log("Preserve the ordinary error while deleting only a deselected child") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.ErrorContains(t, err, "RBAC manager not initialized") + require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) + stored := &v1alpha1.LPXGraphDeployment{} + if test.childGone { + require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored))) + } else { + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.Equal(t, !test.retainLPX, !stored.DeletionTimestamp.IsZero()) + require.Equal(t, child.Spec, stored.Spec) + } + require.Equal(t, previousStatus, &source.Status) + require.Equal(t, source.Status.Components["prefill"], result.Status.Components["prefill"]) + if !test.retainLPX { + require.Nil(t, result.Status.LPX) + require.NotContains(t, result.Status.Components, "lpx") + } else { + require.Equal(t, source.Status.LPX, result.Status.LPX) + require.Equal(t, source.Status.Components, result.Status.Components) + } + if test.retainLPX || test.independentPlacement { + require.Equal(t, source.Status.Placement, result.Status.Placement) + } else { + require.Nil(t, result.Status.Placement) + } + }) + } +} + func TestLPXChildFinalizationAndDeselectionWaitForCleanup(t *testing.T) { t.Log("Deselect LPX while the child reports unfinished cleanup through its finalizer") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") From a7903a52cf857ac7bf8ee53c6abfa69a4af76fb1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:52:28 +0300 Subject: [PATCH 067/196] refactor(operator): relocate generated LPX bindings (#502) Signed-off-by: Andrew Paprotsky (cherry picked from commit 19e3b854aae6eb35eb07274ae744a12c68e3c5d5) Signed-off-by: Andrew Paprotsky --- deploy/operator/cmd/main.go | 2 +- .../internal/controller/dgd_lpx_handoff_envtest_test.go | 4 ++-- .../operator/internal/controller/lpx/attempt_deadline.go | 2 +- .../internal/controller/lpx/attempt_deadline_test.go | 2 +- deploy/operator/internal/controller/lpx/controller_test.go | 2 +- deploy/operator/internal/controller/lpx/eviction.go | 2 +- deploy/operator/internal/controller/lpx/eviction_test.go | 2 +- deploy/operator/internal/controller/lpx/lifecycle.go | 2 +- deploy/operator/internal/controller/lpx/lifecycle_test.go | 4 ++-- deploy/operator/internal/controller/lpx/preflight.go | 2 +- deploy/operator/internal/controller/lpx/render_test.go | 4 ++-- deploy/operator/internal/controller/lpx/watch.go | 2 +- deploy/operator/internal/controller/lpx/watch_test.go | 2 +- .../operator/internal/dynamo/lpx/compiler_fixture_test.go | 2 +- deploy/operator/internal/dynamo/lpx/cyborg_render.go | 2 +- deploy/operator/internal/dynamo/lpx/cyborg_render_test.go | 2 +- deploy/operator/internal/dynamo/lpx/grove_ownership.go | 2 +- .../internal/dynamo/lpx/hybrid_cyborg_bindings_test.go | 2 +- deploy/operator/internal/dynamo/lpx/lpu_partition.go | 2 +- deploy/operator/internal/dynamo/lpx/lpu_partition_test.go | 2 +- .../{thirdparty/capnp => dynamo/lpx/manifest}/LICENSE | 0 .../lpx/manifest}/v2/gbuild_manifest.capnp.go | 3 +++ .../operator/internal/dynamo/lpx/materialization_test.go | 4 ++-- deploy/operator/internal/dynamo/lpx/model_fixture_test.go | 2 +- .../operator/internal/dynamo/lpx/model_manifest_common.go | 2 +- deploy/operator/internal/dynamo/lpx/model_manifest_v2.go | 2 +- .../operator/internal/dynamo/lpx/model_manifest_v2_test.go | 2 +- deploy/operator/internal/dynamo/lpx/model_projection.go | 2 +- deploy/operator/internal/dynamo/lpx/render.go | 2 +- deploy/operator/internal/dynamo/lpx/render_test.go | 4 ++-- .../internal/dynamo/lpx/resolve_selected_workload_test.go | 4 ++-- .../operator/internal/dynamo/lpx/runtime_settings_test.go | 2 +- .../lpxscheduler => dynamo/lpx/scheduler}/LICENSE | 0 .../lpx/scheduler}/v1alpha1/controller_gen.go | 0 .../operator/internal/dynamo/lpx/scheduler/v1alpha1/doc.go | 7 +++++++ .../lpx/scheduler}/v1alpha1/types.go | 3 +++ deploy/operator/internal/dynamo/lpx/workload_projection.go | 2 +- .../internal/dynamo/lpx/workload_projection_test.go | 2 +- .../operator/internal/dynamo/lpx/workload_projection_v2.go | 2 +- .../internal/dynamo/lpx/workload_projection_v2_test.go | 4 ++-- .../operator/internal/dynamo/lpx/workload_projection_v3.go | 2 +- .../internal/dynamo/lpx/workload_projection_v3_test.go | 4 ++-- deploy/operator/internal/features/lpx.go | 2 +- .../internal/thirdparty/lpxscheduler/v1alpha1/doc.go | 4 ---- 44 files changed, 58 insertions(+), 49 deletions(-) rename deploy/operator/internal/{thirdparty/capnp => dynamo/lpx/manifest}/LICENSE (100%) rename deploy/operator/internal/{thirdparty/capnp/gbuild_manifest => dynamo/lpx/manifest}/v2/gbuild_manifest.capnp.go (99%) rename deploy/operator/internal/{thirdparty/lpxscheduler => dynamo/lpx/scheduler}/LICENSE (100%) rename deploy/operator/internal/{thirdparty/lpxscheduler => dynamo/lpx/scheduler}/v1alpha1/controller_gen.go (100%) create mode 100644 deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/doc.go rename deploy/operator/internal/{thirdparty/lpxscheduler => dynamo/lpx/scheduler}/v1alpha1/types.go (99%) delete mode 100644 deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index b0d3cd1d3d43..c54bf58ed9f2 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -66,6 +66,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller" commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/crdmigrator" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/namespace_scope" "github.com/ai-dynamo/dynamo/deploy/operator/internal/observability" @@ -73,7 +74,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/rbac" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index b6eef40688cd..9e4cb49b6b06 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -21,10 +21,10 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/assert" diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index be8fe0dbdbd7..44793bacf084 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -11,7 +11,7 @@ import ( "sort" "time" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index b73ebb72741e..7ac3de7babdb 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 23212505c271..75289e491d3d 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -16,7 +16,7 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index eeae5fcc5bd5..f7e0816f2dd4 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -11,7 +11,7 @@ import ( "slices" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index f66da81d939d..52dc533f331d 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -11,7 +11,7 @@ import ( "testing" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/assert" diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 008694c4d7c6..4b9deca91c64 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -19,7 +19,7 @@ import ( "time" "unicode/utf8" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 41295513b298..04b008c9de6d 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -27,9 +27,9 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index 3743e7662899..0633c592ffa7 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -23,7 +23,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "k8s.io/apimachinery/pkg/util/validation/field" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 7d7b2dba81f9..dcea8f1b6a6f 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -24,8 +24,8 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" "gotest.tools/v3/golden" diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index e96ce799001a..a1bce74bc724 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -12,7 +12,7 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index ef7a9d07095d..5d92ffbfd5de 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -15,8 +15,8 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" diff --git a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go index 89296217e722..148ab4a5f7d8 100644 --- a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go @@ -16,7 +16,7 @@ import ( "capnproto.org/go/capnp/v3" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 088c6eac5f39..38bca3ac7b8f 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -12,7 +12,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 6c3a573c2fd0..327f8a9ab803 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -9,7 +9,7 @@ import ( "encoding/json" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/deploy/operator/internal/dynamo/lpx/grove_ownership.go b/deploy/operator/internal/dynamo/lpx/grove_ownership.go index 7addaf31d996..4a23e8ed57be 100644 --- a/deploy/operator/internal/dynamo/lpx/grove_ownership.go +++ b/deploy/operator/internal/dynamo/lpx/grove_ownership.go @@ -7,7 +7,7 @@ import ( "context" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 110121609599..5f85cbeb3cd4 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -12,7 +12,7 @@ import ( "strings" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" ) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition.go b/deploy/operator/internal/dynamo/lpx/lpu_partition.go index 3ed835ef97c0..eb15aae93bce 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_partition.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_partition.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" corev1 "k8s.io/api/core/v1" ) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go b/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go index 0f27f415fb4b..80b32b670a38 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go @@ -8,7 +8,7 @@ package lpx import ( "testing" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" diff --git a/deploy/operator/internal/thirdparty/capnp/LICENSE b/deploy/operator/internal/dynamo/lpx/manifest/LICENSE similarity index 100% rename from deploy/operator/internal/thirdparty/capnp/LICENSE rename to deploy/operator/internal/dynamo/lpx/manifest/LICENSE diff --git a/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go b/deploy/operator/internal/dynamo/lpx/manifest/v2/gbuild_manifest.capnp.go similarity index 99% rename from deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go rename to deploy/operator/internal/dynamo/lpx/manifest/v2/gbuild_manifest.capnp.go index 38d89a015f62..2bbe7aaf1a68 100644 --- a/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2/gbuild_manifest.capnp.go +++ b/deploy/operator/internal/dynamo/lpx/manifest/v2/gbuild_manifest.capnp.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + // Code generated by capnpc-go. DO NOT EDIT. package gbuildmanifestcapnpv2 diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index 572d3f078f66..0f5125434f28 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -12,8 +12,8 @@ import ( "testing" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" diff --git a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go index 2855820be855..7d62cdd6f5c0 100644 --- a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go @@ -12,7 +12,7 @@ import ( "testing" "capnproto.org/go/capnp/v3" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go index b7e7c70996ca..abe19bd5ed0d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go @@ -12,7 +12,7 @@ import ( "strings" "capnproto.org/go/capnp/v3" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" ) const hxTopologyFamily = "16x8x2x3" diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go index 883500e02159..b831714f5804 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -12,7 +12,7 @@ import ( "strings" "capnproto.org/go/capnp/v3" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" ) const ( diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go index b047d49daee1..308ef88ceb2c 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -9,7 +9,7 @@ import ( "testing" "capnproto.org/go/capnp/v3" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" ) diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index 170197299220..a584b37c176f 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -11,7 +11,7 @@ import ( "slices" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index f87544033c8a..b566c3c2be19 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -13,7 +13,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 35ffeba87f71..0444cc3478b2 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -12,8 +12,8 @@ import ( "testing" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index db6064975701..89145afe2de5 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -11,8 +11,8 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go index 2c00d1b8a9b5..1234eff839dd 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go @@ -11,7 +11,7 @@ import ( "slices" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" ) diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/LICENSE b/deploy/operator/internal/dynamo/lpx/scheduler/LICENSE similarity index 100% rename from deploy/operator/internal/thirdparty/lpxscheduler/LICENSE rename to deploy/operator/internal/dynamo/lpx/scheduler/LICENSE diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/controller_gen.go similarity index 100% rename from deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/controller_gen.go rename to deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/controller_gen.go diff --git a/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/doc.go b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/doc.go new file mode 100644 index 000000000000..f352f86cf051 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/doc.go @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by lpx-scheduler. DO NOT EDIT. + +// Package v1alpha1 contains the published LPX scheduling API types. +package v1alpha1 diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go similarity index 99% rename from deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go rename to deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go index 1a84d61a1d0e..0b3988e19a29 100644 --- a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/types.go +++ b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + // Code generated by lpx-scheduler. DO NOT EDIT. package v1alpha1 diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go index e1c2da6cbeec..7e7cfba9586c 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -13,7 +13,7 @@ import ( "fmt" "hash" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" ) const ( diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go index 308369177833..a3dae2d102f4 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index 1549fb42d20f..c63f9e229c50 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -11,7 +11,7 @@ import ( "slices" "sort" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" ) const ( diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index bb3a7511bc23..8622c45d1a44 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -11,8 +11,8 @@ import ( "math" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" ) diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 0e5d22878085..6f7fbfb19264 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -10,7 +10,7 @@ import ( "fmt" "strconv" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" ) const ( diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index 3fa16187e8bb..20435d1ae7af 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -9,8 +9,8 @@ import ( "encoding/json" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/capnp/gbuild_manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" ) diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go index 24b141cfac18..59a26fed6b3e 100644 --- a/deploy/operator/internal/features/lpx.go +++ b/deploy/operator/internal/features/lpx.go @@ -6,7 +6,7 @@ package features import ( "context" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "k8s.io/client-go/rest" "k8s.io/utils/ptr" diff --git a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go b/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go deleted file mode 100644 index 151bb242f863..000000000000 --- a/deploy/operator/internal/thirdparty/lpxscheduler/v1alpha1/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by lpx-scheduler. DO NOT EDIT. - -// Package v1alpha1 contains the published LPX scheduling API types. -package v1alpha1 From e8ff1f8427742c981b864c2f1c241ee6c5c9dcb0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:07:52 +0300 Subject: [PATCH 068/196] fix(operator): reuse LPX SSH host keys on restart (#503) Signed-off-by: Andrew Paprotsky (cherry picked from commit 401818e8e7e4656a2d09cece62c91738d859920c) Signed-off-by: Andrew Paprotsky --- .../lpx/commands/lpu_partition_agent.sh | 6 +-- .../internal/dynamo/lpx/commands/ssh_setup.sh | 6 +-- .../internal/dynamo/lpx/lpu_commands_test.go | 50 +++++++++++++++++-- .../node-local-v2-specdecode.yaml | 24 +++++---- .../node-local-v3-hx-hybrid.yaml | 17 ++++--- .../node-local-v3-hx-lpu-only.yaml | 8 +-- .../node-local-v3-hx-specdecode.yaml | 24 +++++---- .../testdata/from_dgd_yaml/single_v2.yaml | 8 +-- 8 files changed, 103 insertions(+), 40 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh index b70c758c54bc..497f53319eb3 100644 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh +++ b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh @@ -57,9 +57,9 @@ setup_ssh() { chmod 644 "${HOME}/.ssh/id_rsa.pub" printf "Host *\nIdentityFile %s/.ssh/id_rsa\nStrictHostKeyChecking no\nPort %s\n" "${HOME}" "${MPI_SSH_PORT}" > "${HOME}/.ssh/config" - ssh-keygen -t rsa -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" -N "" -q - ssh-keygen -t ecdsa -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" -N "" -q - ssh-keygen -t ed25519 -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" -N "" -q + test -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" || ssh-keygen -t rsa -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" -N "" -q + test -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" || ssh-keygen -t ecdsa -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" -N "" -q + test -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" || ssh-keygen -t ed25519 -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" -N "" -q } run_sshd() { diff --git a/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh b/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh index cd9b82bd546c..57a53ce34abe 100644 --- a/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh +++ b/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh @@ -10,6 +10,6 @@ chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys chmod 644 $HOME/.ssh/id_rsa.pub printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking no\nPort @@LPX_SSH_PORT@@\n' > $HOME/.ssh/config -ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N '' -ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N '' -ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N '' +test -f "$HOME/.ssh/host_keys/ssh_host_rsa_key" || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N '' +test -f "$HOME/.ssh/host_keys/ssh_host_ecdsa_key" || ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N '' +test -f "$HOME/.ssh/host_keys/ssh_host_ed25519_key" || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N '' diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go b/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go index be7b74bf10b4..e2ddf4bf97eb 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -98,6 +99,49 @@ printf '%s,%s,%s,%s,%s,%s,%s' "${CONFIG_PARTITION_INDEX}" "${LOGICAL_PARTITION_I } } +func TestLPXSSHSetupReusesHostKeys(t *testing.T) { + t.Parallel() + + t.Log("Extract the direct-hybrid SSH setup alongside the embedded worker setup") + _, partitionSetup, found := strings.Cut(lpuPartitionRunCommandSource, "setup_ssh() {\n") + require.True(t, found) + partitionSetup, _, found = strings.Cut(partitionSetup, "\n}") + require.True(t, found) + for name, setup := range map[string]string{"worker": sshSetupCommands, "direct hybrid": partitionSetup} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + t.Log("Isolate the SSH Secret and writable home without changing the process home") + dir := t.TempDir() + for _, name := range []string{"private.key", "private.key.pub"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte("secret"), 0o600)) + } + setup = strings.NewReplacer("${HOME}", dir, "$HOME", dir, "/ssh-pk", dir).Replace(setup) + command := `ssh-keygen() { + while [ "$1" != -f ]; do shift; done + [ ! -e "$2" ] || return 1 + printf 'host key' > "$2" +} +` + setup + + t.Log("Generate fresh keys, reuse them on restart, and fill a missing key") + for attempt := 0; attempt < 3; attempt++ { + if attempt == 2 { + require.NoError(t, os.Remove(filepath.Join(dir, ".ssh/host_keys/ssh_host_ecdsa_key"))) + } + cmd := exec.Command("bash", "-ec", command) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "attempt %d: %s", attempt, out) + for _, keyType := range []string{"rsa", "ecdsa", "ed25519"} { + key, err := os.ReadFile(filepath.Join(dir, ".ssh/host_keys/ssh_host_"+keyType+"_key")) + require.NoError(t, err) + require.Equal(t, "host key", string(key)) + } + } + }) + } +} + func TestLPUCommandDigests(t *testing.T) { t.Parallel() @@ -107,9 +151,9 @@ func TestLPUCommandDigests(t *testing.T) { command string want string }{ - {name: "partition", command: lpuPartitionRunCommand, want: "d6f8a2ce0f1031bcf8e08d0aff75e8aed33cbd07aae1fd3e09d813a03b879911"}, - {name: "worker", command: lpuWorkerRunCommand, want: "a6c8c03474d3feb459a329719ee961e2488a5e4886581adc10deec0691b103aa"}, - {name: "partition worker", command: lpuPartitionWorkerRunCommand, want: "65b022e0e8785f0ce2f5ff49d798db40cf8e538d310f3044473088e78d655886"}, + {name: "partition", command: lpuPartitionRunCommand, want: "e9af66afa3dbcdd589582d3576100cbdb72ae131d647919ef6bd11269d1462dd"}, + {name: "worker", command: lpuWorkerRunCommand, want: "ac7e7dfcf47c1a81bb1e80b7f91f323cc00ceb11bb85e0833fc248334e2b5dd0"}, + {name: "partition worker", command: lpuPartitionWorkerRunCommand, want: "f280af0e42b7c229317251cded7aa9d71ef53dd661e631c520b457e0d04cb3a8"}, {name: "probe", command: lpuV2ProbeCommand, want: "96ac74ae5e413a21fe98077f4d3d5b1dba7a741064e2734ae822ad77ab30e854"}, } diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 3af8e2dcb5ef..93a047cedef4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -417,9 +417,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" @@ -645,9 +647,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" @@ -873,9 +877,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index c2bdff1e529d..5b65423b6fd8 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -304,13 +304,16 @@ spec: /ssh-pk/private.key.pub \"${HOME}/.ssh/authorized_keys\"\n\tchmod 600 \"${HOME}/.ssh/id_rsa\" \"${HOME}/.ssh/authorized_keys\"\n\tchmod 644 \"${HOME}/.ssh/id_rsa.pub\"\n\tprintf \"Host *\\nIdentityFile %s/.ssh/id_rsa\\nStrictHostKeyChecking - no\\nPort %s\\n\" \"${HOME}\" \"${MPI_SSH_PORT}\" > \"${HOME}/.ssh/config\"\n\n\tssh-keygen - -t rsa -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" -N \"\" -q\n\tssh-keygen - -t ecdsa -f \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" -N \"\" -q\n\tssh-keygen - -t ed25519 -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" -N \"\" - -q\n}\n\nrun_sshd() {\n\tsetup_ssh\n\techo \"Starting sshd on port ${MPI_SSH_PORT} - for partition ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}\"\n\texec - /bin/sshd -D -e \\\n\t\t-o \"PidFile none\" \\\n\t\t-o \"Port ${MPI_SSH_PORT}\" + no\\nPort %s\\n\" \"${HOME}\" \"${MPI_SSH_PORT}\" > \"${HOME}/.ssh/config\"\n\n\ttest + -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" || ssh-keygen -t rsa + -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" -N \"\" -q\n\ttest -f + \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" -N \"\" -q\n\ttest + -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" || ssh-keygen -t + ed25519 -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" -N \"\" -q\n}\n\nrun_sshd() + {\n\tsetup_ssh\n\techo \"Starting sshd on port ${MPI_SSH_PORT} for partition + ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}\"\n\texec /bin/sshd + -D -e \\\n\t\t-o \"PidFile none\" \\\n\t\t-o \"Port ${MPI_SSH_PORT}\" \\\n\t\t-o \"AuthorizedKeysFile ${HOME}/.ssh/authorized_keys\" \\\n\t\t-o \"StrictModes no\" \\\n\t\t-o \"PermitRootLogin yes\" \\\n\t\t-o \"PubkeyAuthentication yes\" \\\n\t\t-o \"SetEnv PARTITION_ID=\\\"${PARTITION_ID}\\\" RANK_IN_PARTITION=\\\"${RANK_IN_PARTITION}\\\"\" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index b29e0ab30324..cdffae818faf 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -345,9 +345,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 2814dca51585..d9b5ddc0c450 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -403,9 +403,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" @@ -632,9 +634,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" @@ -861,9 +865,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 538174d8a3b4..f570a4113061 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -428,9 +428,11 @@ spec: $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\nssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key - -N ''\nssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key - -N ''\nssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key + 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" + || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest + -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa + -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" + || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" From cc591291b6ba3dd75cc940e97abac8999ae00de1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:05:15 +0300 Subject: [PATCH 069/196] fix(operator): finish LPX cleanup when request API is removed (#504) Signed-off-by: Andrew Paprotsky (cherry picked from commit 68bd3165aa04cf67c62e694ec3a4453487e517cd) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/lifecycle.go | 2 +- .../internal/controller/lpx/lifecycle_test.go | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 4b9deca91c64..d1945bef7087 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -627,7 +627,7 @@ func (r *graphReconciler) listOwnedLPXRequestsIfAvailable( client.InNamespace(deployment.Namespace), ) if err != nil { - if meta.IsNoMatchError(err) || k8sruntime.IsNotRegisteredError(err) { + if meta.IsNoMatchError(err) || k8sruntime.IsNotRegisteredError(err) || apierrors.IsNotFound(err) { return nil, false, nil } return nil, false, err diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 04b008c9de6d..ddbabdad18f5 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -1167,6 +1167,49 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) } +func TestLPXFinalizationAfterRequestAPIRemoval(t *testing.T) { + resource := lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource() + for _, scenario := range []struct { + name string + listErr error + finalize bool + }{ + {"removed CRD", apierrors.NewNotFound(resource, ""), true}, + {"forbidden", apierrors.NewForbidden(resource, "", errors.New("denied")), false}, + {"transport error", errors.New("connection reset"), false}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Return a request-list error after the operator has discovered the API") + ctx := t.Context() + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + child := newLPXTestDeployment(t, source) + r := newLPXTestReconciler(t, nil, child, source) + r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return scenario.listErr + }, + }) + + t.Log("Ordinary reconciliation must still fail when requests cannot be listed") + _, err := r.listOwnedLPXRequests(ctx, child) + require.Error(t, err) + + t.Log("Allow deletion only when the request API is gone, not on operational errors") + key := client.ObjectKeyFromObject(child) + require.NoError(t, r.Delete(ctx, child)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + if scenario.finalize { + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, key, child))) + } else { + require.ErrorIs(t, err, scenario.listErr) + require.NoError(t, r.Get(ctx, key, child)) + require.Contains(t, child.Finalizers, lpxGraphDeploymentFinalizer) + } + }) + } +} + func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { t.Run(string(pipeline), func(t *testing.T) { From 45c897be6df6d0abedd28029cb278d955945fa91 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:38:05 +0300 Subject: [PATCH 070/196] refactor(lpx): use runtime image entrypoints (#505) Signed-off-by: Andrew Paprotsky (cherry picked from commit d5ef0f8ccc9465a8f6837e0d5cc2a657ba2be082) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 50 ++-- .../lpx/commands/lpu_partition_agent.sh | 165 ---------- .../dynamo/lpx/commands/lpu_partition_env.sh | 6 - .../lpx/commands/lpu_partition_metadata.sh | 71 ----- .../dynamo/lpx/commands/lpu_system_init.sh | 49 --- .../dynamo/lpx/commands/lpu_worker.sh | 42 --- .../internal/dynamo/lpx/commands/ssh_setup.sh | 15 - .../internal/dynamo/lpx/conductor_runtime.go | 12 +- .../internal/dynamo/lpx/cyborg_render_test.go | 7 +- .../internal/dynamo/lpx/cyborg_runtime.go | 31 +- .../dynamo/lpx/cyborg_runtime_test.go | 64 ++-- .../internal/dynamo/lpx/lpu_commands.go | 76 +---- .../internal/dynamo/lpx/lpu_commands_test.go | 168 ----------- .../operator/internal/dynamo/lpx/lpu_pod.go | 6 +- .../internal/dynamo/lpx/pod_roles_test.go | 53 +++- .../internal/dynamo/lpx/render_test.go | 4 +- .../internal/dynamo/lpx/runtime_agent.go | 21 +- .../internal/dynamo/lpx/runtime_ssh.go | 6 +- .../internal/dynamo/lpx/runtime_ssh_test.go | 2 + .../internal/dynamo/lpx/runtime_startup.go | 30 +- .../dynamo/lpx/runtime_startup_test.go | 56 ++-- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 11 +- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 11 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 23 +- .../node-local-v2-specdecode.yaml | 281 +----------------- .../node-local-v3-hx-hybrid.yaml | 209 +------------ .../node-local-v3-hx-lpu-only.yaml | 77 +---- .../node-local-v3-hx-specdecode.yaml | 185 +----------- .../testdata/from_dgd_yaml/single_v2.yaml | 109 +------ 29 files changed, 224 insertions(+), 1616 deletions(-) delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh delete mode 100644 deploy/operator/internal/dynamo/lpx/lpu_commands_test.go diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index dcea8f1b6a6f..dab08a9a1937 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "slices" "sort" @@ -520,9 +519,10 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { if clique.Name == plan.ConductorTemplate { conductors++ require.Equal(t, wantImage, container.Image) - require.Equal(t, []string{"/bin/sh", "-ec"}, container.Command) - require.Contains(t, container.Args[0], `exec "$@"`) - require.Equal(t, []string{"--", "/configs/datacenter.toml", "/tmp/datacenter.toml", "/bin/nova"}, container.Args[1:5]) + require.Equal(t, []string{"/bin/nova"}, container.Command) + configFlag := slices.Index(container.Args, "--datacenter-config-filepath") + require.GreaterOrEqual(t, configFlag, 0) + require.Equal(t, "/configs/datacenter.toml", container.Args[configFlag+1]) if explicit { template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate require.Equal(t, "explicit-conductor", clique.Labels["owner"]) @@ -1234,10 +1234,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test main := agent.Spec.PodSpec.Containers[0] require.Equal(t, "agent", main.Name) - require.Equal(t, []string{"/bin/bash"}, main.Command) - require.Len(t, main.Args, 2) - require.Equal(t, "-c", main.Args[0]) - require.Contains(t, main.Args[1], "exec numactl") + require.Equal(t, []string{"/bin/hydra-entrypoint"}, main.Command) + require.Equal(t, []string{"start"}, main.Args) require.NotContains(t, main.Args, "--agent-env-vars") require.NotNil(t, main.SecurityContext) require.True(t, *main.SecurityContext.Privileged) @@ -1295,9 +1293,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Contains(t, cyborg.Spec.PodSpec.Containers, sidecar) require.NotEmpty(t, cyborg.Spec.PodSpec.Containers) cyborgMain := cyborg.Spec.PodSpec.Containers[0] - require.Equal(t, []string{"/bin/sh", "-ec"}, cyborgMain.Command) - require.Contains(t, cyborgMain.Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) - require.Equal(t, []string{"--", "/configs/lpu_servers", "/tmp/lpu_servers", "/opt/gpu-runtime", "serve"}, cyborgMain.Args[1:]) + require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, cyborgMain.Command) + require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--", "/opt/gpu-runtime", "serve"}, cyborgMain.Args) require.Contains(t, cyborgMain.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) @@ -1498,10 +1495,13 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin agents++ require.False(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) if len(intent.command)+len(intent.args) == 0 { - require.Equal(t, []string{"/bin/bash"}, main.Command) - require.Len(t, main.Args, 2) - require.Equal(t, "-c", main.Args[0]) - require.Contains(t, main.Args[1], "GROVE_PCLQ_POD_INDEX") + if mode.hybrid { + require.Equal(t, []string{"/bin/hydra-entrypoint"}, main.Command) + require.Equal(t, []string{"start"}, main.Args) + } else { + require.Equal(t, []string{"/bin/quasar-entrypoint"}, main.Command) + require.Equal(t, []string{"--partition-metadata"}, main.Args) + } } else { wantCommand := intent.command require.Equal(t, wantCommand, main.Command) @@ -1513,8 +1513,8 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin if len(command) == 0 { command = []string{"/bin/nova"} } - require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) - require.Equal(t, append(slices.Clone(command), intent.args...), main.Args[4:4+len(command)+len(intent.args)]) + require.Equal(t, command, main.Command) + require.True(t, slices.Equal(intent.args, main.Args[:len(intent.args)])) allocation := slices.Index(main.Args, "--allocation") require.GreaterOrEqual(t, allocation, 0) require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) @@ -1529,8 +1529,9 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin if len(wantCommand) == 0 { wantCommand = []string{"/usr/local/bin/cyborg"} } - require.Equal(t, []string{"/bin/sh", "-ec"}, main.Command) - require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[4:]) + require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, main.Command) + require.Equal(t, []string{"--expand-hosts", "--"}, main.Args[:2]) + require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[2:]) require.NotNil(t, main.LivenessProbe) require.NotNil(t, main.ReadinessProbe) require.NotNil(t, main.StartupProbe) @@ -1635,10 +1636,9 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { require.Equal(t, "9", env["TOTAL_REPLICAS"]) require.Equal(t, "/custom-servers", env["SERVER_HOSTS_FILE"]) - t.Log("Run SWA initialization without reading the replaced generated ConfigMap") - command := exec.CommandContext(t.Context(), "/bin/sh", "-ec", main.Args[0], "--", "/bin/sh", "-c", `printf '%s %s' "$SERVER_HOSTS_FILE" "$CYBORG_SWA_CACHE_IDS"`) - command.Env = []string{"SERVER_HOSTS_FILE=/custom-servers", "CYBORG_FPGA_GPI_REPLICA_INDEX=3", "CYBORG_BATCH_SIZE=2"} - output, err := command.CombinedOutput() - require.NoError(t, err, string(output)) - require.Equal(t, "/custom-servers 6,7", string(output)) + t.Log("Select only SWA initialization while preserving the authored command and host-file override") + require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, main.Command) + wantArgs := append([]string{"--swa-batch-ids", "--"}, leader.Spec.Containers[0].Command...) + wantArgs = append(wantArgs, leader.Spec.Containers[0].Args...) + require.Equal(t, wantArgs, main.Args) } diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh deleted file mode 100644 index 497f53319eb3..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_agent.sh +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -AGENT_BINARY="${AGENT_BINARY:-agent-hybrid-v2-roce}" -HOME="${HOME:-/root}" -MPI_SSH_PORT="${MPI_SSH_PORT:-@@LPX_SSH_PORT@@}" -READINESS_PORT="${READINESS_PORT:-@@LPX_READINESS_PORT@@}" -RDMA_PORT="${RDMA_PORT:-@@LPX_RDMA_PORT@@}" - -readarray -t SOURCE_PARTITIONS < "${LPU_CONFIG_DIR}/partition_ids" -readarray -t PARTITION_PATHS < "${LPU_CONFIG_DIR}/partition_paths" -readarray -t TOPOLOGIES < "${LPU_CONFIG_DIR}/topologies" - -SOURCE_PARTITION="${SOURCE_PARTITIONS[$CONFIG_PARTITION_INDEX]}" -PARTITION_PATH="${PARTITION_PATHS[$CONFIG_PARTITION_INDEX]}" -TOPOLOGY="${TOPOLOGIES[$CONFIG_PARTITION_INDEX]}" - -agent_part_from_partition_path() { - local part_path="$1" - local part_name="${part_path##*/}" - if [[ "${part_name}" =~ ^part-?([0-9]+)$ ]]; then - printf "%s\n" "${BASH_REMATCH[1]}" - return - fi - - printf "%s\n" "${SOURCE_PARTITION}" -} - -agent_assemble_dir_from_partition_path() { - local part_path="$1" - local agent_part="$2" - local part_name="${part_path##*/}" - local parent="${part_path%/*}" - if [[ "${part_name}" != "part-${agent_part}" ]]; then - local assemble_dir="/tmp/dynamo-agent-assemble/row-${CONFIG_PARTITION_INDEX}" - mkdir -p "${assemble_dir}" - ln -sfn "${GAS_DIR}/${part_path}" "${assemble_dir}/part-${agent_part}" - printf "%s\n" "${assemble_dir}" - return - fi - if [[ "${parent}" == "${part_path}" ]]; then - printf "%s\n" "${GAS_DIR}" - return - fi - printf "%s/%s\n" "${GAS_DIR}" "${parent}" -} - -AGENT_PART="$(agent_part_from_partition_path "${PARTITION_PATH}")" -AGENT_ASSEMBLE_DIR="$(agent_assemble_dir_from_partition_path "${PARTITION_PATH}" "${AGENT_PART}")" - -setup_ssh() { - mkdir -p "${HOME}/.ssh" "${HOME}/.ssh/host_keys" - cp /ssh-pk/private.key "${HOME}/.ssh/id_rsa" - cp /ssh-pk/private.key.pub "${HOME}/.ssh/id_rsa.pub" - cp /ssh-pk/private.key.pub "${HOME}/.ssh/authorized_keys" - chmod 600 "${HOME}/.ssh/id_rsa" "${HOME}/.ssh/authorized_keys" - chmod 644 "${HOME}/.ssh/id_rsa.pub" - printf "Host *\nIdentityFile %s/.ssh/id_rsa\nStrictHostKeyChecking no\nPort %s\n" "${HOME}" "${MPI_SSH_PORT}" > "${HOME}/.ssh/config" - - test -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" || ssh-keygen -t rsa -f "${HOME}/.ssh/host_keys/ssh_host_rsa_key" -N "" -q - test -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" || ssh-keygen -t ecdsa -f "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" -N "" -q - test -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" || ssh-keygen -t ed25519 -f "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" -N "" -q -} - -run_sshd() { - setup_ssh - echo "Starting sshd on port ${MPI_SSH_PORT} for partition ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}" - exec /bin/sshd -D -e \ - -o "PidFile none" \ - -o "Port ${MPI_SSH_PORT}" \ - -o "AuthorizedKeysFile ${HOME}/.ssh/authorized_keys" \ - -o "StrictModes no" \ - -o "PermitRootLogin yes" \ - -o "PubkeyAuthentication yes" \ - -o "SetEnv PARTITION_ID=\"${PARTITION_ID}\" RANK_IN_PARTITION=\"${RANK_IN_PARTITION}\"" \ - -h "${HOME}/.ssh/host_keys/ssh_host_rsa_key" \ - -h "${HOME}/.ssh/host_keys/ssh_host_ecdsa_key" \ - -h "${HOME}/.ssh/host_keys/ssh_host_ed25519_key" -} - -partition_hostnames() { - for ((rank = 0; rank < NODE_COUNT; rank++)); do - local node_index=$((NODE_OFFSET + rank)) - printf "%s-%d.%s\n" "${GROVE_PCLQ_NAME}" "${node_index}" "${GROVE_HEADLESS_SERVICE}" - done -} - -wait_for_peer_ssh() { - local host="$1" - for i in $(seq 1 120); do - if ssh -o ConnectTimeout=2 "root@${host}" echo ok >/dev/null 2>&1; then - echo "${host}:${MPI_SSH_PORT} reachable" - return - fi - if [[ "${i}" == "120" ]]; then - echo "${host}:${MPI_SSH_PORT} unreachable after 120 attempts" >&2 - exit 1 - fi - sleep 1 - done -} - -run_lpu_system_init -command -v "${AGENT_BINARY}" - -if (( NODE_COUNT == 1 )); then - echo "Starting ${AGENT_BINARY} for logical partition ${LOGICAL_PARTITION_INDEX}, source partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}" - exec "${AGENT_BINARY}" \ - --assemble-dir "${AGENT_ASSEMBLE_DIR}" \ - --part "${AGENT_PART}" \ - --topology "${TOPOLOGY}" \ - --port "${RDMA_PORT}" \ - --readiness-port "${READINESS_PORT}" \ - --stage-name continue -fi - -if [[ "${PARTITION_RANK}" != "0" ]]; then - run_sshd -fi - -setup_ssh -PARTITION_HOSTS="$(partition_hostnames)" -readarray -t NODE_HOSTS <<< "${PARTITION_HOSTS}" - -MPI_HOSTS=() -MPI_HOSTS+=("${POD_IP}:1") -for host in "${NODE_HOSTS[@]:1}"; do - MPI_HOSTS+=("${host}:1") -done -MPI_HOST_LIST=$(IFS=,; echo "${MPI_HOSTS[*]}") - -for host in "${NODE_HOSTS[@]:1}"; do - wait_for_peer_ssh "${host}" -done - -MPI_ARGS=( - --allow-run-as-root - --host "${MPI_HOST_LIST}" - -np "${NODE_COUNT}" - --map-by ppr:1:node:OVERSUBSCRIBE - --mca pml ob1 - --mca btl tcp,self - --mca mtl ^ofi - --mca plm_rsh_args "-p ${MPI_SSH_PORT}" - --bind-to none - -x GROQ_DRACO_NETWORK_UP_CHECK_TIMEOUT - -x DRACO_FORCE_RECONFIGURE - -x DRACO_MAXFILE_OVERRIDE - -x LPU_VSAP_DIB_CREDIT_AUTO_RESTORE_US - --output TAG -) -if [[ -n "${MPI_BTL_TCP_IF_INCLUDE:-}" ]]; then - MPI_ARGS+=(--mca btl_tcp_if_include "${MPI_BTL_TCP_IF_INCLUDE}") -fi - -echo "Starting mpirun for logical partition ${LOGICAL_PARTITION_INDEX}, source partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}" -exec numactl --cpunodebind=0 --membind=0 mpirun "${MPI_ARGS[@]}" -- \ - "${AGENT_BINARY}" \ - --assemble-dir "${AGENT_ASSEMBLE_DIR}" \ - --part "${AGENT_PART}" \ - --topology "${TOPOLOGY}" \ - --multinode \ - --port "${RDMA_PORT}" \ - --readiness-port "${READINESS_PORT}" \ - --stage-name continue diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh deleted file mode 100644 index fe21d019c884..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Preserve legacy Agent names from Dynamo's runtime partition mapping. -export PARTITION_ID="${LOGICAL_PARTITION_INDEX}" -export RANK_IN_PARTITION="${PARTITION_RANK}" diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh deleted file mode 100644 index 3e0e0d51c8a7..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_partition_metadata.sh +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -LPU_CONFIG_DIR="${LPU_CONFIG_DIR:-@@LPX_CONFIG_MOUNT_PATH@@}" -CLIQUE_POD_INDEX="${GROVE_PCLQ_POD_INDEX:-}" -if ! [[ "${CLIQUE_POD_INDEX}" =~ ^[0-9]+$ ]]; then - echo "GROVE_PCLQ_POD_INDEX must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}" >&2 - exit 1 -fi - -readarray -t PARTITION_NODE_COUNTS < "${LPU_CONFIG_DIR}/nodes_per_partition" -readarray -t PARTITION_NODE_OFFSETS < "${LPU_CONFIG_DIR}/partition_node_offsets" -if (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} != ${#PARTITION_NODE_OFFSETS[@]} )); then - echo "nodes_per_partition and partition_node_offsets must contain the same non-zero number of rows" >&2 - exit 1 -fi - -if [[ -f "${LPU_CONFIG_DIR}/partition_models" ]]; then - readarray -t PARTITION_MODELS < "${LPU_CONFIG_DIR}/partition_models" - readarray -t PARTITION_INDICES < "${LPU_CONFIG_DIR}/partition_indices" - if [[ -z "${LPU_MODEL_NAME:-}" ]]; then - echo "LPU_MODEL_NAME is required when partition_models is configured" >&2 - exit 1 - fi - if (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); then - echo "partition_models and partition_indices must match the partition config row count" >&2 - exit 1 - fi -else - PARTITION_MODELS=() - PARTITION_INDICES=() -fi - -CONFIG_PARTITION_INDEX="" -LOGICAL_PARTITION_INDEX="" -PARTITION_RANK="" -NODE_COUNT="" -NODE_OFFSET="" -PARTITION_MATCH_COUNT=0 -for i in "${!PARTITION_NODE_COUNTS[@]}"; do - row_node_count="${PARTITION_NODE_COUNTS[$i]}" - row_node_offset="${PARTITION_NODE_OFFSETS[$i]}" - if ! [[ "${row_node_count}" =~ ^[1-9][0-9]*$ && "${row_node_offset}" =~ ^[0-9]+$ ]]; then - echo "Invalid partition row ${i}: node count ${row_node_count}, node offset ${row_node_offset}" >&2 - exit 1 - fi - if (( ${#PARTITION_MODELS[@]} > 0 )) && [[ "${PARTITION_MODELS[$i]}" != "${LPU_MODEL_NAME}" ]]; then - continue - fi - if (( CLIQUE_POD_INDEX < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count )); then - continue - fi - - CONFIG_PARTITION_INDEX="${i}" - LOGICAL_PARTITION_INDEX="${PARTITION_INDICES[$i]:-${i}}" - PARTITION_RANK="$((CLIQUE_POD_INDEX - row_node_offset))" - NODE_COUNT="${row_node_count}" - NODE_OFFSET="${row_node_offset}" - PARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT + 1)) -done - -if (( PARTITION_MATCH_COUNT != 1 )); then - echo "Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition rows for model ${LPU_MODEL_NAME:-}" >&2 - exit 1 -fi -if ! [[ "${LOGICAL_PARTITION_INDEX}" =~ ^[0-9]+$ ]]; then - echo "Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}" >&2 - exit 1 -fi diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh deleted file mode 100644 index 8acd4b88879d..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_system_init.sh +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -run_lpu_system_init() { - if [[ ! -e /dev/fd ]]; then - ln -svf /proc/self/fd /dev/fd || true - fi - mkdir -p /tmp - mkdir -p /var/run/dpdk/rte || true - chmod 777 /var/run/dpdk/rte || true - ulimit -l unlimited || true - - if command -v lpu-system-init >/dev/null 2>&1; then - local init_ok="unknown" - local init_status=0 - for attempt in $(seq 1 2); do - echo "Running lpu-system-init (attempt ${attempt})..." - local init_output - local -a init_args=(--ci-precommand) - if (( attempt == 2 )); then - init_args+=(--ci-remediation-commands) - fi - set +e - init_output="$(lpu-system-init "${init_args[@]}" 2>&1)" - init_status=$? - set -e - echo "${init_output}" - init_ok="$(echo "${init_output}" | grep -o '"ok": *[a-z]*' | tail -1 | grep -o 'true\|false' || echo "unknown")" - if (( init_status == 0 )) && [[ "${init_ok}" != "false" ]]; then - break - fi - if (( attempt == 1 )); then - if (( init_status != 0 )); then - echo "lpu-system-init exited with status ${init_status}; retrying with remediation" >&2 - else - echo "lpu-system-init reported ok=false; retrying with remediation" >&2 - fi - fi - done - if (( init_status != 0 )) || [[ "${init_ok}" == "false" ]]; then - echo "lpu-system-init failed after 2 attempts; aborting" >&2 - exit 1 - fi - else - echo "lpu-system-init not found; continuing without pre-init" >&2 - fi -} diff --git a/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh b/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh deleted file mode 100644 index 74da21d455b2..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/lpu_worker.sh +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -HOME="${HOME:-/root}" - -if [[ "${LPU_RUN_SYSTEM_INIT:-true}" == "true" ]]; then - run_lpu_system_init -fi - -ulimit -l unlimited || true - -%s - -if command -v setcap >/dev/null 2>&1; then - for binary in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped /bin/agent_v2 /bin/agent_v2_perfetto_trace; do - if [[ -e "$binary" ]]; then - echo "Lifting capabilities of $binary" - setcap cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep "$(realpath "$binary")" || true - fi - done -fi - -# Pass down information from the k8s downward API into SSH sessions. -SETENV_LINE="" -SETENV_LINE="${SETENV_LINE} K8S_POD_NAME=\"$POD_NAME\"" -SETENV_LINE="${SETENV_LINE} K8S_POD_NAMESPACE=\"$POD_NAMESPACE\"" -SETENV_LINE="${SETENV_LINE} K8S_POD_UID=\"$POD_UID\"" -%sif [[ -n "${LPU_MODEL_NAME:-}" ]]; then - SETENV_LINE="${SETENV_LINE} LPU_MODEL_NAME=\"$LPU_MODEL_NAME\"" -fi -exec /bin/sshd -D -e \ - -o "PidFile none" \ - -o "Port @@LPX_SSH_PORT@@" \ - -o "AuthorizedKeysFile $HOME/.ssh/authorized_keys" \ - -o "StrictModes no" \ - -o PermitRootLogin=yes \ - -o PasswordAuthentication=no \ - -o PubkeyAuthentication=yes \ - -o "SetEnv $SETENV_LINE" \ - -h "$HOME/.ssh/host_keys/ssh_host_rsa_key" \ - -h "$HOME/.ssh/host_keys/ssh_host_ecdsa_key" \ - -h "$HOME/.ssh/host_keys/ssh_host_ed25519_key" diff --git a/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh b/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh deleted file mode 100644 index 57a53ce34abe..000000000000 --- a/deploy/operator/internal/dynamo/lpx/commands/ssh_setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -mkdir -p $HOME/.ssh $HOME/.ssh/host_keys -ls -la /ssh-pk/ -cp /ssh-pk/private.key $HOME/.ssh/id_rsa -cp /ssh-pk/private.key.pub $HOME/.ssh/id_rsa.pub -cp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys -chmod 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys -chmod 644 $HOME/.ssh/id_rsa.pub -printf 'Host *\nIdentityFile '$HOME'/.ssh/id_rsa\nStrictHostKeyChecking no\nPort @@LPX_SSH_PORT@@\n' > $HOME/.ssh/config - -test -f "$HOME/.ssh/host_keys/ssh_host_rsa_key" || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N '' -test -f "$HOME/.ssh/host_keys/ssh_host_ecdsa_key" || ssh-keygen -t ecdsa -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N '' -test -f "$HOME/.ssh/host_keys/ssh_host_ed25519_key" || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key -N '' diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go index 481514eacd40..0bef27f2eac2 100644 --- a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go @@ -14,17 +14,19 @@ import ( ) const ( - lpuModelConfigFilepath = lpuConfigMountPath + "/model_config.toml" - lpuExpandedDatacenterConfigFilepath = runtimeTemporaryStorageMountPath + "/datacenter.toml" + lpuModelConfigFilepath = lpuConfigMountPath + "/model_config.toml" + lpuDatacenterConfigFilepath = lpuConfigMountPath + "/datacenter.toml" ) func updateLPUConductorContainer( container *corev1.Container, allocation string, ) { - // Expand this engine's topology before the default or explicitly authored command. + // Nova reads the original configuration and expands hostname templates natively. container.Name = "conductor" - wrapRuntimeStartup(container, "/bin/nova", "datacenter.toml", "") + if len(container.Command) == 0 { + container.Command = []string{"/bin/nova"} + } // Nova and OpenMPI require root in the supported direct-DGD runtime. if container.SecurityContext == nil { @@ -49,7 +51,7 @@ func updateLPUConductorContainer( // configmap "--datacenter-config-filepath", - lpuExpandedDatacenterConfigFilepath, + lpuDatacenterConfigFilepath, "--model-config-filepath", lpuModelConfigFilepath, diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 327f8a9ab803..31e4f214ce7f 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -84,9 +84,8 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { }) require.Contains(t, cyborg.Spec.PodSpec.Volumes, renderTestPodSpec().Volumes[0]) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, renderTestPodSpec().Containers[0].VolumeMounts[0]) - require.Equal(t, []string{"/bin/sh", "-ec"}, cyborg.Spec.PodSpec.Containers[0].Command) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Args[0], `export CYBORG_SWA_CACHE_IDS="${ids}"`) - require.Equal(t, []string{"--", "/configs/lpu_servers", "/tmp/lpu_servers", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args[1:]) + require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, cyborg.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args) t.Log("Preserve an image-owned entrypoint") imageEntrypointPCS := renderTestPCS(true) @@ -103,7 +102,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { input.Stages = map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}} _, err = renderSelectedForTest(imageEntrypointPCS, []*ModelProjection{projection}, input) require.NoError(t, err) - require.Equal(t, []string{"/usr/local/bin/cyborg", "serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args[4:]) + require.Equal(t, []string{"/usr/local/bin/cyborg", "serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args[3:]) t.Log("Reject incomplete endpoint and fanout coverage of the same split-I/O runtime") diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index f07a5538251c..82aa6d27e2a1 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -63,31 +63,28 @@ func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { } } -const cyborgSwaBatchIDsInitialization = `if [ -z "${CYBORG_SWA_CACHE_IDS+x}" ]; then - : "${CYBORG_FPGA_GPI_REPLICA_INDEX:?CYBORG_FPGA_GPI_REPLICA_INDEX is required}" - : "${CYBORG_BATCH_SIZE:?CYBORG_BATCH_SIZE is required}" - base=$((CYBORG_FPGA_GPI_REPLICA_INDEX * CYBORG_BATCH_SIZE)) - ids="${base}" - i=1 - while [ "${i}" -lt "${CYBORG_BATCH_SIZE}" ]; do - ids="${ids},$((base + i))" - i=$((i + 1)) - done - export CYBORG_SWA_CACHE_IDS="${ids}" -fi -` - // wrapCyborgStartup prepares engine-local hosts and SWA IDs in one wrapper. func wrapCyborgStartup(container *corev1.Container, cyborgBatchSize int, configFile string) { if cyborgBatchSize == 1 && configFile == "" { return } - initialization := "" + args := make([]string, 0, 3+len(container.Command)+len(container.Args)) + if configFile != "" { + args = append(args, "--expand-hosts") + } if cyborgBatchSize > 1 { - initialization = cyborgSwaBatchIDsInitialization + args = append(args, "--swa-batch-ids") + } + args = append(args, "--") + if len(container.Command) == 0 { + args = append(args, "/usr/local/bin/cyborg") + } else { + args = append(args, container.Command...) } - wrapRuntimeStartup(container, "/usr/local/bin/cyborg", configFile, initialization) + args = append(args, container.Args...) + container.Command = []string{"/usr/local/bin/cyborg-entrypoint"} + container.Args = args } // cyborgRuntimeIO requires a nonnil normalized build and validates its Cyborg replica domain. diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go index 94c7f6ff4687..48c6f470b0ef 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go @@ -6,7 +6,6 @@ package lpx import ( - "os/exec" "testing" "github.com/stretchr/testify/require" @@ -30,54 +29,33 @@ func TestWrapCyborgDecodeForSwaBatchSkipsBatchOne(t *testing.T) { require.Equal(t, original, container) } -func TestWrapCyborgDecodeForSwaBatch(t *testing.T) { +func TestWrapCyborgStartupPreservesArgumentVector(t *testing.T) { t.Parallel() - t.Log("Define computed and explicitly overridden SWA cache ID scenarios") - tests := []struct { - name string - env []string - want string + for _, test := range []struct { + name string + batchSize int + configFile string + wantFlags []string }{ - { - name: "computes replica-local range with global offset", - env: []string{ - "CYBORG_FPGA_GPI_REPLICA_INDEX=3", - "CYBORG_BATCH_SIZE=2", - }, - want: "6,7", - }, - { - name: "keeps explicit override", - env: []string{ - "CYBORG_FPGA_GPI_REPLICA_INDEX=3", - "CYBORG_BATCH_SIZE=2", - "CYBORG_SWA_CACHE_IDS=100,101", - }, - want: "100,101", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + {"hosts", 1, "lpu_servers", []string{"--expand-hosts", "--"}}, + {"batch", 2, "", []string{"--swa-batch-ids", "--"}}, + {"hosts and batch", 2, "lpu_servers", []string{"--expand-hosts", "--swa-batch-ids", "--"}}, + } { + t.Run(test.name, func(t *testing.T) { t.Parallel() - t.Log("Wrap the actual command and preserve literal argument boundaries") + t.Log("Select image helper options while preserving a caller-owned shell and literal arguments") command := []string{"/bin/sh", "-c"} - args := []string{`printf '%s\n' "${CYBORG_SWA_CACHE_IDS}" "$@"`, "--", "argument with spaces", "literal '$HOME'", ""} + args := []string{`printf '%s\n' "$@"`, "--", "argument with spaces", "literal '$HOME'", ""} container := &corev1.Container{Command: command, Args: args} - wrapCyborgStartup(container, 2, "") - - t.Log("Changing caller-owned slices must not change the wrapped invocation") - command[0] = "mutated-command" - args[2] = "mutated-argument" - cmd := exec.Command(container.Command[0], append(container.Command[1:], container.Args...)...) - cmd.Env = tt.env + want := append(append(append([]string{}, test.wantFlags...), command...), args...) + wrapCyborgStartup(container, test.batchSize, test.configFile) - t.Log("Execute the wrapper and verify cache IDs and forwarded argument bytes") - out, err := cmd.Output() - require.NoError(t, err) - require.Equal(t, tt.want+"\nargument with spaces\nliteral '$HOME'\n\n", string(out)) + t.Log("Keep the wrapped invocation independent of subsequent caller slice mutations") + command[0], args[2] = "mutated-command", "mutated-argument" + require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, container.Command) + require.Equal(t, want, container.Args) }) } } @@ -101,6 +79,6 @@ func TestPinnedCyborgLauncher(t *testing.T) { container := &corev1.Container{Image: pinnedCyborgImage, Command: []string{"/bin/bash", "-lc"}, Args: []string{pinnedCyborgLauncher}} wrapCyborgStartup(container, 2, "lpu_servers") require.Equal(t, pinnedCyborgImage, container.Image) - require.Equal(t, []string{"/bin/bash", "-lc", pinnedCyborgLauncher}, container.Args[4:]) - require.Contains(t, container.Args[0], "export CYBORG_SWA_CACHE_IDS") + require.Equal(t, []string{"/bin/bash", "-lc", pinnedCyborgLauncher}, container.Args[3:]) + require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--"}, container.Args[:3]) } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands.go b/deploy/operator/internal/dynamo/lpx/lpu_commands.go index 77dfc110a01e..4ab2488b66b0 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_commands.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_commands.go @@ -6,11 +6,6 @@ package lpx import ( - _ "embed" - "fmt" - "strconv" - "strings" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -22,77 +17,8 @@ const ( // LPUReadinessPort is the HTTP readiness port used by the LPU runtime. LPUReadinessPort = 19878 lpuRDMAPort = 19877 - - lpuCommandTemplateTokenPrefix = "@@LPX_" - lpuConfigMountPathToken = "@@LPX_CONFIG_MOUNT_PATH@@" - lpuSSHPortToken = "@@LPX_SSH_PORT@@" - lpuReadinessPortToken = "@@LPX_READINESS_PORT@@" - lpuRDMAPortToken = "@@LPX_RDMA_PORT@@" - - embeddedShellSPDXHeader = `# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -` - - lpuPartitionSetEnvCommands = `SETENV_LINE="${SETENV_LINE} PARTITION_ID=\"$PARTITION_ID\"" -SETENV_LINE="${SETENV_LINE} RANK_IN_PARTITION=\"$RANK_IN_PARTITION\"" -` - - lpuV2ProbeCommandTemplate = `if (( NODE_COUNT > 1 )) && [[ "${PARTITION_RANK}" != "0" ]]; then - cat < /dev/null > "/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-@@LPX_SSH_PORT@@}" -else - exec 3<>"/dev/tcp/127.0.0.1/${READINESS_PORT:-@@LPX_READINESS_PORT@@}" - printf "GET /readyz HTTP/1.0\r\n\r\n" >&3 - read -r STATUS <&3 - [[ "${STATUS}" == *" 200 "* ]] -fi` ) -var ( - //go:embed commands/ssh_setup.sh - sshSetupCommandSource string - //go:embed commands/lpu_system_init.sh - lpuSystemInitCommandSource string - //go:embed commands/lpu_partition_metadata.sh - lpuPartitionMetadataCommandSource string - //go:embed commands/lpu_partition_env.sh - lpuPartitionEnvironmentCommandSource string - //go:embed commands/lpu_partition_agent.sh - lpuPartitionRunCommandSource string - //go:embed commands/lpu_worker.sh - lpuWorkerRunCommandSource string - - lpuCommandTemplateReplacer = strings.NewReplacer( - lpuConfigMountPathToken, lpuConfigMountPath, - lpuSSHPortToken, strconv.Itoa(LPUSSHPort), - lpuReadinessPortToken, strconv.Itoa(LPUReadinessPort), - lpuRDMAPortToken, strconv.Itoa(lpuRDMAPort), - ) - - sshSetupCommands = mustExpandLPUCommandTemplate(sshSetupCommandSource) - lpuSystemInitCommands = mustExpandLPUCommandTemplate(lpuSystemInitCommandSource) - lpuPartitionMetadataCommand = mustExpandLPUCommandTemplate(lpuPartitionMetadataCommandSource) - lpuPartitionEnvironmentCommand = mustExpandLPUCommandTemplate(lpuPartitionEnvironmentCommandSource) - lpuPartitionRunCommand = lpuSystemInitCommands + "\n" + lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + "\n" + mustExpandLPUCommandTemplate(lpuPartitionRunCommandSource) - lpuWorkerRunCommandTemplate = mustExpandLPUCommandTemplate(lpuWorkerRunCommandSource) - lpuV2ProbeCommand = lpuPartitionMetadataCommand + "\n" + mustExpandLPUCommandTemplate(lpuV2ProbeCommandTemplate) - - lpuWorkerRunCommand = lpuSystemInitCommands + "\n" + fmt.Sprintf(lpuWorkerRunCommandTemplate, sshSetupCommands, "") - lpuPartitionWorkerRunCommand = lpuSystemInitCommands + "\n" + lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + "\n" + fmt.Sprintf(lpuWorkerRunCommandTemplate, sshSetupCommands, lpuPartitionSetEnvCommands) -) - -func mustExpandLPUCommandTemplate(commandTemplate string) string { - // Remove only the standard source header so adding SPDX does not roll rendered workloads. - commandTemplate = strings.TrimPrefix(commandTemplate, embeddedShellSPDXHeader) - - // Resolve every Go-owned default through the package's single source of truth. - command := lpuCommandTemplateReplacer.Replace(commandTemplate) - if strings.Contains(command, lpuCommandTemplateTokenPrefix) { - panic("unexpanded LPX command template token") - } - return command -} - const ( lpuV2HTTPStartupFailureThreshold int32 = 60 lpuV2ExecStartupFailureThreshold int32 = 240 @@ -123,7 +49,7 @@ func lpuV2ReadinessProbe(useHTTP bool) *corev1.Probe { } } else { probe.Exec = &corev1.ExecAction{ - Command: []string{"/bin/bash", "-c", lpuV2ProbeCommand}, + Command: []string{"/bin/hydra-entrypoint", "ready"}, } } return probe diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go b/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go deleted file mode 100644 index e2ddf4bf97eb..000000000000 --- a/deploy/operator/internal/dynamo/lpx/lpu_commands_test.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "crypto/sha256" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLPUPartitionMetadataDerivesConfigRowAndRank(t *testing.T) { - t.Parallel() - - t.Log("Resolve the Bash interpreter used by the embedded metadata command") - bash, err := exec.LookPath("bash") - require.NoError(t, err) - - t.Log("Define single-model, SpecDecode, and invalid pod-index metadata fixtures") - tests := []struct { - name string - files map[string]string - podIndex string - model string - want string - wantError string - }{ - { - name: "single model", - files: map[string]string{ - "nodes_per_partition": "2\n1\n", - "partition_node_offsets": "0\n2\n", - }, - podIndex: "1", - want: "0,0,1,2,0,0,1", - }, - { - name: "specdecode model with reset offsets", - files: map[string]string{ - "nodes_per_partition": "2\n2\n2\n2\n", - "partition_node_offsets": "0\n2\n0\n2\n", - "partition_models": "draft0\ndraft0\ntarget\ntarget\n", - "partition_indices": "0\n1\n0\n1\n", - }, - podIndex: "1", - model: "target", - want: "2,0,1,2,0,0,1", - }, - { - name: "rejects out of range pod index", - files: map[string]string{ - "nodes_per_partition": "2\n1\n", - "partition_node_offsets": "0\n2\n", - }, - podIndex: "3", - wantError: "matched 0 partition rows", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - t.Log("Write the node-local partition metadata files") - configDir := t.TempDir() - for name, contents := range tt.files { - require.NoError(t, os.WriteFile(filepath.Join(configDir, name), []byte(contents), 0o600)) - } - - t.Log("Execute the embedded metadata and environment commands") - command := lpuPartitionMetadataCommand + "\n" + lpuPartitionEnvironmentCommand + ` -printf '%s,%s,%s,%s,%s,%s,%s' "${CONFIG_PARTITION_INDEX}" "${LOGICAL_PARTITION_INDEX}" "${PARTITION_RANK}" "${NODE_COUNT}" "${NODE_OFFSET}" "${PARTITION_ID}" "${RANK_IN_PARTITION}"` - cmd := exec.Command(bash, "-c", command) - cmd.Env = []string{ - "LPU_CONFIG_DIR=" + configDir, - "GROVE_PCLQ_POD_INDEX=" + tt.podIndex, - "LPU_MODEL_NAME=" + tt.model, - } - out, err := cmd.CombinedOutput() - if tt.wantError != "" { - t.Log("Verify invalid partition metadata is rejected") - require.Error(t, err) - require.Contains(t, string(out), tt.wantError) - return - } - - t.Log("Verify the derived configuration row, logical partition, and rank") - require.NoError(t, err, string(out)) - require.Equal(t, tt.want, string(out)) - }) - } -} - -func TestLPXSSHSetupReusesHostKeys(t *testing.T) { - t.Parallel() - - t.Log("Extract the direct-hybrid SSH setup alongside the embedded worker setup") - _, partitionSetup, found := strings.Cut(lpuPartitionRunCommandSource, "setup_ssh() {\n") - require.True(t, found) - partitionSetup, _, found = strings.Cut(partitionSetup, "\n}") - require.True(t, found) - for name, setup := range map[string]string{"worker": sshSetupCommands, "direct hybrid": partitionSetup} { - t.Run(name, func(t *testing.T) { - t.Parallel() - - t.Log("Isolate the SSH Secret and writable home without changing the process home") - dir := t.TempDir() - for _, name := range []string{"private.key", "private.key.pub"} { - require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte("secret"), 0o600)) - } - setup = strings.NewReplacer("${HOME}", dir, "$HOME", dir, "/ssh-pk", dir).Replace(setup) - command := `ssh-keygen() { - while [ "$1" != -f ]; do shift; done - [ ! -e "$2" ] || return 1 - printf 'host key' > "$2" -} -` + setup - - t.Log("Generate fresh keys, reuse them on restart, and fill a missing key") - for attempt := 0; attempt < 3; attempt++ { - if attempt == 2 { - require.NoError(t, os.Remove(filepath.Join(dir, ".ssh/host_keys/ssh_host_ecdsa_key"))) - } - cmd := exec.Command("bash", "-ec", command) - out, err := cmd.CombinedOutput() - require.NoError(t, err, "attempt %d: %s", attempt, out) - for _, keyType := range []string{"rsa", "ecdsa", "ed25519"} { - key, err := os.ReadFile(filepath.Join(dir, ".ssh/host_keys/ssh_host_"+keyType+"_key")) - require.NoError(t, err) - require.Equal(t, "host key", string(key)) - } - } - }) - } -} - -func TestLPUCommandDigests(t *testing.T) { - t.Parallel() - - t.Log("Define reviewed SHA-256 identities for assembled runtime commands") - tests := []struct { - name string - command string - want string - }{ - {name: "partition", command: lpuPartitionRunCommand, want: "e9af66afa3dbcdd589582d3576100cbdb72ae131d647919ef6bd11269d1462dd"}, - {name: "worker", command: lpuWorkerRunCommand, want: "ac7e7dfcf47c1a81bb1e80b7f91f323cc00ceb11bb85e0833fc248334e2b5dd0"}, - {name: "partition worker", command: lpuPartitionWorkerRunCommand, want: "f280af0e42b7c229317251cded7aa9d71ef53dd661e631c520b457e0d04cb3a8"}, - {name: "probe", command: lpuV2ProbeCommand, want: "96ac74ae5e413a21fe98077f4d3d5b1dba7a741064e2734ae822ad77ab30e854"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - t.Log("Bind the assembled embedded command to its reviewed SHA-256 digest") - require.Equal(t, test.want, fmt.Sprintf("%x", sha256.Sum256([]byte(test.command)))) - }) - } -} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index 12d3f1f63dd0..167ee736c879 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -36,10 +36,8 @@ func ensureLPUNodeTolerations(podSpec *corev1.PodSpec) { } } -func applyLPUWorkerContainerBase(container *corev1.Container, preserveCommand bool) { - if !preserveCommand { - container.Command = []string{"/bin/bash"} - } +func applyLPUWorkerContainerBase(container *corev1.Container) { + // Workers need privileged device access and an interactive runtime session. container.SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), RunAsUser: ptr.To(int64(0)), diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 95c6442fbb15..33a3bd2ec4e5 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" ) @@ -132,7 +133,7 @@ func TestApplyLPUWorkerContainerBaseClampsCPURequestToLimit(t *testing.T) { } t.Log("Apply the LPX worker runtime defaults") - applyLPUWorkerContainerBase(&container, false) + applyLPUWorkerContainerBase(&container) t.Log("Clamp the default request to the configured limit") require.True(t, container.Resources.Requests.Cpu().Equal(resource.MustParse(test.want))) @@ -293,8 +294,8 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { conductorContainer := conductor.Containers[0] require.Equal(t, conductorMounts, conductorContainer.VolumeMounts) require.Equal(t, "conductor", conductorContainer.Name) - require.Equal(t, []string{"/bin/sh", "-ec"}, conductorContainer.Command) - require.Equal(t, []string{runtimeConfigExpansion + "exec \"$@\"\n", "--", "/configs/datacenter.toml", "/tmp/datacenter.toml", "/bin/nova"}, conductorContainer.Args[:5]) + require.Equal(t, []string{"/bin/nova"}, conductorContainer.Command) + requireFlagValue(t, conductorContainer.Args, "--datacenter-config-filepath", "/configs/datacenter.toml") require.Contains(t, conductorContainer.Args, "--instance-model-name") requireFlagValue(t, conductorContainer.Args, "--allocation", "lpu-wkr-m-0") require.GreaterOrEqual(t, len(conductorContainer.Args), len(nodeLocalHXAgentEnvironmentArgs)) @@ -325,8 +326,8 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { agentContainer := agent.Containers[0] require.Equal(t, agentMounts, agentContainer.VolumeMounts) require.Equal(t, lpuAgentContainerName, agentContainer.Name) - require.Equal(t, []string{"/bin/bash"}, agentContainer.Command) - require.Equal(t, []string{"-c", lpuWorkerRunCommand}, agentContainer.Args) + require.Equal(t, []string{"/bin/quasar-entrypoint"}, agentContainer.Command) + require.Empty(t, agentContainer.Args) require.Empty(t, testContainerEnvValue(agentContainer.Env, "LPU_RUN_SYSTEM_INIT")) require.Equal(t, "cvp", testContainerEnvValue(agentContainer.Env, "LPU_FPGA_RECONFIGURE_METHOD")) require.Equal(t, "custom", testContainerEnvValue(agentContainer.Env, "GROQ_V2_RESET_ON_OPEN")) @@ -419,6 +420,48 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { } } +func TestGeneratedAgentLaunchCommands(t *testing.T) { + for _, test := range []struct { + name string + family BuildFamily + direct, multiNode bool + command []string + args []string + }{ + {"XT SSH worker", BuildFamilyXT, false, false, []string{"/bin/quasar-entrypoint"}, []string{"--partition-metadata"}}, + {"HX SSH worker", BuildFamilyHX, false, false, []string{"/bin/quasar-entrypoint"}, nil}, + {"direct single-node", BuildFamilyXT, true, false, []string{"/bin/hydra-entrypoint"}, []string{"start"}}, + {"direct multi-node", BuildFamilyXT, true, true, []string{"/bin/hydra-entrypoint"}, []string{"start"}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Configure a runtime role without an authored startup command") + pod := renderTestPodSpec() + pod.Containers[0].Command, pod.Containers[0].Args = nil, nil + configureAgentScheduling(&pod, test.family) + if test.direct { + require.NoError(t, configureDirectHybridAgentRuntime(&pod, "graph-lpu", "ssh-secret", test.multiNode)) + } else { + require.NoError(t, configureNodeLocalAgentRuntime(&pod, test.family, false, "ssh-secret")) + } + + t.Log("Execute the installed launcher after all role defaults have been applied") + agent := pod.Containers[0] + require.Equal(t, test.command, agent.Command) + require.Equal(t, test.args, agent.Args) + if test.direct { + require.Equal(t, []string{"/bin/hydra-entrypoint", "ready"}, agent.StartupProbe.Exec.Command) + require.Equal(t, []string{"/bin/hydra-entrypoint", "ready"}, agent.ReadinessProbe.Exec.Command) + require.EqualValues(t, 240, agent.StartupProbe.FailureThreshold) + require.EqualValues(t, 6, agent.ReadinessProbe.FailureThreshold) + require.EqualValues(t, 5, agent.ReadinessProbe.PeriodSeconds) + require.EqualValues(t, 1, agent.ReadinessProbe.TimeoutSeconds) + } else { + require.Equal(t, intstr.FromInt(LPUSSHPort), agent.ReadinessProbe.TCPSocket.Port) + } + }) + } +} + func TestGeneratedAgentSSHMountRequirements(t *testing.T) { for _, runtime := range []struct { name string diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 0444cc3478b2..0a776753247b 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -238,8 +238,8 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { conductor := namedClique(t, rendered, "lpu-ldr") require.Nil(t, conductor.Spec.PodSpec.Affinity) require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) - require.Equal(t, []string{"/bin/sh", "-ec"}, conductor.Spec.PodSpec.Containers[0].Command) - require.Equal(t, []string{"/bin/bash", "-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[4:7]) + require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 7a1513b3e6a2..145199426939 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -121,12 +121,13 @@ func configureDirectHybridAgentRuntime( agent.Args, _ = stripNovaOnlyArgs(agent.Args) hasCustomStartup := len(agent.Command) != 0 || len(agent.Args) != 0 agent.Name = lpuAgentContainerName - applyLPUWorkerContainerBase(agent, hasCustomStartup) - if len(agent.Args) == 0 && !hasCustomStartup { + applyLPUWorkerContainerBase(agent) + if !hasCustomStartup { if err := validateGeneratedAgentMounts(agent, true, multiNode); err != nil { return err } - agent.Args = []string{"-c", lpuPartitionRunCommand} + agent.Command = []string{"/bin/hydra-entrypoint"} + agent.Args = []string{"start"} } if !hasCustomStartup || agent.StartupProbe == nil { @@ -200,7 +201,7 @@ func configureNodeLocalConductorRuntime( // Materialize the GPC conductor around the image's Nova binary. setNodeLocalPodIPEnv(conductor, isXT) - if err := validateRuntimeConfigStorage(conductorPodSpec, conductor, "datacenter.toml"); err != nil { + if err := validateRuntimeConfigStorage(conductorPodSpec, conductor, ""); err != nil { return err } updateLPUConductorContainer(conductor, allocation) @@ -249,7 +250,7 @@ func configureNodeLocalAgentRuntime( return err } - // Check only the paths consumed by the selected generated worker script. + // Check only the paths consumed by the selected image launcher. if !preserveAgentEntrypoint { if err := validateGeneratedAgentMounts(agent, isXT, true); err != nil { return err @@ -258,7 +259,7 @@ func configureNodeLocalAgentRuntime( // Apply the family-specific worker resource and initialization requirements. if isXT { - applyLPUWorkerContainerBase(agent, preserveAgentEntrypoint) + applyLPUWorkerContainerBase(agent) agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) agent.SecurityContext.RunAsNonRoot = ptr.To(false) } else { @@ -287,13 +288,11 @@ func configureNodeLocalAgentWorkerContainer( ) { // The Agent pod hosts SSHD; Nova starts /bin/agent in a separate MPI process. if !preserveEntrypoint { - runCommand := lpuWorkerRunCommand + container.Command = []string{"/bin/quasar-entrypoint"} + container.Args = nil if isXT { - runCommand = lpuPartitionWorkerRunCommand - } else { - container.Command = []string{"/bin/bash"} + container.Args = []string{"--partition-metadata"} } - container.Args = []string{"-c", runCommand} } if isXT { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go index ce3a251bbe94..bc355fb6478d 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go @@ -22,8 +22,6 @@ const ( conductorSSHPrivateKeyPath = conductorSSHKeyDir + "/private.key" conductorSSHKeyInitContainerName = "prepare-ssh-key" conductorSSHKeyVolumeName = "single-v2-ssh-key" - conductorSSHKeyInitCommand = "set -euo pipefail; cp " + runtimeSSHSecretPrivateKeyPath + " " + - conductorSSHPrivateKeyPath + "; chmod 600 " + conductorSSHPrivateKeyPath ) func validateSSHVolume(podSpec *corev1.PodSpec, secretName string) error { @@ -67,8 +65,8 @@ func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, se Name: conductorSSHKeyInitContainerName, Image: conductor.Image, ImagePullPolicy: conductor.ImagePullPolicy, - Command: []string{"/bin/bash"}, - Args: []string{"-c", conductorSSHKeyInitCommand}, + Command: []string{"/bin/install"}, + Args: []string{"-m", "0600", runtimeSSHSecretPrivateKeyPath, conductorSSHPrivateKeyPath}, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go index b0d95e5d62b1..f0aea97afdd4 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go @@ -88,6 +88,8 @@ func TestAddConductorSSHKeyValidatesAuthoredStorage(t *testing.T) { require.Equal(t, before.Volumes, pod.Volumes) require.Equal(t, before.Containers, pod.Containers) require.Len(t, pod.InitContainers, 1) + require.Equal(t, []string{"/bin/install"}, pod.InitContainers[0].Command) + require.Equal(t, []string{"-m", "0600", "/ssh-pk/private.key", "/tmp/dynamo-lpu-ssh/private.key"}, pod.InitContainers[0].Args) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup.go b/deploy/operator/internal/dynamo/lpx/runtime_startup.go index 45b617b549c8..cc45c0525c16 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup.go @@ -12,37 +12,15 @@ import ( "k8s.io/utils/ptr" ) -const runtimeConfigExpansion = `: "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" -sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" -shift 2 -` - -// wrapRuntimeStartup prepares an optional fixed config and executes the original argument vector. -// configFile and initialization are operator-owned, never user-supplied shell fragments. -func wrapRuntimeStartup(container *corev1.Container, defaultCommand, configFile, initialization string) { - // Keep paths and the complete caller argument vector out of the shell source. - args := []string{initialization + "exec \"$@\"\n", "--"} - if configFile != "" { - args[0] = runtimeConfigExpansion + args[0] - args = append(args, lpuConfigMountPath+"/"+configFile, runtimeTemporaryStorageMountPath+"/"+configFile) - } - if len(container.Command) == 0 { - args = append(args, defaultCommand) - } else { - args = append(args, container.Command...) - } - args = append(args, container.Args...) - container.Command, container.Args = []string{"/bin/sh", "-ec"}, args -} - -// validateRuntimeConfigStorage requires writable, Pod-local storage for expanded configuration. -func validateRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, configFile string) error { +// validateRuntimeConfigStorage requires the config mount and a writable, Pod-local runtime workspace. +// expandedConfigFile is empty when the runtime reads configuration directly. +func validateRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, expandedConfigFile string) error { // Reject incompatible authored bindings instead of replacing their volumes or mounts. hasConfigMount := false var mount corev1.VolumeMount for _, existing := range container.VolumeMounts { hasConfigMount = hasConfigMount || existing.MountPath == lpuConfigMountPath - if existing.MountPath == runtimeTemporaryStorageMountPath+"/"+configFile { + if expandedConfigFile != "" && existing.MountPath == runtimeTemporaryStorageMountPath+"/"+expandedConfigFile { return fmt.Errorf("LPX runtime configuration cannot overwrite a volume mounted at %q", existing.MountPath) } if existing.MountPath == runtimeTemporaryStorageMountPath { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index a10dc935ea61..2731429ee1cd 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -6,9 +6,6 @@ package lpx import ( - "os" - "os/exec" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -16,37 +13,6 @@ import ( "k8s.io/utils/ptr" ) -func TestRuntimeStartup(t *testing.T) { - for _, file := range []string{"datacenter.toml", "lpu_servers"} { - for _, index := range []string{"0", "42", ""} { - t.Run(file+index, func(t *testing.T) { - t.Log("Render fixed Grove variables before an explicit command with literal arguments") - dir := t.TempDir() - input, output := filepath.Join(dir, "input"), filepath.Join(dir, "output") - require.NoError(t, os.WriteFile(input, []byte("group-${GROVE_PCSG_INDEX}-agent-0\n${GROVE_PCSG_NAME}.${GROVE_HEADLESS_SERVICE}"), 0600)) - container := &corev1.Container{Command: []string{"/bin/sh", "-c"}, Args: []string{`printf '%s\n' "$@"; exit 17`, "--", "argument with spaces", "literal '$HOME'", ""}} - wrapRuntimeStartup(container, "/bin/nova", file, "") - container.Args[2], container.Args[3] = input, output - cmd := exec.CommandContext(t.Context(), container.Command[0], append(container.Command[1:], container.Args...)...) - cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GROVE_PCSG_NAME=pcs-0-group", "GROVE_PCSG_INDEX=" + index, "GROVE_HEADLESS_SERVICE=service"} - actual, err := cmd.CombinedOutput() - if index == "" { - require.Error(t, err) - require.Contains(t, string(actual), "missing GROVE_PCSG_INDEX") - require.NoFileExists(t, output) - return - } - t.Log("Keep exit status, argument boundaries and this engine's expanded file") - require.EqualError(t, err, "exit status 17") - require.Equal(t, "argument with spaces\nliteral '$HOME'\n\n", string(actual)) - data, err := os.ReadFile(output) - require.NoError(t, err) - require.Equal(t, "group-"+index+"-agent-0\npcs-0-group.service", string(data)) - }) - } - } -} - func TestRuntimeConfigStorage(t *testing.T) { for _, security := range []*corev1.SecurityContext{nil, {ReadOnlyRootFilesystem: ptr.To(false)}, {ReadOnlyRootFilesystem: ptr.To(true)}} { t.Log("Allow container-local expansion unless the retained root filesystem is read-only") @@ -88,3 +54,25 @@ func TestRuntimeConfigStorage(t *testing.T) { require.ErrorContains(t, validateRuntimeConfigStorage(&pod, &container, "lpu_servers"), "writable storage") } } + +func TestNativeNovaConfigStorage(t *testing.T) { + t.Log("Mount a file at the obsolete expansion path alongside the original configuration") + pod := renderTestPodSpec() + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: "unrelated", MountPath: "/tmp/datacenter.toml", ReadOnly: true}, + ) + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "unrelated", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated"}}, + }}) + + t.Log("Launch Nova against /configs without trying to overwrite the unrelated mount") + require.NoError(t, configureNodeLocalConductorRuntime(&pod, BuildFamilyXT, "agent", "ssh-secret")) + require.Equal(t, []string{"/bin/nova"}, pod.Containers[0].Command) + requireFlagValue(t, pod.Containers[0].Args, "--datacenter-config-filepath", "/configs/datacenter.toml") + + t.Log("Continue rejecting an overlapping Cyborg expansion destination") + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: "unrelated", MountPath: "/tmp/lpu_servers", ReadOnly: true}, + ) + require.ErrorContains(t, validateRuntimeConfigStorage(&pod, &pod.Containers[0], "lpu_servers"), "cannot overwrite") +} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index d30e505bfcf6..59920c98fbd7 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -235,14 +235,8 @@ spec: - c1v2-b300n1-nic1 containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" + - --expand-hosts - -- - - /configs/lpu_servers - - /tmp/lpu_servers - /usr/local/bin/cyborg - --dynamo-model-name - meta-llama/Llama-3.2-1B-Instruct @@ -254,8 +248,7 @@ spec: - --num-layers - "16" command: - - /bin/sh - - -ec + - /usr/local/bin/cyborg-entrypoint env: - name: CYBORG_BATCH_SIZE value: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 6bc48beefeb1..777d09224ce9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -90,19 +90,12 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" + - --expand-hosts - -- - - /configs/lpu_servers - - /tmp/lpu_servers - /opt/gpu-runtime - serve command: - - /bin/sh - - -ec + - /usr/local/bin/cyborg-entrypoint env: - name: CYBORG_BATCH_SIZE value: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 45615b0fe402..69204b4ece98 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -94,15 +94,6 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" - - -- - - /configs/datacenter.toml - - /tmp/datacenter.toml - - /opt/dynamo-lpu - serve - --agent-path - /bin/agent @@ -115,7 +106,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /tmp/datacenter.toml + - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -148,8 +139,7 @@ spec: - --store-backend - kubernetes command: - - /bin/sh - - -ec + - /opt/dynamo-lpu env: - name: DYN_COMPONENT value: lpx @@ -217,11 +207,12 @@ spec: name: shared-memory initContainers: - args: - - -c - - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; - chmod 600 /tmp/dynamo-lpu-ssh/private.key + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key command: - - /bin/bash + - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 93a047cedef4..bb68359a798a 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -163,15 +163,6 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" - - -- - - /configs/datacenter.toml - - /tmp/datacenter.toml - - /bin/nova - --agent-path - /bin/agent - --agent-wd @@ -183,7 +174,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /tmp/datacenter.toml + - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -216,8 +207,7 @@ spec: - --store-backend - kubernetes command: - - /bin/sh - - -ec + - /bin/nova env: - name: DYN_COMPONENT value: lpx @@ -285,11 +275,12 @@ spec: name: shared-memory initContainers: - args: - - -c - - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; - chmod 600 /tmp/dynamo-lpu-ssh/private.key + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key command: - - /bin/bash + - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} @@ -356,91 +347,9 @@ spec: podSpec: containers: - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit - 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number of - rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is - required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} - != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models - and partition_indices must match the partition config row count\" >&2\n\t\texit - 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count - ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove - pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition - rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif - ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid - logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" - >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime - partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport - RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif - [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" - ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + - --partition-metadata command: - - /bin/bash + - /bin/quasar-entrypoint env: - name: DYN_COMPONENT value: lpx @@ -586,91 +495,9 @@ spec: podSpec: containers: - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit - 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number of - rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is - required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} - != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models - and partition_indices must match the partition config row count\" >&2\n\t\texit - 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count - ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove - pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition - rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif - ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid - logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" - >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime - partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport - RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif - [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" - ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + - --partition-metadata command: - - /bin/bash + - /bin/quasar-entrypoint env: - name: DYN_COMPONENT value: lpx @@ -816,91 +643,9 @@ spec: podSpec: containers: - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit - 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number of - rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is - required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} - != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models - and partition_indices must match the partition config row count\" >&2\n\t\texit - 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count - ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove - pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition - rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif - ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid - logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" - >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime - partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport - RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif - [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" - ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + - --partition-metadata command: - - /bin/bash + - /bin/quasar-entrypoint env: - name: DYN_COMPONENT value: lpx diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 5b65423b6fd8..26fd43939cb4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -229,132 +229,9 @@ spec: podSpec: containers: - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit - 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number of - rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is - required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} - != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models - and partition_indices must match the partition config row count\" >&2\n\t\texit - 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count - ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove - pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition - rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif - ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid - logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" - >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime - partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport - RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nAGENT_BINARY=\"${AGENT_BINARY:-agent-hybrid-v2-roce}\"\nHOME=\"${HOME:-/root}\"\nMPI_SSH_PORT=\"${MPI_SSH_PORT:-2222}\"\nREADINESS_PORT=\"${READINESS_PORT:-19878}\"\nRDMA_PORT=\"${RDMA_PORT:-19877}\"\n\nreadarray - -t SOURCE_PARTITIONS < \"${LPU_CONFIG_DIR}/partition_ids\"\nreadarray - -t PARTITION_PATHS < \"${LPU_CONFIG_DIR}/partition_paths\"\nreadarray - -t TOPOLOGIES < \"${LPU_CONFIG_DIR}/topologies\"\n\nSOURCE_PARTITION=\"${SOURCE_PARTITIONS[$CONFIG_PARTITION_INDEX]}\"\nPARTITION_PATH=\"${PARTITION_PATHS[$CONFIG_PARTITION_INDEX]}\"\nTOPOLOGY=\"${TOPOLOGIES[$CONFIG_PARTITION_INDEX]}\"\n\nagent_part_from_partition_path() - {\n\tlocal part_path=\"$1\"\n\tlocal part_name=\"${part_path##*/}\"\n\tif - [[ \"${part_name}\" =~ ^part-?([0-9]+)$ ]]; then\n\t\tprintf \"%s\\n\" - \"${BASH_REMATCH[1]}\"\n\t\treturn\n\tfi\n\n\tprintf \"%s\\n\" \"${SOURCE_PARTITION}\"\n}\n\nagent_assemble_dir_from_partition_path() - {\n\tlocal part_path=\"$1\"\n\tlocal agent_part=\"$2\"\n\tlocal part_name=\"${part_path##*/}\"\n\tlocal - parent=\"${part_path%/*}\"\n\tif [[ \"${part_name}\" != \"part-${agent_part}\" - ]]; then\n\t\tlocal assemble_dir=\"/tmp/dynamo-agent-assemble/row-${CONFIG_PARTITION_INDEX}\"\n\t\tmkdir - -p \"${assemble_dir}\"\n\t\tln -sfn \"${GAS_DIR}/${part_path}\" \"${assemble_dir}/part-${agent_part}\"\n\t\tprintf - \"%s\\n\" \"${assemble_dir}\"\n\t\treturn\n\tfi\n\tif [[ \"${parent}\" - == \"${part_path}\" ]]; then\n\t\tprintf \"%s\\n\" \"${GAS_DIR}\"\n\t\treturn\n\tfi\n\tprintf - \"%s/%s\\n\" \"${GAS_DIR}\" \"${parent}\"\n}\n\nAGENT_PART=\"$(agent_part_from_partition_path - \"${PARTITION_PATH}\")\"\nAGENT_ASSEMBLE_DIR=\"$(agent_assemble_dir_from_partition_path - \"${PARTITION_PATH}\" \"${AGENT_PART}\")\"\n\nsetup_ssh() {\n\tmkdir - -p \"${HOME}/.ssh\" \"${HOME}/.ssh/host_keys\"\n\tcp /ssh-pk/private.key - \"${HOME}/.ssh/id_rsa\"\n\tcp /ssh-pk/private.key.pub \"${HOME}/.ssh/id_rsa.pub\"\n\tcp - /ssh-pk/private.key.pub \"${HOME}/.ssh/authorized_keys\"\n\tchmod 600 - \"${HOME}/.ssh/id_rsa\" \"${HOME}/.ssh/authorized_keys\"\n\tchmod 644 - \"${HOME}/.ssh/id_rsa.pub\"\n\tprintf \"Host *\\nIdentityFile %s/.ssh/id_rsa\\nStrictHostKeyChecking - no\\nPort %s\\n\" \"${HOME}\" \"${MPI_SSH_PORT}\" > \"${HOME}/.ssh/config\"\n\n\ttest - -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" || ssh-keygen -t rsa - -f \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" -N \"\" -q\n\ttest -f - \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" -N \"\" -q\n\ttest - -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" || ssh-keygen -t - ed25519 -f \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\" -N \"\" -q\n}\n\nrun_sshd() - {\n\tsetup_ssh\n\techo \"Starting sshd on port ${MPI_SSH_PORT} for partition - ${LOGICAL_PARTITION_INDEX} rank ${PARTITION_RANK}\"\n\texec /bin/sshd - -D -e \\\n\t\t-o \"PidFile none\" \\\n\t\t-o \"Port ${MPI_SSH_PORT}\" - \\\n\t\t-o \"AuthorizedKeysFile ${HOME}/.ssh/authorized_keys\" \\\n\t\t-o - \"StrictModes no\" \\\n\t\t-o \"PermitRootLogin yes\" \\\n\t\t-o \"PubkeyAuthentication - yes\" \\\n\t\t-o \"SetEnv PARTITION_ID=\\\"${PARTITION_ID}\\\" RANK_IN_PARTITION=\\\"${RANK_IN_PARTITION}\\\"\" - \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t\t-h \"${HOME}/.ssh/host_keys/ssh_host_ed25519_key\"\n}\n\npartition_hostnames() - {\n\tfor ((rank = 0; rank < NODE_COUNT; rank++)); do\n\t\tlocal node_index=$((NODE_OFFSET - + rank))\n\t\tprintf \"%s-%d.%s\\n\" \"${GROVE_PCLQ_NAME}\" \"${node_index}\" - \"${GROVE_HEADLESS_SERVICE}\"\n\tdone\n}\n\nwait_for_peer_ssh() {\n\tlocal - host=\"$1\"\n\tfor i in $(seq 1 120); do\n\t\tif ssh -o ConnectTimeout=2 - \"root@${host}\" echo ok >/dev/null 2>&1; then\n\t\t\techo \"${host}:${MPI_SSH_PORT} - reachable\"\n\t\t\treturn\n\t\tfi\n\t\tif [[ \"${i}\" == \"120\" ]]; - then\n\t\t\techo \"${host}:${MPI_SSH_PORT} unreachable after 120 attempts\" - >&2\n\t\t\texit 1\n\t\tfi\n\t\tsleep 1\n\tdone\n}\n\nrun_lpu_system_init\ncommand - -v \"${AGENT_BINARY}\"\n\nif (( NODE_COUNT == 1 )); then\n\techo \"Starting - ${AGENT_BINARY} for logical partition ${LOGICAL_PARTITION_INDEX}, source - partition ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir - ${AGENT_ASSEMBLE_DIR}, topology ${TOPOLOGY}\"\n\texec \"${AGENT_BINARY}\" - \\\n\t\t--assemble-dir \"${AGENT_ASSEMBLE_DIR}\" \\\n\t\t--part \"${AGENT_PART}\" - \\\n\t\t--topology \"${TOPOLOGY}\" \\\n\t\t--port \"${RDMA_PORT}\" \\\n\t\t--readiness-port - \"${READINESS_PORT}\" \\\n\t\t--stage-name continue\nfi\n\nif [[ \"${PARTITION_RANK}\" - != \"0\" ]]; then\n\trun_sshd\nfi\n\nsetup_ssh\nPARTITION_HOSTS=\"$(partition_hostnames)\"\nreadarray - -t NODE_HOSTS <<< \"${PARTITION_HOSTS}\"\n\nMPI_HOSTS=()\nMPI_HOSTS+=(\"${POD_IP}:1\")\nfor - host in \"${NODE_HOSTS[@]:1}\"; do\n\tMPI_HOSTS+=(\"${host}:1\")\ndone\nMPI_HOST_LIST=$(IFS=,; - echo \"${MPI_HOSTS[*]}\")\n\nfor host in \"${NODE_HOSTS[@]:1}\"; do\n\twait_for_peer_ssh - \"${host}\"\ndone\n\nMPI_ARGS=(\n\t--allow-run-as-root\n\t--host \"${MPI_HOST_LIST}\"\n\t-np - \"${NODE_COUNT}\"\n\t--map-by ppr:1:node:OVERSUBSCRIBE\n\t--mca pml - ob1\n\t--mca btl tcp,self\n\t--mca mtl ^ofi\n\t--mca plm_rsh_args \"-p - ${MPI_SSH_PORT}\"\n\t--bind-to none\n\t-x GROQ_DRACO_NETWORK_UP_CHECK_TIMEOUT\n\t-x - DRACO_FORCE_RECONFIGURE\n\t-x DRACO_MAXFILE_OVERRIDE\n\t-x LPU_VSAP_DIB_CREDIT_AUTO_RESTORE_US\n\t--output - TAG\n)\nif [[ -n \"${MPI_BTL_TCP_IF_INCLUDE:-}\" ]]; then\n\tMPI_ARGS+=(--mca - btl_tcp_if_include \"${MPI_BTL_TCP_IF_INCLUDE}\")\nfi\n\necho \"Starting - mpirun for logical partition ${LOGICAL_PARTITION_INDEX}, source partition - ${SOURCE_PARTITION}, part path ${PARTITION_PATH}, assemble-dir ${AGENT_ASSEMBLE_DIR}, - topology ${TOPOLOGY}\"\nexec numactl --cpunodebind=0 --membind=0 mpirun - \"${MPI_ARGS[@]}\" -- \\\n\t\"${AGENT_BINARY}\" \\\n\t\t--assemble-dir - \"${AGENT_ASSEMBLE_DIR}\" \\\n\t\t--part \"${AGENT_PART}\" \\\n\t\t--topology - \"${TOPOLOGY}\" \\\n\t\t--multinode \\\n\t\t--port \"${RDMA_PORT}\" - \\\n\t\t--readiness-port \"${READINESS_PORT}\" \\\n\t\t--stage-name - continue\n" + - start command: - - /bin/bash + - /bin/hydra-entrypoint env: - name: DYN_COMPONENT value: lpx @@ -414,45 +291,8 @@ spec: readinessProbe: exec: command: - - /bin/bash - - -c - - "set -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" - >&2\n\texit 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number - of rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME - is required when partition_models is configured\" >&2\n\t\texit - 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} - || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); - then\n\t\techo \"partition_models and partition_indices must match - the partition config row count\" >&2\n\t\texit 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node - count ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit - 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo - \"Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} - partition rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit - 1\nfi\nif ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo - \"Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for - config row ${CONFIG_PARTITION_INDEX}\" >&2\n\texit 1\nfi\n\nif (( - NODE_COUNT > 1 )) && [[ \"${PARTITION_RANK}\" != \"0\" ]]; then\n\tcat - < /dev/null > \"/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-2222}\"\nelse\n\texec - 3<>\"/dev/tcp/127.0.0.1/${READINESS_PORT:-19878}\"\n\tprintf \"GET - /readyz HTTP/1.0\\r\\n\\r\\n\" >&3\n\tread -r STATUS <&3\n\t[[ \"${STATUS}\" - == *\" 200 \"* ]]\nfi" + - /bin/hydra-entrypoint + - ready failureThreshold: 6 periodSeconds: 5 successThreshold: 1 @@ -471,45 +311,8 @@ spec: startupProbe: exec: command: - - /bin/bash - - -c - - "set -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" - >&2\n\texit 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number - of rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME - is required when partition_models is configured\" >&2\n\t\texit - 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} - || ${#PARTITION_INDICES[@]} != ${#PARTITION_NODE_COUNTS[@]} )); - then\n\t\techo \"partition_models and partition_indices must match - the partition config row count\" >&2\n\t\texit 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node - count ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit - 1\n\tfi\n\tif (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo - \"Grove pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} - partition rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit - 1\nfi\nif ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo - \"Invalid logical partition index ${LOGICAL_PARTITION_INDEX} for - config row ${CONFIG_PARTITION_INDEX}\" >&2\n\texit 1\nfi\n\nif (( - NODE_COUNT > 1 )) && [[ \"${PARTITION_RANK}\" != \"0\" ]]; then\n\tcat - < /dev/null > \"/dev/tcp/127.0.0.1/${MPI_SSH_PORT:-2222}\"\nelse\n\texec - 3<>\"/dev/tcp/127.0.0.1/${READINESS_PORT:-19878}\"\n\tprintf \"GET - /readyz HTTP/1.0\\r\\n\\r\\n\" >&3\n\tread -r STATUS <&3\n\t[[ \"${STATUS}\" - == *\" 200 \"* ]]\nfi" + - /bin/hydra-entrypoint + - ready failureThreshold: 240 periodSeconds: 5 successThreshold: 1 diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index cdffae818faf..045f6a12e151 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -86,15 +86,6 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" - - -- - - /configs/datacenter.toml - - /tmp/datacenter.toml - - /bin/nova - --instance-model-name - test/llama32-1b-lp30 - --agent-path @@ -108,7 +99,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /tmp/datacenter.toml + - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -179,8 +170,7 @@ spec: - --agent-env-vars - REDFISH_BMC_ENABLED=false command: - - /bin/sh - - -ec + - /bin/nova env: - name: DYN_COMPONENT value: lpx @@ -248,11 +238,12 @@ spec: name: shared-memory initContainers: - args: - - -c - - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; - chmod 600 /tmp/dynamo-lpu-ssh/private.key + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key command: - - /bin/bash + - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} @@ -316,58 +307,8 @@ spec: minAvailable: 1 podSpec: containers: - - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ - \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; - then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" - command: - - /bin/bash + - command: + - /bin/quasar-entrypoint env: - name: BMC_SSH_ENABLED value: "false" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index d9b5ddc0c450..7a73577bfc3a 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -142,15 +142,6 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" - - -- - - /configs/datacenter.toml - - /tmp/datacenter.toml - - /bin/nova - --instance-model-name - test/v3-specdecode - --agent-path @@ -164,7 +155,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /tmp/datacenter.toml + - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -235,8 +226,7 @@ spec: - --agent-env-vars - REDFISH_BMC_ENABLED=false command: - - /bin/sh - - -ec + - /bin/nova env: - name: DYN_COMPONENT value: lpx @@ -304,11 +294,12 @@ spec: name: shared-memory initContainers: - args: - - -c - - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; - chmod 600 /tmp/dynamo-lpu-ssh/private.key + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key command: - - /bin/bash + - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} @@ -374,58 +365,8 @@ spec: minAvailable: 1 podSpec: containers: - - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ - \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; - then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" - command: - - /bin/bash + - command: + - /bin/quasar-entrypoint env: - name: BMC_SSH_ENABLED value: "false" @@ -605,58 +546,8 @@ spec: minAvailable: 1 podSpec: containers: - - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ - \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; - then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" - command: - - /bin/bash + - command: + - /bin/quasar-entrypoint env: - name: BMC_SSH_ENABLED value: "false" @@ -836,58 +727,8 @@ spec: minAvailable: 1 podSpec: containers: - - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nHOME=\"${HOME:-/root}\"\n\nif [[ - \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" ]]; - then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" - command: - - /bin/bash + - command: + - /bin/quasar-entrypoint env: - name: BMC_SSH_ENABLED value: "false" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index f570a4113061..69d16622580b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -170,15 +170,6 @@ spec: podSpec: containers: - args: - - | - : "${GROVE_PCSG_NAME:?missing GROVE_PCSG_NAME}" "${GROVE_PCSG_INDEX:?missing GROVE_PCSG_INDEX}" "${GROVE_HEADLESS_SERVICE:?missing GROVE_HEADLESS_SERVICE}" - sed -e "s|\${GROVE_PCSG_NAME}|${GROVE_PCSG_NAME}|g" -e "s|\${GROVE_PCSG_INDEX}|${GROVE_PCSG_INDEX}|g" -e "s|\${GROVE_HEADLESS_SERVICE}|${GROVE_HEADLESS_SERVICE}|g" "$1" > "$2" - shift 2 - exec "$@" - - -- - - /configs/datacenter.toml - - /tmp/datacenter.toml - - /bin/nova - --agent-env-vars - EXTRA_RUNTIME_KNOB - --agent-path @@ -192,7 +183,7 @@ spec: - --control-server-addr - $(POD_IP):51045 - --datacenter-config-filepath - - /tmp/datacenter.toml + - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -225,8 +216,7 @@ spec: - --store-backend - kubernetes command: - - /bin/sh - - -ec + - /bin/nova env: - name: DYN_COMPONENT value: lpx @@ -298,11 +288,12 @@ spec: name: shared-memory initContainers: - args: - - -c - - set -euo pipefail; cp /ssh-pk/private.key /tmp/dynamo-lpu-ssh/private.key; - chmod 600 /tmp/dynamo-lpu-ssh/private.key + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key command: - - /bin/bash + - /bin/install image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 name: prepare-ssh-key resources: {} @@ -367,91 +358,9 @@ spec: podSpec: containers: - args: - - -c - - "set -euo pipefail\n\nrun_lpu_system_init() {\n\tif [[ ! -e /dev/fd - ]]; then\n\t\tln -svf /proc/self/fd /dev/fd || true\n\tfi\n\tmkdir -p - /tmp\n\tmkdir -p /var/run/dpdk/rte || true\n\tchmod 777 /var/run/dpdk/rte - || true\n\tulimit -l unlimited || true\n\n\tif command -v lpu-system-init - >/dev/null 2>&1; then\n\t\tlocal init_ok=\"unknown\"\n\t\tlocal init_status=0\n\t\tfor - attempt in $(seq 1 2); do\n\t\t\techo \"Running lpu-system-init (attempt - ${attempt})...\"\n\t\t\tlocal init_output\n\t\t\tlocal -a init_args=(--ci-precommand)\n\t\t\tif - (( attempt == 2 )); then\n\t\t\t\tinit_args+=(--ci-remediation-commands)\n\t\t\tfi\n\t\t\tset - +e\n\t\t\tinit_output=\"$(lpu-system-init \"${init_args[@]}\" 2>&1)\"\n\t\t\tinit_status=$?\n\t\t\tset - -e\n\t\t\techo \"${init_output}\"\n\t\t\tinit_ok=\"$(echo \"${init_output}\" - | grep -o '\"ok\": *[a-z]*' | tail -1 | grep -o 'true\\|false' || echo - \"unknown\")\"\n\t\t\tif (( init_status == 0 )) && [[ \"${init_ok}\" - != \"false\" ]]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\tif (( attempt - == 1 )); then\n\t\t\t\tif (( init_status != 0 )); then\n\t\t\t\t\techo - \"lpu-system-init exited with status ${init_status}; retrying with remediation\" - >&2\n\t\t\t\telse\n\t\t\t\t\techo \"lpu-system-init reported ok=false; - retrying with remediation\" >&2\n\t\t\t\tfi\n\t\t\tfi\n\t\tdone\n\t\tif - (( init_status != 0 )) || [[ \"${init_ok}\" == \"false\" ]]; then\n\t\t\techo - \"lpu-system-init failed after 2 attempts; aborting\" >&2\n\t\t\texit - 1\n\t\tfi\n\telse\n\t\techo \"lpu-system-init not found; continuing - without pre-init\" >&2\n\tfi\n}\n\nset -euo pipefail\n\nLPU_CONFIG_DIR=\"${LPU_CONFIG_DIR:-/configs}\"\nCLIQUE_POD_INDEX=\"${GROVE_PCLQ_POD_INDEX:-}\"\nif - ! [[ \"${CLIQUE_POD_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"GROVE_PCLQ_POD_INDEX - must be a non-negative integer, got ${CLIQUE_POD_INDEX:-}\" >&2\n\texit - 1\nfi\n\nreadarray -t PARTITION_NODE_COUNTS < \"${LPU_CONFIG_DIR}/nodes_per_partition\"\nreadarray - -t PARTITION_NODE_OFFSETS < \"${LPU_CONFIG_DIR}/partition_node_offsets\"\nif - (( ${#PARTITION_NODE_COUNTS[@]} == 0 || ${#PARTITION_NODE_COUNTS[@]} - != ${#PARTITION_NODE_OFFSETS[@]} )); then\n\techo \"nodes_per_partition - and partition_node_offsets must contain the same non-zero number of - rows\" >&2\n\texit 1\nfi\n\nif [[ -f \"${LPU_CONFIG_DIR}/partition_models\" - ]]; then\n\treadarray -t PARTITION_MODELS < \"${LPU_CONFIG_DIR}/partition_models\"\n\treadarray - -t PARTITION_INDICES < \"${LPU_CONFIG_DIR}/partition_indices\"\n\tif - [[ -z \"${LPU_MODEL_NAME:-}\" ]]; then\n\t\techo \"LPU_MODEL_NAME is - required when partition_models is configured\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} != ${#PARTITION_NODE_COUNTS[@]} || ${#PARTITION_INDICES[@]} - != ${#PARTITION_NODE_COUNTS[@]} )); then\n\t\techo \"partition_models - and partition_indices must match the partition config row count\" >&2\n\t\texit - 1\n\tfi\nelse\n\tPARTITION_MODELS=()\n\tPARTITION_INDICES=()\nfi\n\nCONFIG_PARTITION_INDEX=\"\"\nLOGICAL_PARTITION_INDEX=\"\"\nPARTITION_RANK=\"\"\nNODE_COUNT=\"\"\nNODE_OFFSET=\"\"\nPARTITION_MATCH_COUNT=0\nfor - i in \"${!PARTITION_NODE_COUNTS[@]}\"; do\n\trow_node_count=\"${PARTITION_NODE_COUNTS[$i]}\"\n\trow_node_offset=\"${PARTITION_NODE_OFFSETS[$i]}\"\n\tif - ! [[ \"${row_node_count}\" =~ ^[1-9][0-9]*$ && \"${row_node_offset}\" - =~ ^[0-9]+$ ]]; then\n\t\techo \"Invalid partition row ${i}: node count - ${row_node_count}, node offset ${row_node_offset}\" >&2\n\t\texit 1\n\tfi\n\tif - (( ${#PARTITION_MODELS[@]} > 0 )) && [[ \"${PARTITION_MODELS[$i]}\" - != \"${LPU_MODEL_NAME}\" ]]; then\n\t\tcontinue\n\tfi\n\tif (( CLIQUE_POD_INDEX - < row_node_offset || CLIQUE_POD_INDEX >= row_node_offset + row_node_count - )); then\n\t\tcontinue\n\tfi\n\n\tCONFIG_PARTITION_INDEX=\"${i}\"\n\tLOGICAL_PARTITION_INDEX=\"${PARTITION_INDICES[$i]:-${i}}\"\n\tPARTITION_RANK=\"$((CLIQUE_POD_INDEX - - row_node_offset))\"\n\tNODE_COUNT=\"${row_node_count}\"\n\tNODE_OFFSET=\"${row_node_offset}\"\n\tPARTITION_MATCH_COUNT=$((PARTITION_MATCH_COUNT - + 1))\ndone\n\nif (( PARTITION_MATCH_COUNT != 1 )); then\n\techo \"Grove - pod index ${CLIQUE_POD_INDEX} matched ${PARTITION_MATCH_COUNT} partition - rows for model ${LPU_MODEL_NAME:-}\" >&2\n\texit 1\nfi\nif - ! [[ \"${LOGICAL_PARTITION_INDEX}\" =~ ^[0-9]+$ ]]; then\n\techo \"Invalid - logical partition index ${LOGICAL_PARTITION_INDEX} for config row ${CONFIG_PARTITION_INDEX}\" - >&2\n\texit 1\nfi\n\n# Preserve legacy Agent names from Dynamo's runtime - partition mapping.\nexport PARTITION_ID=\"${LOGICAL_PARTITION_INDEX}\"\nexport - RANK_IN_PARTITION=\"${PARTITION_RANK}\"\n\nHOME=\"${HOME:-/root}\"\n\nif - [[ \"${LPU_RUN_SYSTEM_INIT:-true}\" == \"true\" ]]; then\n\trun_lpu_system_init\nfi\n\nulimit - -l unlimited || true\n\nmkdir -p $HOME/.ssh $HOME/.ssh/host_keys\nls - -la /ssh-pk/\ncp /ssh-pk/private.key $HOME/.ssh/id_rsa\ncp /ssh-pk/private.key.pub - $HOME/.ssh/id_rsa.pub\ncp /ssh-pk/private.key.pub $HOME/.ssh/authorized_keys\nchmod - 600 $HOME/.ssh/id_rsa $HOME/.ssh/authorized_keys\nchmod 644 $HOME/.ssh/id_rsa.pub\nprintf - 'Host *\\nIdentityFile '$HOME'/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort - 2222\\n' > $HOME/.ssh/config\n\ntest -f \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" - || ssh-keygen -t rsa -f $HOME/.ssh/host_keys/ssh_host_rsa_key -N ''\ntest - -f \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" || ssh-keygen -t ecdsa - -f $HOME/.ssh/host_keys/ssh_host_ecdsa_key -N ''\ntest -f \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\" - || ssh-keygen -t ed25519 -f $HOME/.ssh/host_keys/ssh_host_ed25519_key - -N ''\n\n\nif command -v setcap >/dev/null 2>&1; then\n\tfor binary - in /bin-unwrapped/agent_v2.unwrapped /bin-unwrapped/agent_v2_perfetto_trace.unwrapped - /bin/agent_v2 /bin/agent_v2_perfetto_trace; do\n\t\tif [[ -e \"$binary\" - ]]; then\n\t\t\techo \"Lifting capabilities of $binary\"\n\t\t\tsetcap - cap_sys_nice,cap_sys_resource,cap_sys_admin,cap_dac_override,cap_sys_rawio,cap_sys_ptrace=+ep - \"$(realpath \"$binary\")\" || true\n\t\tfi\n\tdone\nfi\n\n# Pass down - information from the k8s downward API into SSH sessions.\nSETENV_LINE=\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_NAME=\\\"$POD_NAME\\\"\"\nSETENV_LINE=\"${SETENV_LINE} K8S_POD_NAMESPACE=\\\"$POD_NAMESPACE\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - K8S_POD_UID=\\\"$POD_UID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} PARTITION_ID=\\\"$PARTITION_ID\\\"\"\nSETENV_LINE=\"${SETENV_LINE} - RANK_IN_PARTITION=\\\"$RANK_IN_PARTITION\\\"\"\nif [[ -n \"${LPU_MODEL_NAME:-}\" - ]]; then\n\tSETENV_LINE=\"${SETENV_LINE} LPU_MODEL_NAME=\\\"$LPU_MODEL_NAME\\\"\"\nfi\nexec - /bin/sshd -D -e \\\n\t-o \"PidFile none\" \\\n\t-o \"Port 2222\" \\\n\t-o - \"AuthorizedKeysFile $HOME/.ssh/authorized_keys\" \\\n\t-o \"StrictModes - no\" \\\n\t-o PermitRootLogin=yes \\\n\t-o PasswordAuthentication=no - \\\n\t-o PubkeyAuthentication=yes \\\n\t-o \"SetEnv $SETENV_LINE\" \\\n\t-h - \"$HOME/.ssh/host_keys/ssh_host_rsa_key\" \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ecdsa_key\" - \\\n\t-h \"$HOME/.ssh/host_keys/ssh_host_ed25519_key\"\n" + - --partition-metadata command: - - /bin/bash + - /bin/quasar-entrypoint env: - name: DYN_COMPONENT value: lpx From b9f8c988086764c755dffc9e013d23dedf7676fe Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 16 Sep 2026 15:41:28 +0000 Subject: [PATCH 071/196] chore(codeowners): add LPX co-ownership Signed-off-by: Andrew Paprotsky --- .github/codeowners/areas.yaml | 13 +++++++++++++ CODEOWNERS | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 792590248e98..15abd20e5a08 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -425,6 +425,11 @@ areas: - deploy/utils/tests/ - examples/deployments/ - tests/deploy/ +- label: lpx + github_team: '@ai-dynamo/dynamo-lpx-codeowners' + path_globs: + - deploy/operator/internal/controller/lpx/ + - deploy/operator/internal/dynamo/lpx/ - label: gms github_team: '@ai-dynamo/dynamo-gms-codeowners' path_globs: @@ -980,6 +985,14 @@ shared: owners: - operator - observability +- glob: deploy/operator/internal/controller/lpx/ + owners: + - operator + - lpx +- glob: deploy/operator/internal/dynamo/lpx/ + owners: + - operator + - lpx - glob: deploy/operator/internal/controller/*checkpoint* owners: - operator diff --git a/CODEOWNERS b/CODEOWNERS index 8c3d5ec3f670..a1e699cb20e8 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,6 +18,7 @@ # frontend @ai-dynamo/dynamo-frontend-codeowners # gms @ai-dynamo/dynamo-gms-codeowners # kv-memory @ai-dynamo/dynamo-kv-memory-codeowners +# lpx @ai-dynamo/dynamo-lpx-codeowners # multimodal @ai-dynamo/dynamo-multimodal-codeowners # observability @ai-dynamo/dynamo-observability-codeowners # operator @ai-dynamo/dynamo-operator-codeowners @@ -45,6 +46,7 @@ # @ai-dynamo/dynamo-frontend-codeowners # @ai-dynamo/dynamo-gms-codeowners # @ai-dynamo/dynamo-kv-memory-codeowners +# @ai-dynamo/dynamo-lpx-codeowners # @ai-dynamo/dynamo-multimodal-codeowners # @ai-dynamo/dynamo-observability-codeowners # @ai-dynamo/dynamo-operator-codeowners @@ -539,6 +541,7 @@ requirements.txt /lib/llm/src/block_manager/layout.rs @ai-dynamo/dynamo-kv-memory-codeowners /tests/fault_tolerance/cancellation/ @ai-dynamo/dynamo-fault-tolerance-codeowners /deploy/operator/internal/checkpoint/ @ai-dynamo/dynamo-gms-codeowners +/deploy/operator/internal/dynamo/lpx/ @ai-dynamo/dynamo-lpx-codeowners /dev/observability/prometheus-xpu.yml @ai-dynamo/dynamo-xpu-codeowners /lib/llm/src/block_manager/connector/ @ai-dynamo/dynamo-kv-memory-codeowners /lib/llm/src/block_manager/offload.rs @ai-dynamo/dynamo-kv-memory-codeowners @@ -556,6 +559,7 @@ requirements.txt /lib/llm/src/block_manager/controller.rs @ai-dynamo/dynamo-kv-memory-codeowners /components/src/dynamo/common/multimodal/ @ai-dynamo/dynamo-multimodal-codeowners /components/src/dynamo/trtllm/multimodal/ @ai-dynamo/dynamo-multimodal-codeowners +/deploy/operator/internal/controller/lpx/ @ai-dynamo/dynamo-lpx-codeowners /lib/llm/src/block_manager/distributed.rs @ai-dynamo/dynamo-kv-memory-codeowners /lib/llm/src/block_manager/kv_consolidator/ @ai-dynamo/dynamo-kv-memory-codeowners /tests/serve/multimodal_profiles/vllm_xpu.py @ai-dynamo/dynamo-xpu-codeowners @@ -646,6 +650,7 @@ requirements.txt /lib/llm/src/http/service/metrics.rs @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-observability-codeowners /container/deps/requirements.vllm.txt @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-backend-vllm-codeowners /deploy/operator/internal/checkpoint/ @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-operator-codeowners +/deploy/operator/internal/dynamo/lpx/ @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-lpx-codeowners /lib/llm/src/preprocessor/mm_routing/ @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-multimodal-codeowners @ai-dynamo/dynamo-router-codeowners /lib/llm/src/protocols/openai/audios/ @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-multimodal-codeowners /lib/llm/src/protocols/openai/images/ @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-diffusion-codeowners @@ -680,6 +685,7 @@ requirements.txt /components/src/dynamo/trtllm/snapshot.py @ai-dynamo/dynamo-backend-trtllm-codeowners @ai-dynamo/dynamo-gms-codeowners /components/src/dynamo/vllm/cache_info.py @ai-dynamo/dynamo-backend-vllm-codeowners @ai-dynamo/dynamo-kv-memory-codeowners /container/deps/requirements.frontend.txt @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-frontend-codeowners +/deploy/operator/internal/controller/lpx/ @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-lpx-codeowners /lib/kv-router/src/services/shared_cache/ @ai-dynamo/dynamo-router-codeowners @ai-dynamo/dynamo-kv-memory-codeowners /lib/kv-router/src/zmq_wire/extra_keys.rs @ai-dynamo/dynamo-router-codeowners @ai-dynamo/dynamo-multimodal-codeowners /lib/runtime/src/metrics/frontend_perf.rs @ai-dynamo/dynamo-observability-codeowners @ai-dynamo/dynamo-frontend-codeowners From 347e4dc6f1fb26e0ea23961405ce2f6ef4959d71 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:48:36 +0300 Subject: [PATCH 072/196] refactor(helm): align Grove configuration with upstream (#508) Signed-off-by: Andrew Paprotsky (cherry picked from commit 5a4e4c867d8cba98ef71a0b5e24933e26d24d90d) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 4 ++-- .../platform/templates/validate-values.yaml | 20 ------------------- .../platform/tests/lpx_config_test.yaml | 9 --------- deploy/helm/charts/platform/values.yaml | 3 --- deploy/operator/internal/features/lpx.go | 2 +- 5 files changed, 3 insertions(+), 35 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index e66df58c8186..e2d6d39329c3 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -162,8 +162,8 @@ Kubernetes: `>=1.30.0-0` | global.kai-scheduler.install | bool | `false` | Whether this chart should install the bundled kai-scheduler subchart. When true, deploys kai-scheduler and its CRDs. Integration is automatically enabled. NOTE: For production environments, it is recommended to install kai-scheduler separately. | | global.kai-scheduler.enabled | bool | `false` | Whether to enable Kai Scheduler integration (queue creation, schedulerName injection). Set to true when kai-scheduler is available in the cluster (installed externally). Automatically enabled when install=true. The operator uses this to decide whether to inject schedulerName and queue labels into pod templates. | | global.volcano-scheduler.enabled | bool | `false` | EXPERIMENTAL: Whether to enable Volcano scheduler integration for Grove PodCliqueSets. Set to true when Volcano is available in the cluster and Grove is configured with Volcano scheduler support. The operator uses this to inject schedulerName and map nvidia.com/volcano-queue to Grove's Volcano queue annotation. | -| global.grove.install | bool | `false` | Whether this chart should install the bundled Grove subchart. When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. The bundled version cannot be used with LPX; LPX requires an externally installed scheduler-scoped Grove build that supports disjoint schedulers in one PodCliqueSet. NOTE: For production environments, it is recommended to install Grove separately. | -| global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). LPX users must provide the scheduler-scoped Grove build and enable its lpx-scheduler profile. Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | +| global.grove.install | bool | `false` | Whether this chart should install the bundled Grove subchart. When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. NOTE: For production environments, it is recommended to install Grove separately. | +| global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | | global.snapshot.install | bool | `false` | Whether this chart should install the bundled Snapshot subchart (github.com/ai-dynamo/snapshot): its operator, node agent, and CRDs. NOTE: For production environments, it is recommended to install Snapshot separately. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | diff --git a/deploy/helm/charts/platform/templates/validate-values.yaml b/deploy/helm/charts/platform/templates/validate-values.yaml index c6bbc901cffa..63154dc1504e 100644 --- a/deploy/helm/charts/platform/templates/validate-values.yaml +++ b/deploy/helm/charts/platform/templates/validate-values.yaml @@ -146,26 +146,6 @@ kai-scheduler: {{- end }} {{- end }} -{{- if and - .Values.global.grove.install - (index .Values "dynamo-operator").lpx.enabled -}} - {{- fail ` -CONFIGURATION ERROR: The bundled Grove chart does not support -the scheduler-scoped PodGangs required by LPX workloads. - -Install the scheduler-scoped Grove build separately, then configure: - - global: - grove: - install: false - enabled: true - dynamo-operator: - lpx: - enabled: true -` }} -{{- end }} - {{/* NOTE: dynamo-operator subchart values (e.g. dynamo-operator.nats.enabled, dynamo-operator.webhook.enabled) are validated in the operator subchart's diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index 765ad2972ed5..9018857a2db8 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -8,15 +8,6 @@ templates: - charts/dynamo-operator/templates/manager-rbac.yaml - charts/dynamo-operator/templates/validate-values.yaml tests: - - it: preserves the bundled Grove compatibility error - template: templates/validate-values.yaml - set: - global.grove.install: true - dynamo-operator.lpx.enabled: true - asserts: - - failedTemplate: - errorPattern: "bundled Grove chart does not support" - - it: rejects LPX without Grove integration template: charts/dynamo-operator/templates/validate-values.yaml set: diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 52c4f312606e..d844e901e28f 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -49,13 +49,10 @@ global: grove: # -- Whether this chart should install the bundled Grove subchart. # When true, deploys the Grove operator cluster-wide. Integration is automatically enabled. - # The bundled version cannot be used with LPX; LPX requires an externally installed - # scheduler-scoped Grove build that supports disjoint schedulers in one PodCliqueSet. # NOTE: For production environments, it is recommended to install Grove separately. install: false # -- Whether to enable Grove integration (multinode orchestration via PodCliqueSets). # Set to true when Grove is available in the cluster (installed externally). - # LPX users must provide the scheduler-scoped Grove build and enable its lpx-scheduler profile. # Automatically true when install=true. The operator uses this to decide whether to # create PodCliqueSets for multinode deployments. enabled: false diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go index 59a26fed6b3e..337747c9e1b1 100644 --- a/deploy/operator/internal/features/lpx.go +++ b/deploy/operator/internal/features/lpx.go @@ -40,7 +40,7 @@ func resolveLPX(ctx context.Context, config *rest.Config) (bool, error) { return false, err } - // LPX also watches scheduler-scoped PodGangs, which external Grove installs may omit. + // Discover the PodGang resource before starting its watch. resource = groveschedulerv1alpha1.SchemeGroupVersion.WithResource("podgangs") available, err = detectAPIAvailability(ctx, config, resource.Group, resource.Version, resource.Resource) if err != nil { From c2711ce94b52eb622cb1e4b3a49c1a2699e63606 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:50:26 +0300 Subject: [PATCH 073/196] fix(lpx): retire changed intent before Grove updates (#509) Signed-off-by: Andrew Paprotsky (cherry picked from commit 9085911d4ce1c6e568594ca507b0e34907cce1cf) Signed-off-by: Andrew Paprotsky --- .../controller/lpx/attempt_deadline_test.go | 32 +++-- .../internal/controller/lpx/controller.go | 17 ++- .../controller/lpx/controller_test.go | 117 ++++++++++++++++++ .../internal/controller/lpx/preflight.go | 51 +++++++- 4 files changed, 203 insertions(+), 14 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 7ac3de7babdb..1a4bb03efab5 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -402,15 +402,18 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { name string phase lpxv1alpha1.RequestPhase replacing, keepClique bool + settings bool }{ {name: "scale-in", phase: lpxv1alpha1.RequestPhasePending}, {name: "completed-scale-in", phase: lpxv1alpha1.RequestPhaseBound}, {name: "native-replacement", phase: lpxv1alpha1.RequestPhasePending, replacing: true}, {name: "native-child-still-deleting", phase: lpxv1alpha1.RequestPhasePending, replacing: true, keepClique: true}, {name: "completed-native-replacement", phase: lpxv1alpha1.RequestPhaseBound, replacing: true}, + {name: "settings-edit", phase: lpxv1alpha1.RequestPhasePending, settings: true}, + {name: "completed-settings-edit", phase: lpxv1alpha1.RequestPhaseBound, settings: true}, } { t.Run(tc.name, func(t *testing.T) { - replacing := tc.replacing + replacing := tc.replacing || tc.settings t.Log("Publish two engines, with only the second engine recorded in the scheduling batch") ctx := t.Context() deployment, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -433,7 +436,11 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { deadline := time.Now().UTC().Truncate(time.Second).Add(-time.Minute) deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: deadlineTestAttempt(deployment, pending, deadline)} source.Spec.Scheduling = deadlineTestScheduling() - if replacing { + if tc.settings { + source.Spec.Components[0].LPX.Settings.Raw = []byte(`{"prop_sync":false}`) + deployment.Generation++ + running.Finalizers = []string{"scheduler.example/cleanup"} + } else if replacing { clique := findLPXTestClique(t, objects, desired.plan.ForReplica(1).Agents[0].CliqueName) clique.Finalizers = []string{"grove.example/cleanup"} clique.DeletionTimestamp = ptr.To(metav1.Now()) @@ -454,22 +461,20 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { require.NoError(t, err) require.NoError(t, r.Get(ctx, key, deployment)) attempt := currentLPXAttemptStatus(deployment) + require.Equal(t, replacing && tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) if replacing { require.Len(t, attempt.Requests, 1) require.Equal(t, pending.UID, attempt.Requests[0].UID) require.Empty(t, attempt.Requests[0].AttemptDigest) - if tc.phase == lpxv1alpha1.RequestPhasePending { - require.NotNil(t, attempt.ExceededAt) - } else { + if tc.phase == lpxv1alpha1.RequestPhaseBound { require.NotNil(t, attempt.DisarmedAt) } } else { require.Equal(t, []nvidiacomv1beta1.LPXAttemptRequestStatus{}, attempt.Requests) - require.Nil(t, attempt.ExceededAt) require.NotNil(t, attempt.DisarmedAt) } require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) - if replacing && !tc.keepClique { + if tc.replacing && !tc.keepClique { t.Log("Grove can finish deleting the old clique before scheduler retirement resumes") clique := &grovev1alpha1.PodClique{} cliqueKey := client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.ForReplica(1).Agents[0].CliqueName} @@ -507,15 +512,17 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { } require.NoError(t, r.Get(ctx, key, deployment)) attempt = currentLPXAttemptStatus(deployment) - if replacing && tc.phase == lpxv1alpha1.RequestPhaseBound { + require.Equal(t, replacing && tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) + if tc.replacing && tc.phase == lpxv1alpha1.RequestPhaseBound { require.Nil(t, attempt.DeadlineAt) } else { require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) } } if replacing { - require.Equal(t, tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) - require.Nil(t, attempt.DisarmedAt) + if tc.replacing { + require.Nil(t, attempt.DisarmedAt) + } retiring := getLPXRequest(t, ctx, r.Client, deployment.Namespace, pending.Name) require.False(t, retiring.DeletionTimestamp.IsZero()) retiring.Finalizers = []string{lpxAttemptRecordingFinalizer} @@ -524,15 +531,16 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { require.NoError(t, err) require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pending), &lpxv1alpha1.LPUPipelineRequest{}))) } else { - require.Nil(t, attempt.ExceededAt) require.NotNil(t, attempt.DisarmedAt) } stored := getLPXRequest(t, ctx, r.Client, deployment.Namespace, running.Name) require.Equal(t, running.UID, stored.UID) - require.True(t, stored.DeletionTimestamp.IsZero()) + require.Equal(t, tc.settings, !stored.DeletionTimestamp.IsZero()) pcs := &grovev1alpha1.PodCliqueSet{} require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) require.Equal(t, types.UID("pcs-uid"), pcs.UID) + require.Equal(t, int32(1), pcs.Spec.Replicas) + require.True(t, pcs.DeletionTimestamp.IsZero()) }) } } diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 12644f10b04f..5bef39269f48 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "maps" + "slices" "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -244,8 +245,9 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment, selected *lpxMaterializing) (reconcileOutcome, ctrl.Result, error) { state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} + // Pair retirement with the authoritative PCS, not a cached roster predating current requests. pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { + if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { if !apierrors.IsNotFound(err) { return state, ctrl.Result{}, err } @@ -258,6 +260,19 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { return state, ctrl.Result{}, err } + // Retire obsolete scheduler intent before changing its runtime or engine count. + rosterChanged := pcs != nil && metav1.IsControlledBy(pcs, deployment) && !slices.EqualFunc( + pcs.Spec.Template.PodCliqueScalingGroupConfigs, desired.Spec.Template.PodCliqueScalingGroupConfigs, + func(a, b grovev1alpha1.PodCliqueScalingGroupConfig) bool { + return a.Name == b.Name && slices.Equal(a.CliqueNames, b.CliqueNames) + }) + transition, err := r.retireChangedLPXRequests(ctx, deployment, selected, rosterChanged) + if err != nil { + return state, ctrl.Result{}, err + } + if transition != nil { + return lpxResult(transition), projectLPXLifecycleStatus(deployment, transition), nil + } for _, resource := range resources { if err := r.syncLPXResource(ctx, deployment, resource); err != nil { return state, ctrl.Result{}, err diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 75289e491d3d..a144ad5ecf4c 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -6,6 +6,7 @@ package lpx import ( "context" "errors" + "slices" "testing" "time" @@ -88,6 +89,122 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { } } +func TestLPXIntentChangesRetireRequestsBeforeGroveWrites(t *testing.T) { + for _, tc := range []struct { + name string + replicas int32 + drafts int32 + retire []int + wantWrites []string + }{ + {name: "settings", replicas: 2, retire: []int{0, 1}, wantWrites: []string{"pcs"}}, + {name: "scale in", replicas: 1, retire: []int{1}, wantWrites: []string{"pcs", "scale"}}, + {name: "scale out", replicas: 3, wantWrites: []string{"pcs", "scale"}}, + {name: "metadata", replicas: 2}, + {name: "stale cached PCS", replicas: 2}, + {name: "image", replicas: 2, wantWrites: []string{"pcs"}}, + {name: "add draft", replicas: 1, drafts: 3, retire: []int{0, 1, 2}, wantWrites: []string{"pcs"}}, + {name: "remove draft", replicas: 1, drafts: 1, retire: []int{0, 1, 2}, wantWrites: []string{"pcs"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Publish the engines and hold their scheduler cleanup finalizers") + var child *v1alpha1.LPXGraphDeployment + var source *v1beta1.DynamoGraphDeployment + var registry *lpx.ModelRegistry + if tc.drafts != 0 { + child, source, registry = newLPXSpecDecodeTestDGD(t) + } else { + child, source, registry = newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + } + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) + require.NoError(t, err) + requests := make([]*lpxv1alpha1.LPUPipelineRequest, len(selected.requests)) + for i, desired := range selected.requests { + requests[i] = getLPXRequest(t, t.Context(), r.Client, child.Namespace, desired.requestName) + requests[i].Finalizers = []string{"test.example/scheduler"} + require.NoError(t, r.Update(t.Context(), requests[i])) + } + var wantRetired []string + for _, i := range tc.retire { + wantRetired = append(wantRetired, requests[i].Name) + } + + t.Log("Select the edited source while keeping the original Grove children") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + component := lpx.ServingComponent(source) + component.Replicas = ptr.To(tc.replicas) + switch tc.name { + case "settings": + component.LPX.Settings.Raw = []byte(`{"prop_sync":false}`) + case "metadata": + source.Annotations["test.example/unrelated"] = "updated" + case "image": + component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "runtime:next" + case "add draft", "remove draft": + source.GetComponentByName("draft").Replicas = ptr.To(tc.drafts) + } + source.Generation++ + require.NoError(t, r.Update(t.Context(), source)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(t.Context(), child)) + selected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) + require.Nil(t, rejected) + + t.Log("Affected requests must be deleting before either Grove update, without awaiting finalizers") + var retired, writes []string + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + err := delegated.Get(ctx, key, object, opts...) + if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok && err == nil && tc.name == "stale cached PCS" { + pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames = []string{"stale-agent"} + } + return err + }, + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + retired = append(retired, object.GetName()) + } + return delegated.Delete(ctx, object, opts...) + }, + Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + require.ElementsMatch(t, wantRetired, retired, "retirement must precede PCS updates") + writes = append(writes, "pcs") + } + return delegated.Update(ctx, object, opts...) + }, + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.ElementsMatch(t, wantRetired, retired, "retirement must precede scale updates") + writes = append(writes, subresource) + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + }) + _, _, err = r.reconcileWorkload(t.Context(), child, source, selected) + require.NoError(t, err) + require.ElementsMatch(t, wantRetired, retired) + require.Equal(t, tc.wantWrites, writes) + + t.Log("The PCS and every unchanged scheduler request retain their identity") + pcs := findLPXTestPodCliqueSet(t, objects) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.EqualValues(t, "pcs-uid", pcs.UID) + require.Equal(t, int32(1), pcs.Spec.Replicas) + for _, before := range requests { + after := getLPXRequest(t, t.Context(), r.Client, child.Namespace, before.Name) + require.Equal(t, before.UID, after.UID) + require.Equal(t, before.Finalizers, after.Finalizers) + require.Equal(t, slices.Contains(wantRetired, before.Name), !after.DeletionTimestamp.IsZero()) + } + }) + } +} + func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { t.Log("Publish a speculative engine using its frozen child identity") child, source, registry := newLPXSpecDecodeTestDGD(t) diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index 0633c592ffa7..98c66aa384c4 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -20,10 +20,12 @@ package lpx import ( "context" "fmt" + "maps" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -90,6 +92,53 @@ func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( return nil, rejected, nil } - // Grove accepts the desired spec before individual engine requests are reconciled. return selectedLPX, nil, nil } + +// retireChangedLPXRequests withdraws changed intent before Grove can consume it. +// Deleting requests may still need Grove to remove their Pods to finish cleanup. +func (r *graphReconciler) retireChangedLPXRequests(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, rosterChanged bool) (*lpxDeadlineTransition, error) { + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil || len(requests) == 0 { + return nil, err + } + desiredByName := make(map[string]*lpxModelMaterializing, len(desired.requests)) + for index := range desired.requests { + request := &desired.requests[index] + desiredByName[request.requestName] = request + } + + // Cancel removed batch members before deletion can be mistaken for a timeout. + if attempt := currentLPXAttemptStatus(deployment); attempt != nil && attempt.ExceededAt == nil { + currents := lpxRequestsByName(requests) + maps.DeleteFunc(currents, func(name string, _ *lpxv1alpha1.LPUPipelineRequest) bool { return desiredByName[name] == nil }) + if len(currents) != len(requests) { + want := desiredLPXAttemptStatus(desired, deployment.Generation, attempt.PodCliqueSetUID) + if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, want, currents); transition != nil { + return transition, nil + } + } + } + + // Keep unchanged engines, including those surviving scale and native image rollouts. + for index := range requests { + request := &requests[index] + wanted := desiredByName[request.Name] + if !request.DeletionTimestamp.IsZero() || !rosterChanged && wanted != nil && + wanted.modelProjection.Digest().String() == request.Annotations[dynamolpx.WorkloadDigestAnnotation] { + continue + } + classification, err := r.retireLPXEngineRequest(ctx, deployment, request, "The engine's scheduler intent changed or was removed", true) + if err != nil { + return nil, err + } + if transition, recording := classification.(*lpxDeadlineTransition); recording { + if _, completed, _ := exactLPXAttemptState(currentLPXAttemptStatus(deployment), lpxRequestsByName(requests)); completed { + disarmedAt := metav1.Now() + transition.attempt.DisarmedAt = &disarmedAt + } + return transition, nil + } + } + return nil, nil +} From 9f26fd8948133bd9469b3c7a846c91121dec32ce Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:50:26 +0300 Subject: [PATCH 074/196] fix(lpx): report source watch lookup failures (#510) Signed-off-by: Andrew Paprotsky (cherry picked from commit ef67283937e2d9d690eb2ba9e6489f1ad622302d) Signed-off-by: Andrew Paprotsky --- .../operator/internal/controller/lpx/watch.go | 1 + .../internal/controller/lpx/watch_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index a1bce74bc724..92f8a3abe5a2 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -97,6 +97,7 @@ func (r *graphReconciler) mapLPXSourceToRequests(ctx context.Context, obj client deployments := &nvidiacomv1alpha1.LPXGraphDeploymentList{} ownerKey := obj.GetName() + "/" + string(obj.GetUID()) if err := r.List(ctx, deployments, client.InNamespace(obj.GetNamespace()), client.MatchingFields{lpxSourceOwnerIndex: ownerKey}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX deployments for source", "source", client.ObjectKeyFromObject(obj), "sourceUID", obj.GetUID()) return nil } requests := make([]ctrl.Request, 0, len(deployments.Items)) diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 5d92ffbfd5de..1643a0a93c2c 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -6,7 +6,9 @@ package lpx import ( + "bytes" "context" + "errors" "testing" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -32,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -73,6 +76,21 @@ func TestLPXSourceWatchMapsControllerOwnedMaterializations(t *testing.T) { reconciler := &graphReconciler{Client: kube} require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(owned)}}, reconciler.mapLPXSourceToRequests(t.Context(), source)) + + t.Log("Report failed owner lookups with the exact source identity, without inventing a child name") + lookupErr := errors.New("owner index unavailable") + reconciler.Client = interceptor.NewClient(kube, interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return lookupErr + }, + }) + var logs bytes.Buffer + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.WriteTo(&logs))) + require.Empty(t, reconciler.mapLPXSourceToRequests(ctx, source)) + require.Contains(t, logs.String(), lookupErr.Error()) + require.Contains(t, logs.String(), `"name":"source"`) + require.Contains(t, logs.String(), `"namespace":"workloads"`) + require.Contains(t, logs.String(), `"sourceUID":"source-uid"`) } func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { From d5ae24d489482a8e8c98111d7f59211d29016642 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:50:27 +0300 Subject: [PATCH 075/196] fix(lpx): propagate datacenter encoding errors (#512) Signed-off-by: Andrew Paprotsky (cherry picked from commit a16c8d02ba7f546a95d272c256f8eea195d90add) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/lpu_configmap.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 866cce35618c..fc20abc9bed6 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -86,10 +86,12 @@ func renderLPUConfigMap( } } var datacenterTOML strings.Builder - _ = toml.NewEncoder(&datacenterTOML).Encode(map[string]any{ + if err := toml.NewEncoder(&datacenterTOML).Encode(map[string]any{ "model_base_paths": []string{modelStoragePath}, "datacenters": map[string]any{"racks": datacenterRacks}, - }) + }); err != nil { + return nil, fmt.Errorf("render datacenter.toml: %w", err) + } data := resolvedPartitionData(projections) if projections[0].pipeline == PipelineLPX { From e5366351e43bea3f263e671cf9d3e52ebd8f021b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:50:27 +0300 Subject: [PATCH 076/196] refactor(lpx): separate retirement from operational errors (#514) Signed-off-by: Andrew Paprotsky (cherry picked from commit 5d87cda9b816c73d1f327e06dc7c9fa433913ae9) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 10 ++- .../controller/lpx/controller_test.go | 3 +- .../operator/internal/controller/lpx/grove.go | 12 ++-- .../internal/controller/lpx/lifecycle.go | 65 +++++++------------ .../internal/controller/lpx/lifecycle_test.go | 64 +++++++++++++----- 5 files changed, 87 insertions(+), 67 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 5bef39269f48..416a3f42f316 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -278,15 +278,13 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a return state, ctrl.Result{}, err } } - synced, changed, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired) + synced, changed, retiring, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired) if err != nil { - var retiring *lpxRetiring - if errors.As(err, &retiring) { - state = lpxResult(retiring) - return state, projectLPXLifecycleStatus(deployment, retiring), nil - } return state, ctrl.Result{}, err } + if retiring != nil { + return lpxResult(retiring), projectLPXLifecycleStatus(deployment, retiring), nil + } if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { return state, ctrl.Result{}, err } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index a144ad5ecf4c..c646685f12a8 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -876,8 +876,9 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs) + _, _, retiring, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs) require.ErrorContains(t, err, "input revision") + require.Nil(t, retiring) allPCS := &grovev1alpha1.PodCliqueSetList{} require.NoError(t, r.List(t.Context(), allPCS)) require.Empty(t, allPCS.Items) diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index d87bf2f4d4ce..0b7a49067bce 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -33,15 +33,19 @@ func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( deployment *nvidiacomv1alpha1.LPXGraphDeployment, existingPodCliqueSet *grovev1alpha1.PodCliqueSet, desired *grovev1alpha1.PodCliqueSet, -) (*grovev1alpha1.PodCliqueSet, bool, error) { +) (*grovev1alpha1.PodCliqueSet, bool, *lpxRetiring, error) { // A missing observation permits only creation after the publication fence. if existingPodCliqueSet == nil { - if err := r.fenceLPXPublicationBeforeGroveSpecWrite(ctx, deployment, desired.Name); err != nil { - return nil, false, err + if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { + return nil, false, nil, err + } + retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, desired.Name, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") + if err != nil || retiring != nil { + return nil, false, retiring, err } } // Match normal PCS synchronization; Grove owns rollout and immutable-field validation. modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired, commoncontroller.WithPreservedListOrder()) - return synced, modified, err + return synced, modified, nil, err } diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index d1945bef7087..657b8f5ff606 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -123,9 +123,6 @@ type lpxRetiring struct { retirementReason string } -// Error carries retirement through Grove's error-shaped write boundary. -func (*lpxRetiring) Error() string { return "LPX publication retired before Grove spec write" } - func (*lpxRejected) lpxClassification() {} func (*lpxClosed) lpxClassification() {} func (*lpxOpen) lpxClassification() {} @@ -911,19 +908,20 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( retire := make(map[int32]bool) intentional := make(map[int32]bool) for replicaIndex, plan := range replicaPlans { - identity, incomplete, err := r.observeLPXGroveIdentity(ctx, deployment, desired, snapshot, plan, allAgentGroups) + identity, incomplete, drift, err := r.observeLPXGroveIdentity(ctx, deployment, desired, snapshot, plan, allAgentGroups) if err != nil { - var drift *lpxRetiring if errors.Is(err, errLPXEngineReplacing) { intentional[int32(replicaIndex)] = true incomplete = err.Error() - } else if errors.As(err, &drift) { - incomplete = drift.retirementReason } else { return nil, nil, nil, err } retire[int32(replicaIndex)] = true } + if drift != nil { + incomplete = drift.retirementReason + retire[int32(replicaIndex)] = true + } if incomplete != "" { if pending == nil { pending = &lpxClosed{incomplete: incomplete} @@ -1013,21 +1011,6 @@ func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassifica return (*lpxSchedulerObserved)(status) } -func (r *graphReconciler) fenceLPXPublicationBeforeGroveSpecWrite( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, -) error { - if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { - return err - } - retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, pcsName, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") - if err == nil && retiring != nil { - err = retiring - } - return err -} - func (r *graphReconciler) createPublishedLPXRequest( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -1168,7 +1151,7 @@ func (r *graphReconciler) observeLPXGroveIdentity( snapshot *lpxGroveSnapshot, plan *lpx.MaterializationPlan, allAgentGroups map[string]struct{}, -) (*lpxGroveIdentity, string, error) { +) (*lpxGroveIdentity, string, *lpxRetiring, error) { incomplete := snapshot.incomplete // Observe native retirement before gang availability or successor revision readiness. agentUIDs := make([]types.UID, 0, len(plan.Agents)) @@ -1180,14 +1163,14 @@ func (r *graphReconciler) observeLPXGroveIdentity( incomplete = fmt.Sprintf("Waiting for Agent PodClique %q", agent.CliqueName) continue } - return nil, "", err + return nil, "", nil, err } if !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup) || clique.UID == "" { incomplete = fmt.Sprintf("Waiting for owned Agent PodClique %q", agent.CliqueName) continue } if !clique.DeletionTimestamp.IsZero() { - return nil, "", errLPXEngineReplacing + return nil, "", nil, errLPXEngineReplacing } agentUIDs = append(agentUIDs, clique.UID) if !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || @@ -1203,7 +1186,7 @@ func (r *graphReconciler) observeLPXGroveIdentity( allAgentGroups, ) if wait != "" { - return nil, wait, nil + return nil, wait, nil, nil } identity := &lpxGroveIdentity{ @@ -1220,7 +1203,7 @@ func (r *graphReconciler) observeLPXGroveIdentity( } // A conductor needs complete identity; Cyborg can retain its reference while hashes converge. - cyborg, wait, err := r.observeLPXServingClique( + cyborg, wait, retiring, err := r.observeLPXServingClique( ctx, deployment, snapshot, @@ -1228,8 +1211,8 @@ func (r *graphReconciler) observeLPXGroveIdentity( desired, plan, ) - if err != nil || (wait != "" && cyborg == nil) { - return nil, wait, err + if err != nil || retiring != nil || (wait != "" && cyborg == nil) { + return nil, wait, retiring, err } identity.cyborgClique = cyborg if wait != "" { @@ -1237,7 +1220,7 @@ func (r *graphReconciler) observeLPXGroveIdentity( incomplete = wait } } - return identity, incomplete, nil + return identity, incomplete, nil, nil } func (r *graphReconciler) listLPXPublicationPodGangs( @@ -1334,7 +1317,7 @@ func (r *graphReconciler) observeLPXServingClique( ordinaryPodGangName string, desired *lpxMaterializing, plan *lpx.MaterializationPlan, -) (*lpxv1alpha1.CyborgPodCliqueReference, string, error) { +) (*lpxv1alpha1.CyborgPodCliqueReference, string, *lpxRetiring, error) { // Every supported plan has exactly one ordinary-scheduler serving clique. isCyborg := plan.CyborgClique != "" name, role := plan.ConductorClique, lpxv1alpha1.PodRoleConductor @@ -1349,9 +1332,9 @@ func (r *graphReconciler) observeLPXServingClique( key := types.NamespacedName{Namespace: deployment.Namespace, Name: name} if err := r.apiReader.Get(ctx, key, clique); err != nil { if apierrors.IsNotFound(err) { - return nil, fmt.Sprintf("Waiting for %s PodClique %q", waitingRoleName, name), nil + return nil, fmt.Sprintf("Waiting for %s PodClique %q", waitingRoleName, name), nil, nil } - return nil, "", err + return nil, "", nil, err } if clique.UID == "" || clique.Labels[grovecommon.LabelPartOfKey] != snapshot.podCliqueSet.Name || @@ -1365,13 +1348,13 @@ func (r *graphReconciler) observeLPXServingClique( return nil, fmt.Sprintf( "%s PodClique %q does not match the current ordinary-scheduler identity", roleName, name, - ), nil + ), nil, nil } if !isCyborg { if clique.Status.CurrentPodCliqueSetGenerationHash == nil || *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash { - return nil, fmt.Sprintf("Waiting for current conductor PodClique %q", clique.Name), nil + return nil, fmt.Sprintf("Waiting for current conductor PodClique %q", clique.Name), nil, nil } - return nil, "", nil + return nil, "", nil, nil } // Cyborg publication additionally binds device claims and exact Grove revision hashes. @@ -1379,7 +1362,7 @@ func (r *graphReconciler) observeLPXServingClique( return nil, fmt.Sprintf( "Cyborg PodClique %q lacks the current LPU scaling-group owner identity", clique.Name, - ), nil + ), nil, nil } if !hasLPXCyborgDeviceIntent(clique.Spec.PodSpec) || !hasExpectedLPXCyborgClaimReferences(snapshot.podCliqueSet, clique, plan.CyborgTemplate) { reason := fmt.Sprintf("Cyborg PodClique %q GPU intent or DRA Claim references changed during Grove materialization", clique.Name) @@ -1387,9 +1370,9 @@ func (r *graphReconciler) observeLPXServingClique( pcs := snapshot.podCliqueSet if pcs.Status.ObservedGeneration != nil && *pcs.Status.ObservedGeneration == pcs.Generation && clique.Status.CurrentPodCliqueSetGenerationHash != nil && *clique.Status.CurrentPodCliqueSetGenerationHash == snapshot.generationHash { - return nil, "", &lpxRetiring{retirementReason: reason} + return nil, "", &lpxRetiring{retirementReason: reason}, nil } - return nil, reason, nil + return nil, reason, nil, nil } reference := &lpxv1alpha1.CyborgPodCliqueReference{Name: clique.Name, UID: string(clique.UID)} @@ -1403,9 +1386,9 @@ func (r *graphReconciler) observeLPXServingClique( return reference, fmt.Sprintf( "Waiting for Grove to materialize the current PodCliqueSet generation in Cyborg PodClique %q", clique.Name, - ), nil + ), nil, nil } - return reference, "", nil + return reference, "", nil, nil } func selectLPXPublicationPodGangs( diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index ddbabdad18f5..06cf2d24fff0 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -693,8 +693,9 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { }) t.Log("Only bookkeeping changes; root metadata additions, edits and removals are ignored") - synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) require.NoError(t, err) + require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, updates) want := before.DeepCopy() @@ -705,16 +706,18 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { t.Log("Root metadata differences alone are a no-op once bookkeeping is current") updates = 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) require.NoError(t, err) + require.Nil(t, retiring) require.False(t, modified) require.Zero(t, updates) require.Equal(t, want, synced) t.Log("Template metadata still updates through Spec without changing root metadata or native controls") desired.Spec.Template.Cliques[0].Annotations = map[string]string{"test.example/template": "next"} - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) require.NoError(t, err) + require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, updates) want.Spec = desired.Spec @@ -732,8 +735,9 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.NoError(t, base.Update(ctx, concurrent)) desired.Spec.Replicas += 2 updates = 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) + require.Nil(t, retiring) require.Nil(t, synced) require.False(t, modified) require.Equal(t, 1, updates) @@ -753,8 +757,9 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers = []corev1.Container{ {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, } - observed, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + observed, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.NoError(t, err) + require.Nil(t, retiring) require.True(t, modified) t.Log("Model API-defaulted live fields without changing the applied generation") @@ -768,8 +773,9 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("Apply authored ordering changes but ignore unchanged desired specs") - synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, observed, desired) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, observed, desired) require.NoError(t, err) + require.Nil(t, retiring) require.Equal(t, reorder, modified) require.Equal(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers, synced.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) require.Equal(t, before, observed, "synchronization must not mutate its observation") @@ -778,8 +784,9 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("The next reconciliation is a no-op") - _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) require.NoError(t, err) + require.Nil(t, retiring) require.False(t, modified) }) } @@ -819,19 +826,39 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { }) t.Log("A failed publication observation prevents initial creation") - synced, modified, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.ErrorIs(t, err, fenceErr) + require.Nil(t, retiring) require.False(t, modified) require.Nil(t, synced) require.Zero(t, writes) require.Equal(t, 1, lists) require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) - t.Log("Recover the observation and create the exact child-owned PCS") + t.Log("Retire an owned request without creating a PCS while its finalizer holds deletion") failFence = false + request := deadlineTestRequest(dgd, "previous-attempt", time.Now(), lpxv1alpha1.RequestPhaseBound) + request.ResourceVersion = "" + request.Finalizers = []string{"test.example/scheduler"} + require.NoError(t, base.Create(ctx, request)) + writes, lists = 0, 0 + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + require.NoError(t, err) + require.NotNil(t, retiring) + require.False(t, modified) + require.Nil(t, synced) + require.Zero(t, writes) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(request), request)) + require.NotNil(t, request.DeletionTimestamp) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) + + t.Log("Finish request cleanup and create the exact child-owned PCS") + request.Finalizers = nil + require.NoError(t, base.Update(ctx, request)) writes, lists = 0, 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.NoError(t, err) + require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, lists) require.Equal(t, 1, writes) @@ -848,8 +875,9 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) t.Log("A stale missing observation does not adopt an already-created PCS") - _, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) + require.Nil(t, retiring) require.False(t, modified) t.Log("An in-place update does not require listing every scheduler request") @@ -860,8 +888,9 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" failFence = true writes, lists = 0, 0 - synced, modified, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired) require.NoError(t, err) + require.Nil(t, retiring) require.True(t, modified) require.Equal(t, live.UID, synced.UID) require.Equal(t, 1, writes) @@ -892,13 +921,14 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { require.NoError(t, reconciler.Update(ctx, drifted)) require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + _, _, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( ctx, dgd, drifted, rendered, ) require.NoError(t, err) + require.Nil(t, retiring) require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) t.Log("Correct the spec in place without stopping the existing PCS") @@ -942,13 +972,14 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { reconciler.apiReader = reader t.Log("Restore the explicit hold in place without rotating the existing request") - _, _, err = reconciler.reconcileGrovePodCliqueSetForLPX( + _, _, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX( ctx, dgd, drifted, rendered, ) require.NoError(t, err) + require.Nil(t, retiring) recreated := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) @@ -1012,13 +1043,14 @@ func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { return delegated.Update(ctx, object, opts...) }, }) - synced, changed, err := reconciler.reconcileGrovePodCliqueSetForLPX( + synced, changed, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( ctx, successorDGD, pcs, successorRendered, ) require.ErrorIs(t, err, admissionErr) + require.Nil(t, retiring) require.Nil(t, synced) require.False(t, changed) @@ -1032,6 +1064,8 @@ func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { func TestLPXRetirementOverridesRuntimeReadiness(t *testing.T) { t.Log("Retirement must replace both successful and pending runtime results") retiring := &lpxRetiring{retirementReason: "test retirement"} + _, isError := any(retiring).(error) + require.False(t, isError, "expected retirement must stay outside the operational error channel") for _, base := range []reconcileOutcome{{State: nvidiacomv1beta1.DGDStateSuccessful}, {State: nvidiacomv1beta1.DGDStatePending}} { require.Equal(t, lpxResult(retiring), overlayLPXResult(base, retiring)) } From 8151289f80f510459bf348b653fceb26709bb2b2 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:50:28 +0300 Subject: [PATCH 077/196] refactor(lpx): isolate pure request decisions (#515) Signed-off-by: Andrew Paprotsky (cherry picked from commit 507aee6019ec3690feed2dca80eed1f6ba41d3f1) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/lifecycle.go | 95 +++++++++----- .../internal/controller/lpx/lifecycle_test.go | 120 +++++++++++++++++- 2 files changed, 185 insertions(+), 30 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 657b8f5ff606..637fb5354afb 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -774,7 +774,6 @@ func (r *graphReconciler) reconcileSelectedLPX( return classification, err } -//nolint:gocyclo // Current-request retirement must remain visibly ordered before Grove observation. func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -798,17 +797,8 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( // Validate staged identities without crossing the scheduler-publication boundary. if !allowPublication { - for index := range desired.requests { - request := &desired.requests[index] - identity := groveIdentities[request.replicaIndex] - if identity != nil && identity.complete && currents[request.requestName] == nil { - return &lpxClosed{ - incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization", - }, nil - } - } - if pending != nil { - return pending, nil + if closed := classifyLPXPublicationReadiness(desired.requests, currents, groveIdentities, pending); closed != nil { + return closed, nil } } @@ -855,14 +845,40 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return transition, nil } } + return classifyLPXCurrentRequests(desired.requests, currents, pending), nil +} +// classifyLPXPublicationReadiness requires identities to cover every request's replica index. +// Identity entries, current requests and pending state may be nil. +func classifyLPXPublicationReadiness( + requests []lpxModelMaterializing, + currents map[string]*lpxv1alpha1.LPUPipelineRequest, + identities []*lpxGroveIdentity, + pending *lpxClosed, +) *lpxClosed { + // Complete identities with no request must wait for the Grove synchronization boundary. + for _, request := range requests { + identity := identities[request.replicaIndex] + if identity != nil && identity.complete && currents[request.requestName] == nil { + return &lpxClosed{incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization"} + } + } + return pending +} + +// classifyLPXCurrentRequests accepts absent requests and a nil pending state. +func classifyLPXCurrentRequests( + requests []lpxModelMaterializing, + currents map[string]*lpxv1alpha1.LPUPipelineRequest, + pending *lpxClosed, +) lpxClassification { // Stream terminal, side-effect-free classifications in request order. var firstNonBound lpxClassification - for _, request := range desired.requests { + for _, request := range requests { if live := currents[request.requestName]; live != nil { classification := classifyPublishedLPX(live) if state, ok := classification.(*lpxSchedulerObserved); ok && lpxSchedulerResult(state).State == nvidiacomv1beta1.DGDStateFailed { - return classification, nil + return classification } if _, bound := classification.(*lpxBound); !bound && firstNonBound == nil { firstNonBound = classification @@ -872,12 +888,12 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( // Keep pending behind every published non-bound request. if firstNonBound != nil { - return firstNonBound, nil + return firstNonBound } if pending != nil { - return pending, nil + return pending } - return &lpxBound{}, nil + return &lpxBound{} } func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( @@ -938,22 +954,15 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( request := &desired.requests[index] live := currents[request.requestName] identity := groveIdentities[request.replicaIndex] - if identity != nil && identity.complete { - spec, err := json.Marshal(struct { - Spec lpxv1alpha1.LPUPipelineRequestSpec - AgentUIDs []types.UID - }{request.modelProjection.RequestSpec(deployment.Namespace, identity.podGangName, identity.cyborgClique), identity.agentUIDs}) - if err != nil { - return nil, nil, nil, err - } - request.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) - } else if live != nil { - request.attemptDigest = live.Annotations[lpxAttemptDigestAnnotation] + digest, intentChanged, err := resolveLPXCurrentRequestIntent(deployment, *request, identity, live) + if err != nil { + return nil, nil, nil, err } + request.attemptDigest = digest if live == nil { continue } - if identity != nil && identity.complete && validateCurrentLPXRequest(deployment, request, identity, live) != nil { + if intentChanged { intentional[request.replicaIndex] = true } if intentional[request.replicaIndex] || !live.DeletionTimestamp.IsZero() || lpx.ValidateRequestSize(live) != nil { @@ -988,6 +997,34 @@ func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( return groveIdentities, pending, nil, nil } +// resolveLPXCurrentRequestIntent does not mutate its inputs. deployment must be non-nil; +// identity and live may be nil while their observations are pending. +func resolveLPXCurrentRequestIntent( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired lpxModelMaterializing, + identity *lpxGroveIdentity, + live *lpxv1alpha1.LPUPipelineRequest, +) (string, bool, error) { + // Preserve the observed digest until Grove supplies a complete replacement identity. + if identity == nil || !identity.complete { + if live != nil { + desired.attemptDigest = live.Annotations[lpxAttemptDigestAnnotation] + } + return desired.attemptDigest, false, nil + } + + // Compare immutable intent using the digest derived from the complete Grove identity. + spec, err := json.Marshal(struct { + Spec lpxv1alpha1.LPUPipelineRequestSpec + AgentUIDs []types.UID + }{desired.modelProjection.RequestSpec(deployment.Namespace, identity.podGangName, identity.cyborgClique), identity.agentUIDs}) + if err != nil { + return "", false, err + } + desired.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) + return desired.attemptDigest, live != nil && validateCurrentLPXRequest(deployment, &desired, identity, live) != nil, nil +} + // classifyPublishedLPX classifies a non-nil published request from its current scheduler receipt. func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassification { // Expose only scheduler status observed for the current request generation. diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 06cf2d24fff0..6e545c7a6a54 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -1328,6 +1328,47 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { } } +func TestLPXRequestReadinessDecisions(t *testing.T) { + t.Log("Keep staged publication ahead of Grove pending and retain nil continuation") + pending := &lpxClosed{incomplete: "Grove is incomplete"} + staged := &lpxClosed{incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization"} + for _, test := range []struct { + name string + identity *lpxGroveIdentity + published bool + pending *lpxClosed + want *lpxClosed + }{ + {name: "staged before pending", identity: &lpxGroveIdentity{complete: true}, pending: pending, want: staged}, + {name: "staged without pending", identity: &lpxGroveIdentity{complete: true}, want: staged}, + {name: "published before pending", identity: &lpxGroveIdentity{complete: true}, published: true, pending: pending, want: pending}, + {name: "incomplete identity", identity: &lpxGroveIdentity{}, pending: pending, want: pending}, + {name: "missing identity", pending: pending, want: pending}, + {name: "published continues", identity: &lpxGroveIdentity{complete: true}, published: true}, + {name: "missing continues"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Resolve the requested replica without changing its current or desired request") + requests := []lpxModelMaterializing{{requestName: "engine", replicaIndex: 1}} + beforeRequests := slices.Clone(requests) + currents := map[string]*lpxv1alpha1.LPUPipelineRequest{} + if test.published { + currents["engine"] = deadlineTestRequest(&nvidiacomv1alpha1.LPXGraphDeployment{}, "engine", time.Now(), lpxv1alpha1.RequestPhaseBound) + } + before := currents["engine"].DeepCopy() + require.Equal(t, test.want, classifyLPXPublicationReadiness(requests, currents, []*lpxGroveIdentity{nil, test.identity}, test.pending)) + var want lpxClassification = &lpxBound{} + if test.pending != nil { + want = test.pending + } + require.Equal(t, want, classifyLPXCurrentRequests(requests, currents, test.pending)) + require.Equal(t, beforeRequests, requests) + require.Equal(t, before, currents["engine"]) + }) + } + require.Equal(t, &lpxClosed{incomplete: "Grove is incomplete"}, pending) +} + func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { tests := []struct { name string @@ -1421,6 +1462,22 @@ func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { if test.wantMessage != "" { require.Contains(t, result.Message, test.wantMessage) } + + t.Log("The first failure outranks open requests; otherwise the first non-Bound receipt wins") + before := request.DeepCopy() + later := request.DeepCopy() + later.Status.Diagnostics[0].Detail = "a later scheduler receipt" + requests := []lpxModelMaterializing{{requestName: "open"}, {requestName: "observed"}, {requestName: "later"}} + currents := map[string]*lpxv1alpha1.LPUPipelineRequest{"open": {}, "observed": request, "later": later} + pending := &lpxClosed{incomplete: "Grove is incomplete"} + var want lpxClassification = &lpxOpen{} + if test.wantState == nvidiacomv1beta1.DGDStateFailed { + want = observed + } + require.Equal(t, want, classifyLPXCurrentRequests(requests, currents, pending)) + slices.Reverse(requests) + require.Equal(t, (*lpxSchedulerObserved)(later.Status), classifyLPXCurrentRequests(requests, currents, pending)) + require.Equal(t, before, request) }) } } @@ -1579,8 +1636,69 @@ func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { changed.podGangUID = "replacement-podgang-uid" require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") + t.Log("Resolve current intent from complete identities and preserve inputs while Grove is incomplete") + complete := *identity + complete.complete, complete.agentUIDs = true, []types.UID{"agent-uid"} + computed, intentChanged, err := resolveLPXCurrentRequestIntent(dgd, *projection, &complete, nil) + require.NoError(t, err) + require.False(t, intentChanged) + require.True(t, strings.HasPrefix(computed, "sha256:")) + intent := *projection + intent.attemptDigest = computed + current.Annotations = lpxRequestAnnotations(dgd, &intent, &complete) + intent.attemptDigest = "supplied-digest" + replacement := complete + replacement.agentUIDs = []types.UID{"replacement-agent-uid"} + for _, test := range []struct { + name string + identity *lpxGroveIdentity + live *lpxv1alpha1.LPUPipelineRequest + drift bool + wantDigest string + wantChanged bool + }{ + {name: "match", identity: &complete, live: current, wantDigest: computed}, + {name: "new request", identity: &complete, wantDigest: computed}, + {name: "spec drift", identity: &complete, live: current, drift: true, wantDigest: computed, wantChanged: true}, + {name: "agent replacement", identity: &replacement, live: current, wantChanged: true}, + {name: "incomplete with live request", identity: identity, live: current, drift: true, wantDigest: computed}, + {name: "missing identity with live request", live: current, drift: true, wantDigest: computed}, + {name: "incomplete without request", identity: identity, wantDigest: intent.attemptDigest}, + {name: "missing identity and request", wantDigest: intent.attemptDigest}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Compare immutable intent without changing the supplied digest or observed request") + live := test.live.DeepCopy() + if test.drift { + live.Spec.PodGangRef.Name = "other-gang" + } + beforeLive, beforeDGD, beforeIntent := live.DeepCopy(), dgd.DeepCopy(), intent + beforeSpec := intent.modelProjection.RequestSpec(dgd.Namespace, complete.podGangName, nil) + var beforeIdentity *lpxGroveIdentity + if test.identity != nil { + copy := *test.identity + copy.agentUIDs = slices.Clone(copy.agentUIDs) + beforeIdentity = © + } + got, changed, err := resolveLPXCurrentRequestIntent(dgd, intent, test.identity, live) + require.NoError(t, err) + require.Equal(t, test.wantChanged, changed) + if test.wantDigest == "" { + require.NotEqual(t, computed, got, "replacement Agent identity changes the attempt digest") + } else { + require.Equal(t, test.wantDigest, got) + } + require.Equal(t, beforeLive, live) + require.Equal(t, beforeDGD, dgd) + require.Equal(t, beforeIntent, intent) + require.Equal(t, beforeIdentity, test.identity) + require.Equal(t, beforeSpec, intent.modelProjection.RequestSpec(dgd.Namespace, complete.podGangName, nil)) + }) + } + + t.Log("Keep published requests when the deployment generation changes without intent drift") createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - _, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) require.NoError(t, err) From 936226639100d58016f070e6c44b9572b8bcf7df Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Wed, 16 Sep 2026 13:57:47 -0700 Subject: [PATCH 078/196] fix(lpx): stop enabling worker tty and stdin by default (#513) Signed-off-by: Steven Davidovitz (cherry picked from commit 4c3adfa6d50400d0f179df40ed289a24714aae72) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/render_test.go | 4 ++-- deploy/operator/internal/dynamo/lpx/lpu_pod.go | 4 +--- deploy/operator/internal/dynamo/lpx/pod_roles_test.go | 4 ++-- deploy/operator/internal/dynamo/lpx/runtime_agent.go | 2 -- .../internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 2 -- .../dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v2-specdecode.yaml | 6 ------ .../testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml | 6 ------ .../internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml | 2 -- 12 files changed, 5 insertions(+), 33 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index dab08a9a1937..c9ec68781fa9 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1239,8 +1239,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NotContains(t, main.Args, "--agent-env-vars") require.NotNil(t, main.SecurityContext) require.True(t, *main.SecurityContext.Privileged) - require.True(t, main.Stdin) - require.True(t, main.TTY) + require.False(t, main.Stdin) + require.False(t, main.TTY) require.NotNil(t, main.StartupProbe) require.NotNil(t, main.ReadinessProbe) runtimeEnv := make(map[string]string, len(main.Env)) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go index 167ee736c879..4eb89f9f4c35 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_pod.go @@ -37,13 +37,11 @@ func ensureLPUNodeTolerations(podSpec *corev1.PodSpec) { } func applyLPUWorkerContainerBase(container *corev1.Container) { - // Workers need privileged device access and an interactive runtime session. + // Workers need privileged device access. container.SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), RunAsUser: ptr.To(int64(0)), } - container.TTY = true - container.Stdin = true // Agent scheduling already initialized device resources; add runtime CPU and hugepage requirements. cpuRequest := *resource.NewQuantity(defaultLPUAgentCPUs, resource.DecimalSI) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 33a3bd2ec4e5..695508a1d951 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -341,8 +341,8 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), }, agentContainer.SecurityContext) - require.True(t, agentContainer.TTY) - require.True(t, agentContainer.Stdin) + require.False(t, agentContainer.TTY) + require.False(t, agentContainer.Stdin) require.Nil(t, agentContainer.LivenessProbe) require.Nil(t, agentContainer.StartupProbe) require.Nil(t, agentContainer.Lifecycle) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 145199426939..80a07b8ba2d5 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -307,8 +307,6 @@ func configureNodeLocalAgentWorkerContainer( RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), } - container.TTY = true - container.Stdin = true // Remove health and lifecycle behavior owned by the source image process. container.LivenessProbe = nil diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 59920c98fbd7..12129f36cddc 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -557,8 +557,6 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 777d09224ce9..87a5df1486e1 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -346,8 +346,6 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 69204b4ece98..c4d57c74f0e9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -337,8 +337,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index bb68359a798a..4c59040fffb5 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -409,8 +409,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage @@ -557,8 +555,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage @@ -705,8 +701,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 26fd43939cb4..bfadf80f9656 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -317,8 +317,6 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 045f6a12e151..6cb2ea731e3b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -385,8 +385,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 7a73577bfc3a..b44d7b0ace29 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -445,8 +445,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage @@ -626,8 +624,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage @@ -807,8 +803,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 69d16622580b..1f7b7c4b0480 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -422,8 +422,6 @@ spec: runAsGroup: 0 runAsNonRoot: false runAsUser: 0 - stdin: true - tty: true volumeMounts: - mountPath: /nfs name: model-storage From bce7d642d61ccdf59066428e0d9ed7b1126ab4a4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:06:09 +0300 Subject: [PATCH 079/196] fix(operator): make LPX workload names readable (#517) Use the immutable LGD name as the PCS prefix with a four-hex identity suffix. Give the fixed lpx scaling group agt and cond cliques for both Nova and Cyborg, with compact Agent ordinals for multiple models. Keep private endpoints stable across component changes and validate actual GPU pod hostnames before publishing the rendered workload. Reuse the validated materialization plan throughout rendering and runtime configuration instead of duplicating naming state and reconstruction. BREAKING CHANGE: Generated LPX resource names change with no legacy naming or upgrade compatibility path. Signed-off-by: Andrew Paprotsky (cherry picked from commit ae3f2265e8fc38eff85231e5499fe53cf6570c42) Signed-off-by: Andrew Paprotsky --- .../dgd_lpx_handoff_envtest_test.go | 2 +- .../internal/controller/lpx/controller.go | 11 +- .../controller/lpx/controller_test.go | 127 ++++- .../internal/controller/lpx/lifecycle.go | 6 +- .../internal/controller/lpx/preflight.go | 20 - .../internal/controller/lpx/render.go | 2 +- .../internal/controller/lpx/render_test.go | 11 +- deploy/operator/internal/dynamo/graph_lpx.go | 54 +- .../internal/dynamo/graph_lpx_test.go | 43 +- .../internal/dynamo/lpx/cyborg_render.go | 11 +- .../internal/dynamo/lpx/cyborg_render_test.go | 71 ++- .../dynamo/lpx/hybrid_cyborg_bindings.go | 13 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 27 +- .../internal/dynamo/lpx/lpu_configmap.go | 19 +- .../internal/dynamo/lpx/lpu_configmap_test.go | 33 +- .../internal/dynamo/lpx/materialization.go | 127 ++--- .../dynamo/lpx/materialization_test.go | 82 ++- .../internal/dynamo/lpx/pod_roles_test.go | 12 +- deploy/operator/internal/dynamo/lpx/render.go | 28 +- .../internal/dynamo/lpx/render_test.go | 52 +- .../lpx/resolve_selected_workload_test.go | 10 +- .../internal/dynamo/lpx/runtime_ssh_test.go | 2 +- .../internal/dynamo/lpx/selected_workload.go | 11 - .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 448 ++++++++-------- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 364 ++++++------- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 314 +++++------ .../node-local-v2-specdecode.yaml | 388 +++++++------- .../node-local-v3-hx-hybrid.yaml | 308 +++++------ .../node-local-v3-hx-lpu-only.yaml | 380 +++++++------- .../node-local-v3-hx-specdecode.yaml | 496 +++++++++--------- .../testdata/from_dgd_yaml/single_v2.yaml | 322 ++++++------ 31 files changed, 1962 insertions(+), 1832 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 9e4cb49b6b06..8ccbcb125b5a 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -284,7 +284,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { require.NoError(t, env.Client().Delete(t.Context(), quota)) pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: dynamo.PCSNameForLPX(child), Namespace: source.Namespace}} configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: source.Namespace}} - service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-lpx", Namespace: source.Namespace}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: pcs.Name + "-serve", Namespace: source.Namespace}} require.EventuallyWithT(t, func(c *assert.CollectT) { if !assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) { return diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 416a3f42f316..45be592c8aab 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -487,9 +487,9 @@ func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployme } func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { - // Use the same normalized identity for endpoint creation and retirement. - component := lpx.ServingComponent(source) - serviceName := dynamo.NormalizeKubeResourceName(dynamo.PCSNameForLPX(deployment) + "-" + component.ComponentName) + // One serving endpoint belongs to the engine, independently of mutable component names. + pcsName := dynamo.PCSNameForLPX(deployment) + serviceName := pcsName + "-serve" if !commoncontroller.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, source.Annotations) { service := &corev1.Service{} @@ -502,6 +502,9 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a } return client.IgnoreNotFound(r.Delete(ctx, service, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &service.UID, ResourceVersion: &service.ResourceVersion}})) } + + // Source component metadata is needed only when publishing the serving endpoint. + component := lpx.ServingComponent(source) service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ ServiceName: serviceName, Namespace: deployment.Namespace, ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, @@ -514,6 +517,6 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a } // Keep the endpoint on this materialization even when another child shares the source. service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue - service.Spec.Selector[grovecommon.LabelPartOfKey] = dynamo.PCSNameForLPX(deployment) + service.Spec.Selector[grovecommon.LabelPartOfKey] = pcsName return r.syncLPXResource(ctx, deployment, service) } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index c646685f12a8..c5ce3a84d59c 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "slices" + "strings" "testing" "time" @@ -27,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -325,7 +327,6 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { preserve bool }{ {name: "intent", reason: "LPXRejected", message: "providerOverride"}, - {name: "name budget", reason: "LPXRejected", message: "spec.components[0].name: Invalid value"}, {name: "selected workload", reason: "LPXRejected", message: "replica count"}, {name: "render", reason: "LPXReconciliationFailed", message: "model storage volume mount"}, {name: "invalid source", reason: "LPXReconciliationFailed", message: "source"}, @@ -346,7 +347,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { pcs.UID = "staged-pcs" require.NoError(t, r.Update(t.Context(), pcs)) endpoint := &corev1.Service{} - endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: pcsKey.Name + "-lpx"} + endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: pcsKey.Name + "-serve"} require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) endpoint.UID = "staged-endpoint" require.NoError(t, r.Update(t.Context(), endpoint)) @@ -370,8 +371,6 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { switch scenario.name { case "intent": source.Spec.ProviderOverride = &v1beta1.ProviderOverride{Target: "PodCliqueSet"} - case "name budget": - component.ComponentName = "serving-engines" case "selected workload": component.Replicas = ptr.To(int32(-1)) case "render": @@ -445,7 +444,7 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" foreignRuntime.UID = "foreign-runtime" foreignPCS.Name = dynamo.PCSNameForLPX(child) - foreignEndpoint.Name = foreignPCS.Name + "-lpx" + foreignEndpoint.Name = foreignPCS.Name + "-serve" foreignRuntime.Name = foreignPCS.Name + "-runtime" foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences foreignRuntime.OwnerReferences = ordinary.OwnerReferences @@ -565,7 +564,7 @@ func TestLPXEndpointLifecycle(t *testing.T) { t.Log("Publish the LPX-owned endpoint with its serving-role selector") require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) service := &corev1.Service{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child) + "-lpx"} + key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child) + "-serve"} require.NoError(t, r.Get(t.Context(), key, service)) require.True(t, metav1.IsControlledBy(service, child)) require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) @@ -641,7 +640,7 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { for _, object := range []client.Object{ &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: root, Namespace: child.Namespace}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: lpx.LPUConfigMapName(root, pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-" + lpx.ServingComponent(source).ComponentName, Namespace: child.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-serve", Namespace: child.Namespace}}, } { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) require.True(t, metav1.IsControlledBy(object, child)) @@ -656,6 +655,110 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { } } +func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { + t.Log("Publish a speculative engine with long deployment and component names") + _, source, registry := newLPXSpecDecodeTestDGD(t) + source.Name = "apaprotskyi-gpt-oss-20b-lp20-b300" + source.Spec.Components[0].ComponentName = "serving-component-with-name" + source.Spec.Components[1].ComponentName = "draft-component-with-name" + source.Spec.Components[1].Replicas = ptr.To(int32(1)) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + child := newLPXTestDeployment(t, source) + r := newLPXTestReconciler(t, registry, child, source) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), request) + require.NoError(t, err) + root := dynamo.PCSNameForLPX(child) + require.True(t, strings.HasPrefix(root, source.Name+"-")) + require.Len(t, root, len(source.Name)+len("-ffff")) + pcs := &grovev1alpha1.PodCliqueSet{} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: root} + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + pcs.UID = "stable-pcs" + require.NoError(t, r.Update(t.Context(), pcs)) + endpoint := &corev1.Service{} + endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: root + "-serve"} + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + endpoint.UID = "stable-endpoint" + require.NoError(t, r.Update(t.Context(), endpoint)) + require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) + beforeGroup := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] + beforeOwners := endpoint.OwnerReferences + + for _, change := range []string{"rename serving component", "move conductor"} { + t.Log(change) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + if change == "rename serving component" { + source.Spec.Components[0].ComponentName = "renamed-serving-component" + } else { + conductor := *source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) + source.Spec.Components[0].Roles = slices.DeleteFunc(source.Spec.Components[0].Roles, func(role v1beta1.ComponentRoleSpec) bool { + return role.Name == v1beta1.ComponentRoleLPXConductor + }) + source.Spec.Components[1].Roles = append(source.Spec.Components[1].Roles, conductor) + } + source.Generation++ + require.NoError(t, r.Update(t.Context(), source)) + child.Generation++ + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + require.NoError(t, r.Update(t.Context(), child)) + + t.Log("Keep one PCS and endpoint while updating their serving component metadata") + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + require.Equal(t, "stable-pcs", string(pcs.UID)) + require.Equal(t, "stable-endpoint", string(endpoint.UID)) + require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) + require.Equal(t, beforeGroup.Name, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Name) + require.Equal(t, beforeGroup.CliqueNames, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) + require.Equal(t, beforeOwners, endpoint.OwnerReferences) + require.Equal(t, source.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, lpx.ServingComponent(source).ComponentName, endpoint.Spec.Selector[consts.KubeLabelDynamoComponent]) + require.Equal(t, root, endpoint.Spec.Selector[grovecommon.LabelPartOfKey]) + require.Equal(t, consts.KubeLabelValueTrue, endpoint.Spec.Selector[dynamo.LPXServingLabel]) + allPCS, allEndpoints := &grovev1alpha1.PodCliqueSetList{}, &corev1.ServiceList{} + require.NoError(t, r.List(t.Context(), allPCS)) + require.NoError(t, r.List(t.Context(), allEndpoints)) + require.Len(t, allPCS.Items, 1) + require.Len(t, allEndpoints.Items, 1) + + t.Log("Select only the conductor and share one runtime config across the clique roster") + require.NotEmpty(t, pcs.Spec.Template.Cliques) + configHash := pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash] + require.NotEmpty(t, configHash) + cliqueNames := make([]string, 0, len(pcs.Spec.Template.Cliques)) + for _, clique := range pcs.Spec.Template.Cliques { + cliqueNames = append(cliqueNames, clique.Name) + require.Equal(t, configHash, clique.Annotations[consts.AnnotationExtraResourcesHash]) + for key, value := range clique.Labels { + require.Empty(t, validation.IsValidLabelValue(value), key) + } + if clique.Name == "cond" { + for key, value := range endpoint.Spec.Selector { + if key != grovecommon.LabelPartOfKey { + require.Equal(t, value, clique.Labels[key], key) + } + } + } else { + require.NotContains(t, clique.Labels, dynamo.LPXServingLabel) + } + } + require.ElementsMatch(t, []string{"cond", "agt0", "agt1"}, cliqueNames) + + t.Log("Keep runtime Agent addresses aligned with the clique names") + config := &corev1.ConfigMap{} + configKey := client.ObjectKey{Namespace: child.Namespace, Name: lpx.LPUConfigMapName(root, configHash)} + require.NoError(t, r.Get(t.Context(), configKey, config)) + require.True(t, metav1.IsControlledBy(config, child)) + require.Contains(t, config.Data["datacenter.toml"], "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}") + require.Contains(t, config.Data["datacenter.toml"], "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}") + } +} + func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) { t.Log("Publish a runtime ConfigMap under the exact LPX child") source := newLPXTestSource(lpx.PipelineSingle, "test-build") @@ -759,14 +862,8 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { }}, {name: "disabled Grove", messages: []string{"Grove is disabled"}}, {name: componentProvider, messages: []string{"requires the Grove workload provider"}}, - {name: "too long serving name", componentName: "serving-engines", messages: []string{ - "spec.components[0].name: Invalid value", "combined Grove resource name length 46 exceeds the 45-character limit", - }}, - {name: "reordered shared draft with too long serving name", componentName: "serving-engines", sharedDraft: true, messages: []string{ - "spec.components[2].name: Invalid value", "combined Grove resource name length 46 exceeds the 45-character limit", - }}, - {name: "maximum legal serving name", componentName: "serving-engine"}, - {name: "reordered shared draft with maximum legal serving name", componentName: "serving-engine", sharedDraft: true}, + {name: "long serving name", componentName: strings.Repeat("serving-", 7) + "engine"}, + {name: "reordered shared draft with long serving name", componentName: strings.Repeat("serving-", 7) + "engine", sharedDraft: true}, } { t.Run(scenario.name, func(t *testing.T) { t.Log("Reconcile authored LPX intent against cold remote builds") diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 637fb5354afb..eb23009cf2c7 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -349,11 +349,11 @@ func (r *graphReconciler) prepareLPXMaterializing( return nil, nil, fmt.Errorf("refusing to use replicas from a foreign LPX scaling group") } plan.Replicas = group.Spec.Replicas + if err := plan.ValidateReplicaCount(); err != nil { + return nil, &lpxRejected{reason: err.Error()}, nil + } } } - if err := plan.ValidateReplicaCount(); err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil - } workloadDigest := workload.Digest().String() requests := make([]lpxModelMaterializing, 0, len(projections)*int(plan.Replicas)) for replicaIndex := int32(0); replicaIndex < plan.Replicas; replicaIndex++ { diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index 98c66aa384c4..cefa4d5e7c92 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -22,11 +22,8 @@ import ( "fmt" "maps" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation/field" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -49,23 +46,6 @@ func (r *graphReconciler) reconcileLPXSafetyPreflight( return nil, classification, nil } - // The serving component owns every Grove name, including the shared draft cliques. - serving := dynamolpx.ServingComponent(source) - for index := range source.Spec.Components { - component := &source.Spec.Components[index] - if component != serving { - continue - } - combinedLength := len(dynamo.PCSNameForLPX(deployment)) + dynamo.LPXComponentNameBudget(component.ComponentName) - if combinedLength > consts.MaxCombinedGroveResourceNameLength { - err := field.Invalid(field.NewPath("spec", "components").Index(index).Child("name"), component.ComponentName, - fmt.Sprintf("combined Grove resource name length %d exceeds the %d-character limit; shorten the deployment or component name", - combinedLength, consts.MaxCombinedGroveResourceNameLength)) - return nil, &lpxRejected{reason: err.Error()}, nil - } - break - } - if len(requests) == 0 { if err := dynamolpx.ValidateSelectedIntent(source).ToAggregate(); err != nil { return nil, &lpxRejected{reason: err.Error()}, nil diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 89023b9c0957..d66cf09aca61 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -38,7 +38,7 @@ func renderPodCliqueSet( ) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { // Apply shared Dynamo defaults before materializing the independently owned LPX roles. pcs, input, err := dynamo.RenderLPXBasePodCliqueSet(ctx, source, operatorConfig, runtimeConfig, - kubeClient, secretsRetriever, workload, plan.PodCliqueSetName) + kubeClient, secretsRetriever, workload, plan) if err != nil { return nil, nil, err } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index c9ec68781fa9..ada3e08523ef 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -128,7 +128,7 @@ func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { t.Log("The Cyborg role receives every topology domain, not an empty mounted directory") items := map[string]string{} for _, clique := range pcs.Spec.Template.Cliques { - if clique.Name != selected.CyborgTemplateName() { + if clique.Name != plan.CyborgTemplate { continue } require.Equal(t, binding.Name, clique.Annotations[commonconsts.KubeAnnotationTopologyClusterTopologyName]) @@ -1230,7 +1230,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, lpx.SchedulerName, agent.Spec.PodSpec.SchedulerName) require.Equal(t, int32(plan.Agents[0].Replicas), agent.Spec.Replicas) require.Contains(t, cyborg.Spec.StartsAfter, agent.Name) - require.NotContains(t, cyborg.Spec.StartsAfter, "lpu-ldr") + require.NotContains(t, cyborg.Spec.StartsAfter, "cond") main := agent.Spec.PodSpec.Containers[0] require.Equal(t, "agent", main.Name) @@ -1356,7 +1356,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, lpuConfigName, extraResources[1].GetName()) require.Equal(t, dgd.Namespace, decodeConfig.Namespace) require.Len(t, decodeConfig.Data, 2) - require.Equal(t, []string{"lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) + require.Equal(t, []string{"lpx-${GROVE_PCSG_INDEX}-agt-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) require.NotEmpty(t, decodeConfig.Data["tokenizer_dir"]) decodeHash := lpx.LPUConfigMapHash(decodeConfig) require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) @@ -1600,15 +1600,16 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { t.Log("Render the complete LPX workload with ordinary pod spec overrides") selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) + plan := mustPlanSelectedLPX(t, source, selected) pcs, _, err := renderPodCliqueSet(t.Context(), source, &configv1alpha1.OperatorConfiguration{ MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, }, &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, - mustPlanSelectedLPX(t, source, selected), newLPXRenderDeployment(t, source)) + plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { - return clique.Name == selected.CyborgTemplateName() + return clique.Name == plan.CyborgTemplate }) require.GreaterOrEqual(t, cliqueIndex, 0) podSpec := pcs.Spec.Template.Cliques[cliqueIndex].Spec.PodSpec diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index f76c637d8752..2dc91d7cbc49 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -6,7 +6,7 @@ package dynamo import ( "context" "crypto/sha256" - "encoding/base32" + "encoding/hex" "encoding/json" "fmt" "maps" @@ -29,7 +29,7 @@ import ( // RenderLPXBasePodCliqueSet applies shared Dynamo defaults to the Grove envelope // and independently owned LPX role templates. The LPX controller completes the // compiled workload and its identity. All pointer inputs except secretsRetriever -// must be non-nil; preflight supplies the validated workload and child PCS name. +// must be non-nil; preflight supplies the validated workload and its materialization plan. func RenderLPXBasePodCliqueSet( ctx context.Context, dynamoDeployment *v1beta1.DynamoGraphDeployment, @@ -38,14 +38,14 @@ func RenderLPXBasePodCliqueSet( kubeClient client.Reader, secretsRetriever SecretsRetriever, selectedWorkload *dynamolpx.SelectedWorkload, - pcsName string, + plan *dynamolpx.MaterializationPlan, ) (*grovev1alpha1.PodCliqueSet, *dynamolpx.RenderInput, error) { // Reuse the Grove envelope, but keep LPX compilation out of normal rendering. pcs, err := newGrovePodCliqueSet(dynamoDeployment, operatorConfig, runtimeConfig) if err != nil { return nil, nil, err } - pcs.Name = pcsName + pcs.Name = plan.PodCliqueSetName queue, err := resolveGroveSchedulerQueue(ctx, dynamoDeployment.Annotations, runtimeConfig) if err != nil { return nil, nil, err @@ -71,7 +71,7 @@ func RenderLPXBasePodCliqueSet( discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), validatedQueueName: queue, groveClusterTopologyDomains: topologyDomains, - }, selectedWorkload, pcsName) + }, selectedWorkload, plan) if err != nil { return nil, nil, err } @@ -252,35 +252,33 @@ func lpxSchedulingMetadata(metadata map[string]string) map[string]string { return selected } -// PCSNameForLPX derives a stable identity from the owning LGD, not its mutable source. +// PCSNameForLPX keeps the owning LGD's name readable without depending on mutable components. +// The DGD handoff gives the LGD the same name as its source deployment. +// deployment must be non-nil and have a valid Kubernetes name. func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment) string { - // Eight characters fit Grove's name budget; a leading letter also permits Services. - digest := sha256.Sum256([]byte(deployment.Namespace + "/" + deployment.Name + "/" + string(deployment.UID))) - return "l" + strings.ToLower(base32.StdEncoding.EncodeToString(digest[:5])[:7]) -} - -func longestLPXCliqueNameLength(componentName string) int { - // At most eight drafts and one target use a single-digit model index. - return len(strings.ToLower(componentName)) + len(commonconsts.GroveRoleSuffixWorker) + 5 -} + // Reserve room for Grove's group, role, replica indexes, and Pod suffixes. + const maxPrefixLength = dynamolpx.MaxPodCliqueSetNameLength - 1 - 4 + prefix := strings.ReplaceAll(deployment.Name, ".", "-") + if prefix[0] >= '0' && prefix[0] <= '9' { + prefix = "lpx-" + prefix + } + prefix = strings.TrimRight(prefix[:min(len(prefix), maxPrefixLength)], "-") -// LPXComponentNameBudget reserves the serving group's name and its longest clique. -// Shared draft components use the serving component's generated names. -func LPXComponentNameBudget(componentName string) int { - return len(componentName) + longestLPXCliqueNameLength(componentName) + // Hash the original identity so truncation, dot normalization, and replacement stay distinct. + digest := sha256.Sum256([]byte(deployment.Namespace + "/" + deployment.Name + "/" + string(deployment.UID))) + return prefix + "-" + hex.EncodeToString(digest[:2]) } // renderLPXComponents merges ordinary Dynamo defaults independently into every // authored role. The full source DGD supplies discovery and shared defaults; // LPX component's roles are returned to the same PCS renderer. -// Preflight has validated the runtime shape and every Agent template. -func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, materializationName string) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { - // Pass naming and runtime inputs; deployment identity is stamped only on final resources. +// Preflight supplies the non-nil validated workload and materialization plan. +func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, plan *dynamolpx.MaterializationPlan) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { + // Pass runtime inputs; deployment identity is stamped only on final resources. input := &dynamolpx.RenderInput{ - MaterializationName: materializationName, - MinAvailable: p.component.MinAvailable, - Stages: make(map[string]corev1.PodTemplateSpec), - SSHSecretName: p.operatorConfig.MPI.SSHSecretName, + MinAvailable: p.component.MinAvailable, + Stages: make(map[string]corev1.PodTemplateSpec), + SSHSecretName: p.operatorConfig.MPI.SSHSecretName, } // Resolve preserved alpha metadata once for all independently rendered roles. @@ -337,7 +335,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m role.MinAvailable = nil defaults := ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)) if workload.BuildFamily() == dynamolpx.BuildFamilyXT { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, materializationName, lpuTemplate.Spec) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan, lpuTemplate.Spec) if err != nil { return nil, nil, err } @@ -351,7 +349,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, m gpu := p gpu.component = role gpu.componentName = component.ComponentName - gpu.r = ServiceRole{Name: workload.CyborgTemplateName(), Role: RoleMain, Replicas: *role.Replicas} + gpu.r = ServiceRole{Name: plan.CyborgTemplate, Role: RoleMain, Replicas: *role.Replicas} gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, defaults, p.groveClusterTopologyDomains) if err != nil { diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 700878e47d4b..d2a44b367204 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -109,6 +109,8 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { func TestLPXPCSNameUsesStableMaterializationIdentity(t *testing.T) { deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "chat", Namespace: "workloads", UID: "materialization-uid"}} name := PCSNameForLPX(deployment) + t.Log("Keep the deployment name visible with four hexadecimal identity characters") + require.Equal(t, "chat-3efb", name) t.Log("An ordinary DGD named chat-lpx must not collide with the LPX PCS of chat") require.NotEqual(t, PCSNameForDGD("chat-lpx", nil), name) require.NotEqual(t, PCSNameForDGD(deployment.Name, nil), name) @@ -137,15 +139,42 @@ func TestLPXPCSNameUsesStableMaterializationIdentity(t *testing.T) { deployment.Name = "chat-example" require.NotEqual(t, dotted, PCSNameForLPX(deployment), "normalizing dots must not erase materialization identity") - t.Log("The source-independent PCS name fits every supported component's Grove and Service name budget") - for componentLength := 1; componentLength <= 13; componentLength++ { - for _, materializationName := range []string{"c", strings.Repeat("long.", 40) + "chat", "chat.example"} { - deployment.Name = materializationName + t.Log("Names sharing the same visible prefix still identify different deployments") + deployment.Name = strings.Repeat("a", 33) + "-first" + first := PCSNameForLPX(deployment) + deployment.Name = strings.Repeat("a", 33) + "-second" + require.NotEqual(t, first, PCSNameForLPX(deployment)) +} + +func TestLPXPCSNamePreservesReadablePrefixWithinGroveBudget(t *testing.T) { + for _, test := range []struct { + name string + prefix string + }{ + {name: "c", prefix: "c"}, + {name: "gpt-oss-120b-production", prefix: "gpt-oss-120b-production"}, + {name: "apaprotskyi-llama3-8b-lp20", prefix: "apaprotskyi-llama3-8b-lp20"}, + {name: "apaprotskyi-gpt-oss-20b-lp20-b300", prefix: "apaprotskyi-gpt-oss-20b-lp20-b300"}, + {name: "chat.example", prefix: "chat-example"}, + {name: "120b-chat", prefix: "lpx-120b-chat"}, + {name: strings.Repeat("a", 33), prefix: strings.Repeat("a", 33)}, + {name: strings.Repeat("a", 32) + "-chat", prefix: strings.Repeat("a", 32)}, + {name: strings.Repeat("long.", 40) + "chat", prefix: "long-long-long-long-long-long-lon"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Retain the readable deployment prefix while bounding the complete Grove name") + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: test.name, Namespace: "workloads", UID: "materialization-uid", + }} got := PCSNameForLPX(deployment) - require.Empty(t, validation.IsDNS1035Label(got+"-"+strings.Repeat("e", componentLength)), got) - require.LessOrEqual(t, len(got)+max(componentLength, 8)+componentLength+len("-engine-gpu"), commonconsts.MaxCombinedGroveResourceNameLength) + require.True(t, strings.HasPrefix(got, test.prefix+"-"), got) + require.Len(t, got, len(test.prefix)+5) + require.Regexp(t, `-[a-f0-9]{4}$`, got) + require.LessOrEqual(t, len(got), lpx.MaxPodCliqueSetNameLength) + require.LessOrEqual(t, len(got)+len("lpx")+len("cond"), commonconsts.MaxCombinedGroveResourceNameLength) + require.Empty(t, validation.IsDNS1035Label(got+"-serve"), got) require.Equal(t, got, PCSNameForLPX(deployment)) - } + }) } } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 38bca3ac7b8f..f0fd3b9ff2d7 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -26,7 +26,6 @@ func configureHybridCyborg( workloadDigest string, modelStorage lpuModelStorage, agentTemplateNames []string, - plan *MaterializationPlan, cyborgConfigMap *corev1.ConfigMap, ) error { container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) @@ -79,14 +78,6 @@ func configureHybridCyborg( } cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) } - if projection.configuredBuild.Family == BuildFamilyXT { - cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) - } - if plan.ConductorTemplate != "" { - cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, plan.ConductorTemplate) - } - if projection.configuredBuild.Family != BuildFamilyXT { - cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) - } + cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 31e4f214ce7f..98c0e6741c89 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -7,15 +7,72 @@ package lpx import ( "encoding/json" + "strings" "testing" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" ) +func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { + t.Parallel() + + t.Log("Project a hybrid engine with the longest PCS name and maximum scheduling replica count") + fixture := newV3CompilerFixture() + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + projection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyHx16x8x2x3, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + projection.stage = testRenderComponentName + projection.configuredBuild.IOFPGACount = 1 + projection.configuredBuild.IOFanoutFactor = 1 + workload := &SelectedWorkload{ + modelProjections: []*ModelProjection{projection}, + scalingGroupReplicas: int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes), + } + pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) + plan, err := workload.PlanNodeLocalMaterialization(pcsName) + require.NoError(t, err) + + t.Log("Validate actual GPU widths rather than assuming the largest int32 pod index") + for _, test := range []struct { + name string + replicas int32 + wantError bool + }{ + {name: "one GPU per engine", replicas: 1}, + {name: "hostname at DNS limit", replicas: 100_000_000}, + {name: "hostname over DNS limit", replicas: 100_000_001, wantError: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Render the GPU width beneath the same readable PCS name") + pcs := renderTestPCS(true) + pcs.Name = pcsName + cyborg := namedClique(t, pcs, "cond") + cyborg.Spec.Replicas = test.replicas + cyborg.Spec.MinAvailable = ptr.To(test.replicas) + _, err := RenderSelectedNodeLocal(pcs, workload, plan, RenderInput{ + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, + SSHSecretName: "ssh-secret", + }) + if test.wantError { + require.ErrorContains(t, err, "materialized Cyborg Pod hostname") + require.ErrorContains(t, err, "must be no more than 63 characters") + return + } + require.NoError(t, err) + hostname := materializedPodHostname(plan.ForReplica(plan.Replicas-1).CyborgClique, int(test.replicas)-1) + require.Empty(t, validation.IsDNS1123Label(hostname)) + if test.replicas > 1 { + require.Len(t, hostname, validation.DNS1123LabelMaxLength) + } + }) + } +} + func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { t.Parallel() @@ -42,7 +99,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { t.Log("Render the Cyborg runtime contract") pcs := renderTestPCS(true) - decode := namedClique(t, pcs, "lpu-engine-gpu") + decode := namedClique(t, pcs, "cond") decode.Spec.Replicas = 4 decode.Spec.MinAvailable = ptr.To[int32](4) decode.Spec.PodSpec.ResourceClaims = nil @@ -56,12 +113,11 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) input := RenderInput{ - MaterializationName: "dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", } rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, input) require.NoError(t, err) - cyborg := namedClique(t, rendered, "lpu-engine-gpu") + cyborg := namedClique(t, rendered, "cond") t.Log("Verify the rendered Cyborg runtime I/O contract") require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ @@ -89,7 +145,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { t.Log("Preserve an image-owned entrypoint") imageEntrypointPCS := renderTestPCS(true) - imageEntrypointCyborg := namedClique(t, imageEntrypointPCS, "lpu-engine-gpu") + imageEntrypointCyborg := namedClique(t, imageEntrypointPCS, "cond") imageEntrypointCyborg.Spec.Replicas = 4 imageEntrypointCyborg.Spec.MinAvailable = ptr.To[int32](4) imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args = []string{"serve"} @@ -117,13 +173,12 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Reject incomplete Cyborg coverage during runtime rendering") pcs := renderTestPCS(true) - cyborg := namedClique(t, pcs, "lpu-engine-gpu") + cyborg := namedClique(t, pcs, "cond") cyborg.Spec.Replicas = test.replicas cyborg.Spec.MinAvailable = ptr.To(test.replicas) cyborg.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} _, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - MaterializationName: "dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", }) require.ErrorContains(t, err, test.wantError) }) diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 2482643bde8c..0ce8eea46352 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -76,20 +76,17 @@ func ApplySelectedCyborgContainerDefaults( } // RenderCyborgConfigMap renders the XT hybrid configuration before Cyborg defaults are merged. -// The workload must contain the selected hybrid model; agentPodSpec supplies its merged storage. +// The workload and plan must be non-nil, validated, and describe the selected hybrid model. +// agentPodSpec supplies its merged storage. The workload and plan are not mutated. func (w *SelectedWorkload) RenderCyborgConfigMap( namespace string, - root string, + plan *MaterializationPlan, agentPodSpec corev1.PodSpec, ) (*corev1.ConfigMap, error) { storage, err := lpuModelStorageBinding(agentPodSpec) if err != nil { return nil, err } - plan, err := w.PlanNodeLocalMaterialization(root) - if err != nil { - return nil, err - } build := &w.modelProjections[0].configuredBuild tokenizerDir := "" if build.Path != "" { @@ -105,7 +102,7 @@ func (w *SelectedWorkload) RenderCyborgConfigMap( } // Cyborg supplies the PCS prefix; startup supplies this engine's Grove index. - serverPrefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" + serverPrefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(build.Partitions)) offset := 0 for index, partition := range build.Partitions { @@ -113,7 +110,7 @@ func (w *SelectedWorkload) RenderCyborgConfigMap( offset += partition.effectiveNodeCount() } - return renderRuntimeConfigMap(namespace, root+"-decode", map[string]string{ + return renderRuntimeConfigMap(namespace, plan.PodCliqueSetName+"-decode", map[string]string{ "tokenizer_dir": tokenizerDir, "lpu_servers": strings.Join(servers, "\n"), }) diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 5f85cbeb3cd4..9e761fb01ea5 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -46,26 +46,35 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { runtimePath, err := buildRuntimePath(projection.configuredBuild.Path, "/models") require.NoError(t, err) podSpec := renderTestPodSpec() - initial, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + initial, err := workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) require.NoError(t, err) t.Log("Verify every engine replica addresses only its own Agents") for _, replicas := range []int32{1, 2, 10, 12} { t.Run(strconv.Itoa(int(replicas)), func(t *testing.T) { workload.scalingGroupReplicas = replicas - configMap, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + scaledPlan, err := workload.PlanNodeLocalMaterialization("test-dgd") + require.NoError(t, err) + configMap, err := workload.RenderCyborgConfigMap("test-namespace", scaledPlan, podSpec) require.NoError(t, err) require.Equal(t, initial, configMap) require.True(t, *configMap.Immutable) - prefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" - require.Equal(t, prefix+"lpu-wkr-m-0-0\n"+prefix+"lpu-wkr-m-0-2", configMap.Data["lpu_servers"]) + prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + require.Equal(t, prefix+"agt-0\n"+prefix+"agt-2", configMap.Data["lpu_servers"]) require.Equal(t, filepath.Join(runtimePath, "tokenizer"), configMap.Data["tokenizer_dir"]) + + t.Log("Resolve Cyborg server addresses to the last engine replica's actual Agent hostnames") + lastReplica := scaledPlan.ForReplica(replicas - 1) + servers := strings.Split(strings.ReplaceAll(configMap.Data["lpu_servers"], "${GROVE_PCSG_INDEX}", strconv.Itoa(int(replicas-1))), "\n") + for index, offset := range []int{0, 2} { + require.Equal(t, lastReplica.Agents[0].CliqueName+"-"+strconv.Itoa(offset), "test-dgd-0-"+servers[index]) + } }) } t.Log("Render a Cap'n Proto build with a nested tokenizer path") projection.configuredBuild.RuntimeTokenizerPath = "metadata/tokenizer" - configMap, err := workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + configMap, err := workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) t.Log("Verify the nested tokenizer path is rooted in the runtime build") require.NoError(t, err) @@ -74,14 +83,14 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Reject oversized rendered Cyborg configuration before publication") projection.configuredBuild.RuntimeTokenizerPath = strings.Repeat("x", corev1.MaxSecretSize) - _, err = workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + _, err = workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) require.ErrorContains(t, err, "rendered LPX ConfigMap") require.ErrorContains(t, err, "maximum is 1048576") t.Log("Render a Cap'n Proto build without tokenizer metadata") projection.configuredBuild.RuntimeTokenizerPath = "" projection.configuredBuild.Path = "relative/build" - _, err = workload.RenderCyborgConfigMap("test-namespace", "test-dgd", podSpec) + _, err = workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) t.Log("Verify missing tokenizer metadata is rejected") require.EqualError(t, err, "capnp manifest build is missing model.tokenizer.path") @@ -127,9 +136,9 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { workload := &SelectedWorkload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1} plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - configMap, err := workload.RenderCyborgConfigMap("test", "test-dgd", renderTestPodSpec()) + configMap, err := workload.RenderCyborgConfigMap("test", plan, renderTestPodSpec()) require.NoError(t, err) - prefix := plan.LPXScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" + prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(test.wantOffsets)) for index, offset := range test.wantOffsets { servers[index] = prefix + strconv.Itoa(offset) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index fc20abc9bed6..4a739cd589e9 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -21,7 +21,6 @@ import ( "github.com/pelletier/go-toml/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" ) @@ -35,22 +34,10 @@ type lpuModelStorage struct { mount corev1.VolumeMount } -func boundedAuxiliaryName(root, suffix string) string { - candidate := root + suffix - if len(candidate) <= validation.DNS1123SubdomainMaxLength { - return candidate - } - - // Distinguish overlong resource names using the full name, including its suffix. - digest := sha256.Sum256([]byte(candidate)) - hashSuffix := fmt.Sprintf("-%x", digest[:4]) - prefix := strings.TrimRight(candidate[:validation.DNS1123SubdomainMaxLength-len(hashSuffix)], "-.") - return prefix + hashSuffix -} - // LPUConfigMapName names the immutable runtime table using its Pod-template content hash. +// The root is the PCS identity carried by Grove's part-of Pod label. func LPUConfigMapName(root, configHash string) string { - return boundedAuxiliaryName(root, fmt.Sprintf("-lpu-%.16s", configHash)) + return fmt.Sprintf("%s-lpu-%.16s", root, configHash) } func renderLPUConfigMap( @@ -111,7 +98,7 @@ func renderRuntimeConfigMap(namespace, namePrefix string, data map[string]string Immutable: ptr.To(true), Data: data, } - configMap.Name = boundedAuxiliaryName(namePrefix, fmt.Sprintf("-%.16s", LPUConfigMapHash(configMap))) + configMap.Name = fmt.Sprintf("%s-%.16s", namePrefix, LPUConfigMapHash(configMap)) // Reject oversized configuration before any caller can publish it. totalSize := 0 diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 3b92a586b198..741063e87b5e 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -32,21 +32,24 @@ func TestRenderRuntimeConfigMapSizeLimit(t *testing.T) { require.ErrorContains(t, err, "data is 1048577 bytes; maximum is 1048576") } -func TestLPXAuxiliaryNamesAreBounded(t *testing.T) { - t.Log("Use a source name already at the Kubernetes length limit") - root := strings.Repeat("a", validation.DNS1123SubdomainMaxLength) - - t.Log("Render the LPU and decode ConfigMap names") - lpuName := LPUConfigMapName(root, "0123456789abcdef") - decodeName := boundedAuxiliaryName(root, "-decode-0123456789abcdef") - - t.Log("Keep names valid and distinct while leaving short names unchanged") - require.Len(t, lpuName, validation.DNS1123SubdomainMaxLength) - require.Len(t, decodeName, validation.DNS1123SubdomainMaxLength) - require.Empty(t, validation.IsDNS1123Subdomain(lpuName)) - require.Empty(t, validation.IsDNS1123Subdomain(decodeName)) - require.NotEqual(t, lpuName, decodeName) - require.Equal(t, "short-lpu-0123456789abcdef", LPUConfigMapName("short", "0123456789abcdef")) +func TestLPXRuntimeConfigNamesMatchPodIdentity(t *testing.T) { + for _, root := range []string{"short", strings.Repeat("a", MaxPodCliqueSetNameLength), strings.Repeat("a", validation.LabelValueMaxLength)} { + t.Run(root, func(t *testing.T) { + t.Log("Render immutable runtime tables for PCS and Pod-label identity bounds") + data := map[string]string{"runtime": "config"} + lpu, err := renderRuntimeConfigMap("test-namespace", root+"-lpu", data) + require.NoError(t, err) + decode, err := renderRuntimeConfigMap("test-namespace", root+"-decode", data) + require.NoError(t, err) + + t.Log("Resolve the same LPU table from Pod identity and preserve each role suffix") + require.Equal(t, LPUConfigMapName(root, LPUConfigMapHash(lpu)), lpu.Name) + require.Equal(t, root+"-lpu-"+LPUConfigMapHash(lpu)[:16], lpu.Name) + require.Equal(t, root+"-decode-"+LPUConfigMapHash(decode)[:16], decode.Name) + require.Empty(t, validation.IsDNS1123Subdomain(lpu.Name)) + require.Empty(t, validation.IsDNS1123Subdomain(decode.Name)) + }) + } } func TestLPURuntimeBuildRef(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index e839c9dbf190..151f17c4bbd7 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -8,9 +8,7 @@ package lpx import ( "crypto/sha256" "fmt" - "math" "slices" - "strconv" "strings" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -18,9 +16,15 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) -const minimumHashedCliqueNameLength = 8 +const ( + lpxScalingGroupTemplateName = "lpx" + conductorTemplateName = "cond" +) -const maximumCyborgPodIndex = math.MaxInt32 +// MaxPodCliqueSetNameLength reserves Grove's combined name budget for the fixed +// scaling group and longest role name, independently of the selected components. +const MaxPodCliqueSetNameLength = commonconsts.MaxCombinedGroveResourceNameLength - + len(lpxScalingGroupTemplateName) - len(conductorTemplateName) // LPXAttemptRequestStatus stores a name (lpxRequestName uses at most 60 bytes), // sha256 digest, and server-issued UUID. Reserve a full DNS label plus JSON syntax @@ -60,8 +64,6 @@ type MaterializationPlan struct { CyborgTemplate string // LPXScalingGroup is the materialized Grove scaling-group name. LPXScalingGroup string - // LPXScalingGroupTemplate is the Grove scaling-group template name. - LPXScalingGroupTemplate string // ReplicaIndex is the scaling-group replica represented by this projection. ReplicaIndex int32 // Replicas is the total scaling-group replica count. @@ -71,88 +73,57 @@ type MaterializationPlan struct { // PlanNodeLocalMaterialization derives the exact identities shared by graph // rendering and lifecycle observation without mutating the workload. func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*MaterializationPlan, error) { - componentName := w.LPXComponentName() - cyborgTemplateName := w.CyborgTemplateName() - if strings.TrimSpace(pcsName) == "" || strings.TrimSpace(componentName) == "" { - return nil, fmt.Errorf("PodCliqueSet and selected LPX component names are required") + // Leave room for the fixed scaling group and every LPX role in Grove's name budget. + if strings.TrimSpace(pcsName) == "" { + return nil, fmt.Errorf("PodCliqueSet name is required") + } + if len(pcsName) > MaxPodCliqueSetNameLength { + return nil, fmt.Errorf("PodCliqueSet name %q exceeds the LPX maximum of %d characters", pcsName, MaxPodCliqueSetNameLength) } - conductorTemplate := "" - if w.Pipeline() != PipelineLPX { - conductorTemplate = strings.ToLower(fmt.Sprintf("%s-%s", componentName, commonconsts.GroveRoleSuffixLeader)) - if problems := validation.IsDNS1123Label(conductorTemplate); len(problems) != 0 { - return nil, fmt.Errorf("conductor template name %q is invalid: %s", conductorTemplate, strings.Join(problems, "; ")) - } + // Nova and Cyborg both implement the authored conductor role. + conductorTemplate, cyborgTemplate := conductorTemplateName, "" + if w.Pipeline() == PipelineLPX { + conductorTemplate, cyborgTemplate = "", conductorTemplateName } agents := make([]ExpectedAgent, 0, len(w.modelProjections)) - agentScalingGroupTemplateBudget := validation.DNS1123LabelMaxLength for index, projection := range w.modelProjections { - name := strings.ToLower(fmt.Sprintf("%s-%s-m-%d", componentName, commonconsts.GroveRoleSuffixWorker, index)) - if problems := validation.IsDNS1123Label(name); len(problems) != 0 { - return nil, fmt.Errorf("Agent template name %q is invalid: %s", name, strings.Join(problems, "; ")) + name := "agt" + if len(w.modelProjections) > 1 { + name = fmt.Sprintf("agt%d", index) } agents = append(agents, ExpectedAgent{TemplateName: name, Replicas: projection.agentReplicas}) - agentScalingGroupTemplateBudget = min( - agentScalingGroupTemplateBudget, - materializedScalingGroupTemplateNameBudget(pcsName, name, projection.agentReplicas-1), - ) - } - maxScalingGroupTemplateLength := validation.DNS1123LabelMaxLength - if conductorTemplate != "" { - maxScalingGroupTemplateLength = materializedScalingGroupTemplateNameBudget(pcsName, conductorTemplate, 0) - } - maxScalingGroupTemplateLength = min(maxScalingGroupTemplateLength, agentScalingGroupTemplateBudget) - if cyborgTemplateName != "" { - maxScalingGroupTemplateLength = min( - maxScalingGroupTemplateLength, - materializedScalingGroupTemplateNameBudget( - pcsName, - cyborgTemplateName, - maximumCyborgPodIndex, - ), - ) - } - lpxScalingGroupTemplate, err := boundedNameTo(strings.ToLower(componentName), maxScalingGroupTemplateLength) - if err != nil { - return nil, fmt.Errorf("derive LPU scaling-group template name: %w", err) } // Construct the plan with Grove's canonical replica-zero scaling-group identity. plan := &MaterializationPlan{ PodCliqueSetName: pcsName, ConductorTemplate: conductorTemplate, - CyborgTemplate: cyborgTemplateName, + CyborgTemplate: cyborgTemplate, Agents: agents, LPXScalingGroup: grovecommon.GeneratePodCliqueScalingGroupName( - grovecommon.ResourceNameReplica{Name: pcsName, Replica: 0}, lpxScalingGroupTemplate, + grovecommon.ResourceNameReplica{Name: pcsName, Replica: 0}, lpxScalingGroupTemplateName, ), - LPXScalingGroupTemplate: lpxScalingGroupTemplate, - Replicas: w.scalingGroupReplicas, + Replicas: w.scalingGroupReplicas, } plan = plan.ForReplica(0) return plan, plan.ValidateReplicaCount() } -// ValidateReplicaCount checks the longest engine names and bounds request/status allocation. +// ValidateReplicaCount checks conductor/Agent hostnames and bounds request/status allocation. func (p *MaterializationPlan) ValidateReplicaCount() error { // Charge every model/replica for its complete persisted request identity. if p.Replicas < 0 || int64(p.Replicas)*int64(len(p.Agents)) > int64(schedulingAttemptRequestBytesBudget/maximumSchedulingAttemptRequestBytes) { return fmt.Errorf("LPX replica count exceeds the scheduling status size budget") } - plan := p.ForReplica(max(0, p.Replicas-1)) - if plan.ConductorClique != "" { - if err := validateMaterializedPodHostnameAtIndex("conductor", plan.ConductorClique, 0); err != nil { - return err - } - } - for _, agent := range plan.Agents { - if err := validateMaterializedPodHostnameAtIndex("Agent", agent.CliqueName, agent.Replicas-1); err != nil { + if p.ConductorTemplate != "" { + if err := p.validatePodHostname("conductor", p.ConductorTemplate, 0); err != nil { return err } } - if plan.CyborgClique != "" { - if err := validateMaterializedPodHostnameAtIndex("Cyborg", plan.CyborgClique, maximumCyborgPodIndex); err != nil { + for _, agent := range p.Agents { + if err := p.validatePodHostname("Agent", agent.TemplateName, agent.Replicas-1); err != nil { return err } } @@ -191,7 +162,9 @@ func materializedCliqueNameForReplica(pcsName, templateName string, replica int3 ) } -func validateMaterializedPodHostnameAtIndex(role, cliqueName string, podIndex int) error { +func (p *MaterializationPlan) validatePodHostname(role, templateName string, podIndex int) error { + // Validate the longest replica name without copying the full materialization plan. + cliqueName := materializedCliqueNameForReplica(p.LPXScalingGroup, templateName, max(0, p.Replicas-1)) hostname := materializedPodHostname(cliqueName, podIndex) if problems := validation.IsDNS1123Label(hostname); len(problems) != 0 { return fmt.Errorf( @@ -207,39 +180,3 @@ func validateMaterializedPodHostnameAtIndex(role, cliqueName string, podIndex in func materializedPodHostname(cliqueName string, podIndex int) string { return fmt.Sprintf("%s-%d", cliqueName, podIndex) } - -func materializedScalingGroupTemplateNameBudget(pcsName, childTemplateName string, podIndex int) int { - // Grove materializes "-0--0--". - return validation.DNS1123LabelMaxLength - - len(pcsName) - - len(childTemplateName) - - 7 - - len(strconv.Itoa(podIndex)) -} - -func boundedNameTo(candidate string, maxLength int) (string, error) { - if maxLength > validation.DNS1123LabelMaxLength { - maxLength = validation.DNS1123LabelMaxLength - } - if maxLength < minimumHashedCliqueNameLength { - return "", fmt.Errorf( - "available DNS label budget %d is smaller than the %d-character collision-resistant minimum", - maxLength, - minimumHashedCliqueNameLength, - ) - } - if len(candidate) <= maxLength && len(validation.IsDNS1123Label(candidate)) == 0 { - return candidate, nil - } - digest := sha256.Sum256([]byte(candidate)) - hash := fmt.Sprintf("%x", digest[:4]) - prefixLength := maxLength - len(hash) - 1 - if prefixLength <= 0 { - return hash, nil - } - prefix := strings.TrimRight(candidate[:min(len(candidate), prefixLength)], "-.") - if prefix == "" { - return hash, nil - } - return prefix + "-" + hash, nil -} diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index 0f5125434f28..dc2d028f3c57 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -12,6 +12,7 @@ import ( "testing" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" @@ -76,42 +77,69 @@ func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { hybridProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, hybrid))) lpuOnlyProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) - t.Log("Use component-derived names at the minimum collision-resistant group budget") - tests := []struct { + t.Log("Reserve readable roles at the maximum PCS length and scheduling replica count") + for _, test := range []struct { name string - pcsLength int + models int projection *ModelProjection }{ - { - name: "hybrid", pcsLength: 14, - projection: hybridProjection, - }, - { - name: "LPU-only", pcsLength: 26, - projection: lpuOnlyProjection, - }, - } - for _, test := range tests { + {name: "hybrid", models: 1, projection: hybridProjection}, + {name: "LPU-only", models: 1, projection: lpuOnlyProjection}, + {name: "SpecDecode", models: 2, projection: lpuOnlyProjection}, + {name: "maximum draft fanout", models: maxSpecDecodeNumDrafts + 1, projection: lpuOnlyProjection}, + } { t.Run(test.name, func(t *testing.T) { - test.projection.stage = "longcomponent" - workload := &SelectedWorkload{modelProjections: []*ModelProjection{test.projection}, scalingGroupReplicas: 1} - pcsName := strings.Repeat("a", test.pcsLength) - t.Log("Plan and verify materialization within the DNS hostname budget") + t.Log("Build independent projections without using component names in identities") + projections := make([]*ModelProjection, test.models) + for index := range projections { + projection := *test.projection + projection.stage = strings.Repeat("component", 7) + if test.models > 1 { + projection.pipeline = PipelineSpecDecode + } + projections[index] = &projection + } + workload := &SelectedWorkload{ + modelProjections: projections, + scalingGroupReplicas: int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes / test.models), + } + pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) plan, err := workload.PlanNodeLocalMaterialization(pcsName) require.NoError(t, err) - require.Equal(t, "longcomponent-wkr-m-0", plan.Agents[0].TemplateName) - require.Len(t, plan.LPXScalingGroupTemplate, 8) - hostname := materializedPodHostname(plan.Agents[0].CliqueName, plan.Agents[0].Replicas-1) - if test.projection.pipeline == PipelineLPX { - require.Equal(t, "longcomponent-engine-gpu", plan.CyborgTemplate) - hostname = materializedPodHostname(plan.CyborgClique, maximumCyborgPodIndex) + require.Equal(t, pcsName+"-0-lpx", plan.LPXScalingGroup) + + t.Log("Validate every role's Grove name budget and the last replica's Pod hostnames") + lastReplica := plan.ForReplica(plan.Replicas - 1) + hostnames := make([]string, 0, test.models+1) + if lastReplica.ConductorClique != "" { + require.Equal(t, "cond", plan.ConductorTemplate) + hostnames = append(hostnames, materializedPodHostname(lastReplica.ConductorClique, 0)) + require.Equal(t, commonconsts.MaxCombinedGroveResourceNameLength, + len(pcsName)+len(lpxScalingGroupTemplateName)+len(plan.ConductorTemplate)) } - require.Len(t, hostname, validation.DNS1123LabelMaxLength) - require.Empty(t, validation.IsDNS1123Label(hostname)) + for _, agent := range lastReplica.Agents { + require.LessOrEqual(t, len(pcsName)+len(lpxScalingGroupTemplateName)+len(agent.TemplateName), commonconsts.MaxCombinedGroveResourceNameLength) + hostnames = append(hostnames, materializedPodHostname(agent.CliqueName, agent.Replicas-1)) + } + if lastReplica.CyborgClique != "" { + require.Equal(t, "cond", plan.CyborgTemplate) + hostnames = append(hostnames, materializedPodHostname(lastReplica.CyborgClique, 0)) + } + for _, hostname := range hostnames { + require.Empty(t, validation.IsDNS1123Label(hostname)) + } + + t.Log("Renaming authored components leaves every materialized identity unchanged") + for _, projection := range projections { + projection.stage = "short" + } + renamed, err := workload.PlanNodeLocalMaterialization(pcsName) + require.NoError(t, err) + require.Equal(t, plan, renamed) - t.Log("Reject a PCS name that leaves one character too little for the group") + t.Log("Reject a PCS name one character beyond Grove's combined name budget") _, err = workload.PlanNodeLocalMaterialization(pcsName + "a") - require.ErrorContains(t, err, "derive LPU scaling-group template name") + require.ErrorContains(t, err, "exceeds the LPX maximum of 38 characters") }) } } diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 695508a1d951..4df87480cdef 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -266,7 +266,7 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { agent := *base.DeepCopy() t.Log("Lower the image into the Nova conductor and privileged SSH worker roles") - require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret"), "writable storage") + require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret"), "writable storage") require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), "writable storage") conductor = *base.DeepCopy() agent = *base.DeepCopy() @@ -286,7 +286,7 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { ) conductorMounts := slices.Clone(conductor.Containers[0].VolumeMounts) agentMounts := slices.Clone(agent.Containers[0].VolumeMounts) - require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "lpu-wkr-m-0", "ssh-secret")) + require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret")) require.NoError(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret")) t.Log("Verify the conductor starts Nova with immutable placement and source behavior") @@ -297,7 +297,7 @@ func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { require.Equal(t, []string{"/bin/nova"}, conductorContainer.Command) requireFlagValue(t, conductorContainer.Args, "--datacenter-config-filepath", "/configs/datacenter.toml") require.Contains(t, conductorContainer.Args, "--instance-model-name") - requireFlagValue(t, conductorContainer.Args, "--allocation", "lpu-wkr-m-0") + requireFlagValue(t, conductorContainer.Args, "--allocation", "agt") require.GreaterOrEqual(t, len(conductorContainer.Args), len(nodeLocalHXAgentEnvironmentArgs)) staticSuffixIndex := len(conductorContainer.Args) - len(nodeLocalHXAgentEnvironmentArgs) require.Equal(t, nodeLocalHXAgentEnvironmentArgs, conductorContainer.Args[staticSuffixIndex:]) @@ -378,7 +378,7 @@ func TestConfigureNodeLocalXTConductorSSHInitUsesMainImage(t *testing.T) { podSpec.Containers = append([]corev1.Container{{Name: "metrics", Image: "metrics-sidecar", ImagePullPolicy: corev1.PullAlways}}, podSpec.Containers...) t.Log("Configure the conductor without deriving its runtime identity from list position") - require.NoError(t, configureNodeLocalConductorRuntime(&podSpec, BuildFamilyXT, "lpu-wkr-m-0", "ssh-secret")) + require.NoError(t, configureNodeLocalConductorRuntime(&podSpec, BuildFamilyXT, "agt", "ssh-secret")) t.Log("Run SSH initialization with the resolved conductor image and preserve the sidecar") require.Equal(t, "metrics", podSpec.Containers[0].Name) @@ -399,7 +399,7 @@ func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() require.NoError(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", sshSecretName, false)) require.NoError(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, sshSecretName)) - require.NoError(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", sshSecretName)) + require.NoError(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "agt", sshSecretName)) for _, test := range []struct { name string @@ -926,7 +926,7 @@ func TestConfigureNodeLocalLPURuntimeRolesRejectsUnsupportedShape(t *testing.T) conductor := *base.DeepCopy() agent := *base.DeepCopy() - require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agent", "ssh-secret"), test.errorText) + require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret"), test.errorText) require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), test.errorText) }) } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index b566c3c2be19..49c3c653c3ee 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -31,10 +31,8 @@ const ( ExecutionRoleLabel = "lpx.nvidia.com/execution-role" ) -// RenderInput contains the materialization name and fresh stage templates consumed by rendering. +// RenderInput contains the fresh stage templates and runtime settings consumed by rendering. type RenderInput struct { - // MaterializationName is the owning LPXGraphDeployment's bounded identity root. - MaterializationName string // CyborgConfigMap is rendered before the user's Cyborg defaults are merged. CyborgConfigMap *corev1.ConfigMap // MinAvailable is the minimum number of complete engine replicas in the gang. @@ -65,6 +63,13 @@ func RenderSelectedNodeLocal( ) ([]client.Object, error) { projections := workload.modelProjections + // Hybrid input owns one GPU clique before the LPU roles are appended. + hybrid := projections[0].pipeline == PipelineLPX + var cyborg *grovev1alpha1.PodCliqueTemplateSpec + if hybrid { + cyborg = pcs.Spec.Template.Cliques[0] + } + workloadDigest := workload.Digest().String() agentTemplateNames := make([]string, 0, len(plan.Agents)) for _, agent := range plan.Agents { @@ -89,7 +94,7 @@ func RenderSelectedNodeLocal( } modelStorage.volume = *modelStorage.volume.DeepCopy() modelStorage.mount = *modelStorage.mount.DeepCopy() - configMap, err := renderLPUConfigMap(namespace, input.MaterializationName, modelStorage.mount.MountPath, projections, plan.Agents) + configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStorage.mount.MountPath, projections, plan.Agents) if err != nil { return nil, err } @@ -193,19 +198,17 @@ func RenderSelectedNodeLocal( pcs.Annotations = workloadAnnotations(pcs.Annotations, workloadDigest) // Node-local uses canonical annotation absence for backward compatibility. delete(pcs.Annotations, ExecutionBackendAnnotation) - var cyborg *grovev1alpha1.PodCliqueTemplateSpec - for _, clique := range pcs.Spec.Template.Cliques { - if clique.Name == plan.CyborgTemplate { - cyborg = clique - } - } selectedTemplateNames := agentTemplateNames if conductorTemplateName != "" { selectedTemplateNames = append([]string{conductorTemplateName}, selectedTemplateNames...) } - hybrid := projections[0].pipeline == PipelineLPX if hybrid { + // Bound GPU hostnames using the rendered width of the last engine replica. + if err := plan.validatePodHostname("Cyborg", plan.CyborgTemplate, int(cyborg.Spec.Replicas)-1); err != nil { + return nil, err + } + // HX Cyborg may inherit its Agent's configuration mount. container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) if cyborgConfigMap == nil && slices.ContainsFunc(container.VolumeMounts, @@ -220,7 +223,6 @@ func RenderSelectedNodeLocal( workloadDigest, modelStorage, agentTemplateNames, - plan, cyborgConfigMap, ); err != nil { return nil, err @@ -233,7 +235,7 @@ func RenderSelectedNodeLocal( members = append(members, plan.CyborgTemplate) } pcs.Spec.Template.PodCliqueScalingGroupConfigs = []grovev1alpha1.PodCliqueScalingGroupConfig{{ - Name: plan.LPXScalingGroupTemplate, + Name: lpxScalingGroupTemplateName, CliqueNames: members, Annotations: map[string]string{WorkloadDigestAnnotation: workloadDigest}, Replicas: ptr.To(plan.Replicas), diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 0a776753247b..5768928b933d 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -15,6 +15,7 @@ import ( manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -25,7 +26,7 @@ import ( const ( testRenderComponentName = "lpu" - testAgentTemplateName = "lpu-wkr-m-0" + testAgentTemplateName = "agt" testTargetStageName = "target" ) @@ -86,7 +87,7 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model return nil, err } if workload.BuildFamily() == BuildFamilyXT && workload.Pipeline() == PipelineLPX { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, input.MaterializationName, input.Stages[testRenderComponentName].Spec) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, plan, input.Stages[testRenderComponentName].Spec) if err != nil { return nil, err } @@ -147,9 +148,8 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Log("Render into the fresh PCS without retaining stale runtime identity") rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - MaterializationName: "test-dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, - SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, + SSHSecretName: "ssh-secret", }) require.NoError(t, err) require.Same(t, pcs, rendered) @@ -164,14 +164,14 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Log("Resolve runtime mounts using the authored storage path") if test.family == lpxv1alpha1.TargetFamilyXt8888 { - conductor := namedClique(t, rendered, "lpu-ldr") + conductor := namedClique(t, rendered, "cond") require.Equal(t, "kept", conductor.Annotations["user"]) require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) } else { - cyborg := namedClique(t, rendered, "lpu-engine-gpu").Spec.PodSpec.Containers[0] + cyborg := namedClique(t, rendered, "cond").Spec.PodSpec.Containers[0] require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/model-cache"}) require.Contains(t, cyborg.Env, corev1.EnvVar{ Name: "GBUILD_MANIFEST_PATH", Value: "/model-cache/model-build/manifest.v2.capnp.bin", @@ -228,20 +228,21 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { t.Log("Render conductor and Agent roles from the base PodSpec") rendered, err := renderSelectedForTest(renderTestPCS(false), []*ModelProjection{projection}, RenderInput{ - MaterializationName: "test-dgd", - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, - SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, + SSHSecretName: "ssh-secret", }) require.NoError(t, err) t.Log("Verify conductor-owned placement and entrypoint behavior") - conductor := namedClique(t, rendered, "lpu-ldr") + conductor := namedClique(t, rendered, "cond") require.Nil(t, conductor.Spec.PodSpec.Affinity) require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) + requireFlagValue(t, conductor.Spec.PodSpec.Containers[0].Args, "--allocation", "agt") + require.Equal(t, []string{"agt"}, conductor.Spec.StartsAfter) t.Log("Verify Agent-owned affinity and logical model binding") agent := namedClique(t, rendered, testAgentTemplateName) @@ -299,7 +300,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { plan, err := selected.PlanNodeLocalMaterialization(pcs.Name) require.NoError(t, err) extraResources, err := RenderSelectedNodeLocal(pcs, selected, plan, RenderInput{ - MaterializationName: source.Name, Stages: stages, SSHSecretName: "ssh-secret", + Stages: stages, SSHSecretName: "ssh-secret", }) require.NoError(t, err) @@ -340,6 +341,29 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "[draft.iop]")) require.Equal(t, "draft0\ndraft1\ntarget", configMap.Data["partition_models"]) require.Equal(t, before, source) + + t.Log("Bind conductor allocation and Nova hostnames to the renamed Agent cliques") + requireFlagValue(t, conductor.Spec.PodSpec.Containers[0].Args, "--allocation", "agt0:agt1:agt2") + require.Equal(t, []string{"agt0", "agt1", "agt2"}, conductor.Spec.StartsAfter) + var datacenter struct { + Datacenters struct { + Racks map[string]struct { + NodeNameTemplate string `toml:"node_name_template"` + } + } + } + require.NoError(t, toml.Unmarshal([]byte(configMap.Data["datacenter.toml"]), &datacenter)) + require.Len(t, datacenter.Datacenters.Racks, len(plan.Agents)) + for _, agent := range plan.Agents { + rack := datacenter.Datacenters.Racks[agent.TemplateName] + hostname := strings.NewReplacer( + "${GROVE_PCSG_NAME}", plan.LPXScalingGroup, + "${GROVE_PCSG_INDEX}", "0", + "{node}", "0", + "${GROVE_HEADLESS_SERVICE}", pcs.Name, + ).Replace(rack.NodeNameTemplate) + require.Equal(t, agent.CliqueName+"-0."+pcs.Name, hostname) + } } func projectRenderFixture(t *testing.T, family lpxv1alpha1.TargetFamily, pipeline Pipeline, snapshot *BuildSnapshot) *ModelProjection { @@ -365,10 +389,10 @@ func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { } one := int32(1) pcs.Spec.Template.Cliques = []*grovev1alpha1.PodCliqueTemplateSpec{{ - Name: "lpu-engine-gpu", + Name: "cond", Labels: map[string]string{"kai.scheduler/queue": "legacy-queue"}, Spec: grovev1alpha1.PodCliqueSpec{ - RoleName: "lpu-engine-gpu", Replicas: 1, MinAvailable: &one, + RoleName: "cond", Replicas: 1, MinAvailable: &one, PodSpec: corev1.PodSpec{ SchedulerName: corev1.DefaultSchedulerName, Volumes: []corev1.Volume{ diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index 89145afe2de5..c84f34a09753 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -38,12 +38,12 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Equal(t, PipelineSingle, hx.Pipeline()) require.Equal(t, BuildFamilyHX, hx.BuildFamily()) require.Equal(t, lpxv1alpha1.WorkloadModeV3HxLPUOnly, hx.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) - require.Empty(t, hx.CyborgTemplateName()) require.Equal(t, "LPX", hx.LPXComponentName()) plan, err := hx.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) - require.Equal(t, "lpx", plan.LPXScalingGroupTemplate) - require.Equal(t, "lpx-ldr", plan.ConductorTemplate) + require.Equal(t, "test-pcs-0-lpx", plan.LPXScalingGroup) + require.Equal(t, "cond", plan.ConductorTemplate) + require.Empty(t, plan.CyborgTemplate) require.NotEmpty(t, plan.ConductorClique) t.Log("Scale Nova engines without changing their model or workload digest") @@ -81,10 +81,10 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Equal(t, PipelineLPX, xt.Pipeline()) require.Equal(t, BuildFamilyXT, xt.BuildFamily()) require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, xt.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) - require.Equal(t, "lpx-engine-gpu", xt.CyborgTemplateName()) require.Len(t, xt.modelProjections[0].configuredBuild.Partitions, 2) plan, err = xt.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) + require.Equal(t, "cond", plan.CyborgTemplate) require.EqualValues(t, 2, plan.Replicas) replica := plan.ForReplica(1) require.NotEqual(t, plan.Agents[0].CliqueName, replica.Agents[0].CliqueName) @@ -188,7 +188,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { require.Equal(t, "lpx", projections[2].stage) require.Equal( t, - []string{"lpx-wkr-m-0", "lpx-wkr-m-1", "lpx-wkr-m-2"}, + []string{"agt0", "agt1", "agt2"}, []string{ plan.Agents[0].TemplateName, plan.Agents[1].TemplateName, diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go index f0aea97afdd4..57ebceeabdec 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go @@ -108,7 +108,7 @@ func TestXTRuntimesRejectSSHVolumeCollision(t *testing.T) { const wantError = `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"` require.ErrorContains(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", "mpi.ssh", false), wantError) require.ErrorContains(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "mpi.ssh"), wantError) - require.ErrorContains(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "lpu-wkr-m-0", "mpi.ssh"), wantError) + require.ErrorContains(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "agt", "mpi.ssh"), wantError) t.Log("Preserve the authored volume in every rejected runtime") for _, pod := range []*corev1.PodSpec{direct, agent, conductor} { diff --git a/deploy/operator/internal/dynamo/lpx/selected_workload.go b/deploy/operator/internal/dynamo/lpx/selected_workload.go index ac4091d14b8b..041e40d0b9d6 100644 --- a/deploy/operator/internal/dynamo/lpx/selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/selected_workload.go @@ -5,8 +5,6 @@ package lpx -import "strings" - // SelectedWorkload is the immutable result of resolving exactly one DGD for // LPX scheduling and runtime materialization. // Its methods require a successfully resolved, non-nil workload with projections. @@ -41,12 +39,3 @@ func (w *SelectedWorkload) Pipeline() Pipeline { func (w *SelectedWorkload) LPXComponentName() string { return w.modelProjections[len(w.modelProjections)-1].stage } - -// CyborgTemplateName derives the hybrid engine's GPU-role template name. -// A non-hybrid workload has no Cyborg template. -func (w *SelectedWorkload) CyborgTemplateName() string { - if w.Pipeline() != PipelineLPX { - return "" - } - return strings.ToLower(w.LPXComponentName()) + "-engine-gpu" -} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 12129f36cddc..beebf1ccfcc9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -3,23 +3,23 @@ data: lpu_servers: |- - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-1 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-2 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-3 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-4 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-5 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-6 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-7 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-8 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-9 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-10 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-11 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-12 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-13 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-14 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-15 - lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-16 + lpx-${GROVE_PCSG_INDEX}-agt-0 + lpx-${GROVE_PCSG_INDEX}-agt-1 + lpx-${GROVE_PCSG_INDEX}-agt-2 + lpx-${GROVE_PCSG_INDEX}-agt-3 + lpx-${GROVE_PCSG_INDEX}-agt-4 + lpx-${GROVE_PCSG_INDEX}-agt-5 + lpx-${GROVE_PCSG_INDEX}-agt-6 + lpx-${GROVE_PCSG_INDEX}-agt-7 + lpx-${GROVE_PCSG_INDEX}-agt-8 + lpx-${GROVE_PCSG_INDEX}-agt-9 + lpx-${GROVE_PCSG_INDEX}-agt-10 + lpx-${GROVE_PCSG_INDEX}-agt-11 + lpx-${GROVE_PCSG_INDEX}-agt-12 + lpx-${GROVE_PCSG_INDEX}-agt-13 + lpx-${GROVE_PCSG_INDEX}-agt-14 + lpx-${GROVE_PCSG_INDEX}-agt-15 + lpx-${GROVE_PCSG_INDEX}-agt-16 tokenizer_dir: /testdata/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer immutable: true metadata: @@ -27,7 +27,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: lstg3y7d-decode-cd1df68308eaf508 + name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a namespace: inference-engine --- data: @@ -36,9 +36,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 gas_dir: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c model_config.toml: | @@ -185,7 +185,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: lstg3y7d-lpu-70f8ece55681fe1b + name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 namespace: inference-engine --- metadata: @@ -196,7 +196,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 labels: nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm - name: lstg3y7d + name: dgd-lpx-v2-vllm-94cd namespace: inference-engine spec: replicas: 1 @@ -206,200 +206,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: cd1df68308eaf508bdf4d2ab25abbdc0c374769b498191e1d9dcca12dbfa8d88 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 - scheduling.lpu.nvidia.com/role: cyborg-worker - labels: - lpx.nvidia.com/execution-role: gpu - lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm - nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: dgd-lpx-v2-vllm-lpu - name: lpu-engine-gpu - spec: - minAvailable: 1 - podSpec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - c1v2-b300n1-nic1 - containers: - - args: - - --expand-hosts - - -- - - /usr/local/bin/cyborg - - --dynamo-model-name - - meta-llama/Llama-3.2-1B-Instruct - - --model-card - - /etc/cyborg/model-card.json - - --use-disagg-prefill - - --nixl-kv-layout - - HND - - --num-layers - - "16" - command: - - /usr/local/bin/cyborg-entrypoint - env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - - name: CYBORG_FPGA_GPI_REPLICA_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: DYN_COMPONENT - value: decode - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_FORWARDPASS_METRIC_PORT - value: "20380" - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: DYN_NAMESPACE - value: inference-engine-dgd-lpx-v2-vllm - - name: DYN_PARENT_DGD_K8S_NAME - value: dgd-lpx-v2-vllm - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: inference-engine - - name: DYN_SYSTEM_ENABLED - value: "true" - - name: DYN_SYSTEM_PORT - value: "9090" - - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS - value: '["generate"]' - - name: ETCD_ENDPOINTS - value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin - - name: GLUE_RDMA_PATH_MTU - value: "1024" - - name: NATS_SERVER - value: nats-address - - name: NIXL_TELEMETRY_ENABLE - value: "n" - - name: NIXL_TELEMETRY_EXPORTER - value: prometheus - - name: NIXL_TELEMETRY_PROMETHEUS_PORT - value: "19090" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: RDMA_PORT - value: "19877" - - name: SERVER_HOSTS_FILE - value: /tmp/lpu_servers - - name: TOKENIZER_DIR - valueFrom: - configMapKeyRef: - key: tokenizer_dir - name: lstg3y7d-decode-cd1df68308eaf508 - - name: TOTAL_REPLICAS - value: "1" - image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 - livenessProbe: - failureThreshold: 1 - httpGet: - path: /live - port: system - periodSeconds: 5 - timeoutSeconds: 4 - name: main - ports: - - containerPort: 9090 - name: system - protocol: TCP - - containerPort: 19090 - name: nixl - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /health - port: system - periodSeconds: 10 - timeoutSeconds: 4 - resources: - limits: - nvidia.com/gpu: "1" - startupProbe: - failureThreshold: 720 - httpGet: - path: /live - port: system - periodSeconds: 10 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /configs - name: config - - mountPath: /etc/cyborg/model-card.json - name: cyborg-model-card - readOnly: true - subPath: model-card.json - - mountPath: /dev/infiniband - name: infiniband - - mountPath: /nfs - name: model-storage - - mountPath: /dev/shm - name: shared-memory - restartPolicy: Always - schedulerName: default-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - tolerations: - - key: nvidia.com/gpu - operator: Exists - volumes: - - configMap: - name: dgd-lpx-v2-vllm-cyborg-model-card - name: cyborg-model-card - - hostPath: - path: /dev/infiniband - type: Directory - name: infiniband - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lstg3y7d-decode-cd1df68308eaf508 - name: config - replicas: 1 - roleName: lpu-engine-gpu - startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 70f8ece55681fe1ba61a15bbecddf456fe9857e535d513e6ca0ac2fd00694a00 + nvidia.com/extra-resources-hash: d73a69201a8e19d2013c560e1155903bacf71d3a2cbdaaf4f3aea4deafa8d033 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/model: default @@ -414,7 +221,7 @@ spec: nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm nvidia.com/metrics-enabled: "true" nvidia.com/selector: dgd-lpx-v2-vllm-lpu - name: lpu-wkr-m-0 + name: agt spec: minAvailable: 17 podSpec: @@ -489,7 +296,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: lstg3y7d-lpu-70f8ece55681fe1b + name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -523,7 +330,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: lstg3y7d-lpu-70f8ece55681fe1b + name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 - name: UCX_TLS value: tcp image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f @@ -616,10 +423,203 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: lstg3y7d-lpu-70f8ece55681fe1b + name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 name: config replicas: 17 - roleName: lpu-wkr-m-0 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 3aba9d08778da54a7d7c53f0d1f5bdeb06f931a8f6390287d34c5f039dc58075 + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm + nvidia.com/dynamo-namespace: inference-engine-dgd-lpx-v2-vllm + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-lpx-v2-vllm-lpu + name: cond + spec: + minAvailable: 1 + podSpec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - c1v2-b300n1-nic1 + containers: + - args: + - --expand-hosts + - -- + - /usr/local/bin/cyborg + - --dynamo-model-name + - meta-llama/Llama-3.2-1B-Instruct + - --model-card + - /etc/cyborg/model-card.json + - --use-disagg-prefill + - --nixl-kv-layout + - HND + - --num-layers + - "16" + command: + - /usr/local/bin/cyborg-entrypoint + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: inference-engine-dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-lpx-v2-vllm + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin + - name: GLUE_RDMA_PATH_MTU + value: "1024" + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: SERVER_HOSTS_FILE + value: /tmp/lpu_servers + - name: TOKENIZER_DIR + valueFrom: + configMapKeyRef: + key: tokenizer_dir + name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a + - name: TOTAL_REPLICAS + value: "1" + image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: "1" + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /configs + name: config + - mountPath: /etc/cyborg/model-card.json + name: cyborg-model-card + readOnly: true + subPath: model-card.json + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: nvidia.com/gpu + operator: Exists + volumes: + - configMap: + name: dgd-lpx-v2-vllm-cyborg-model-card + name: cyborg-model-card + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a + name: config + replicas: 1 + roleName: cond + startsAfter: + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -629,10 +629,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 cliqueNames: - - lpu-wkr-m-0 - - lpu-engine-gpu + - agt + - cond minAvailable: 1 - name: lpu + name: lpx replicas: 2 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 87a5df1486e1..889c5a52c696 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 data: - lpu_servers: lpu-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-0 + lpu_servers: lpx-${GROVE_PCSG_INDEX}-agt-0 tokenizer_dir: /testdata/node-local-v2-connected-lpx/tokenizer immutable: true metadata: @@ -10,7 +10,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: ltudmjcv-decode-97936407dc305636 + name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 namespace: test --- data: @@ -19,9 +19,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 gas_dir: /nfs/node-local-v2-connected-lpx model_config.toml: | @@ -49,7 +49,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - name: ltudmjcv-lpu-6bebe5759af346e9 + name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 namespace: test --- metadata: @@ -60,7 +60,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid - name: ltudmjcv + name: node-local-v2-hybrid-9d06 namespace: test spec: replicas: 1 @@ -70,174 +70,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 97936407dc3056369df53ef5ff8f62f4b4f451ae1af986e6fc78ad4d75c37c31 - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 - scheduling.lpu.nvidia.com/role: cyborg-worker - labels: - lpx.nvidia.com/execution-role: gpu - lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid - nvidia.com/dynamo-namespace: test-node-local-v2-hybrid - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v2-hybrid-lpu - name: lpu-engine-gpu - spec: - minAvailable: 1 - podSpec: - containers: - - args: - - --expand-hosts - - -- - - /opt/gpu-runtime - - serve - command: - - /usr/local/bin/cyborg-entrypoint - env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - - name: CYBORG_FPGA_GPI_REPLICA_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: DYN_COMPONENT - value: decode - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_FORWARDPASS_METRIC_PORT - value: "20380" - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: DYN_NAMESPACE - value: test-node-local-v2-hybrid - - name: DYN_PARENT_DGD_K8S_NAME - value: node-local-v2-hybrid - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: test - - name: DYN_SYSTEM_ENABLED - value: "true" - - name: DYN_SYSTEM_PORT - value: "9090" - - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS - value: '["generate"]' - - name: ETCD_ENDPOINTS - value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/node-local-v2-connected-lpx/manifest.v2.capnp.bin - - name: NATS_SERVER - value: nats-address - - name: NIXL_TELEMETRY_ENABLE - value: "n" - - name: NIXL_TELEMETRY_EXPORTER - value: prometheus - - name: NIXL_TELEMETRY_PROMETHEUS_PORT - value: "19090" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: RDMA_PORT - value: "19877" - - name: SERVER_HOSTS_FILE - value: /tmp/lpu_servers - - name: TOKENIZER_DIR - valueFrom: - configMapKeyRef: - key: tokenizer_dir - name: ltudmjcv-decode-97936407dc305636 - - name: TOTAL_REPLICAS - value: "1" - image: example.invalid/gpu-runtime:test - livenessProbe: - failureThreshold: 1 - httpGet: - path: /live - port: system - periodSeconds: 5 - timeoutSeconds: 4 - name: main - ports: - - containerPort: 9090 - name: system - protocol: TCP - - containerPort: 19090 - name: nixl - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /health - port: system - periodSeconds: 10 - timeoutSeconds: 4 - resources: - claims: - - name: gpu - startupProbe: - failureThreshold: 720 - httpGet: - path: /live - port: system - periodSeconds: 10 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /configs - name: config - - mountPath: /dev/infiniband - name: infiniband - - mountPath: /nfs - name: model-storage - - mountPath: /dev/shm - name: shared-memory - resourceClaims: - - name: gpu - resourceClaimTemplateName: decode-gpu - restartPolicy: Always - schedulerName: default-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - volumes: - - hostPath: - path: /dev/infiniband - type: Directory - name: infiniband - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: ltudmjcv-decode-97936407dc305636 - name: config - replicas: 1 - roleName: lpu-engine-gpu - startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 6bebe5759af346e96ad5bf5bb097effd88b0b56bb7339adbda3868add7aac2e9 + nvidia.com/extra-resources-hash: 5a21b650237fd262f004a640a67c5ecac0781784fa057703ee95223ab6a30c59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/model: default @@ -252,7 +85,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v2-hybrid nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v2-hybrid-lpu - name: lpu-wkr-m-0 + name: agt spec: minAvailable: 4 podSpec: @@ -278,7 +111,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: ltudmjcv-lpu-6bebe5759af346e9 + name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -312,7 +145,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: ltudmjcv-lpu-6bebe5759af346e9 + name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -405,10 +238,177 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: ltudmjcv-lpu-6bebe5759af346e9 + name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 name: config replicas: 4 - roleName: lpu-wkr-m-0 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: node-local-v2-hybrid + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: f8838533ed7ff8246dfdef9db2185d041489d1dc07addbf57f5893ae77eb5139 + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid + nvidia.com/dynamo-namespace: test-node-local-v2-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-hybrid-lpu + name: cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - --expand-hosts + - -- + - /opt/gpu-runtime + - serve + command: + - /usr/local/bin/cyborg-entrypoint + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v2-connected-lpx/manifest.v2.capnp.bin + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: RDMA_PORT + value: "19877" + - name: SERVER_HOSTS_FILE + value: /tmp/lpu_servers + - name: TOKENIZER_DIR + valueFrom: + configMapKeyRef: + key: tokenizer_dir + name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 + - name: TOTAL_REPLICAS + value: "1" + image: example.invalid/gpu-runtime:test + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + claims: + - name: gpu + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /configs + name: config + - mountPath: /dev/infiniband + name: infiniband + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - hostPath: + path: /dev/infiniband + type: Directory + name: infiniband + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 + name: config + replicas: 1 + roleName: cond + startsAfter: + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -418,10 +418,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 cliqueNames: - - lpu-wkr-m-0 - - lpu-engine-gpu + - agt + - cond minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index c4d57c74f0e9..38637a1c3f57 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -7,9 +7,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 model_config.toml: | type = 'Single' @@ -53,7 +53,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: lgoyrwtj-lpu-62ffb8651b497e5f + name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 namespace: test --- metadata: @@ -64,7 +64,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only - name: lgoyrwtj + name: node-local-v2-lpu-only-33b1 namespace: test spec: replicas: 1 @@ -74,7 +74,151 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 62ffb8651b497e5f141bbf328eecc74fc4f1729d8314b66928832afd105631ee + nvidia.com/extra-resources-hash: 6acbae87b5dfe4f0837a02ffddf18a9af54e8ce74b645d0267c059517b8c1e2b + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v2-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v2-lpu-only-lpu + name: agt + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v2-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /configs + name: config + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 + name: config + replicas: 4 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: node-local-v2-lpu-only + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 6acbae87b5dfe4f0837a02ffddf18a9af54e8ce74b645d0267c059517b8c1e2b scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/role: conductor @@ -88,7 +232,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v2-lpu-only nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v2-lpu-only-lpu - name: lpu-ldr + name: cond spec: minAvailable: 1 podSpec: @@ -118,7 +262,7 @@ spec: -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - --allocation - - lpu-wkr-m-0 + - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" @@ -247,156 +391,12 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: lgoyrwtj-lpu-62ffb8651b497e5f + name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 name: config replicas: 1 - roleName: lpu-ldr + roleName: cond startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 62ffb8651b497e5f141bbf328eecc74fc4f1729d8314b66928832afd105631ee - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 - scheduling.lpu.nvidia.com/model: default - scheduling.lpu.nvidia.com/role: agent - scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly - labels: - lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only - nvidia.com/dynamo-namespace: test-node-local-v2-lpu-only - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v2-lpu-only-lpu - name: lpu-wkr-m-0 - spec: - minAvailable: 4 - podSpec: - containers: - - args: - - serve - command: - - /opt/dynamo-lpu - env: - - name: DYN_COMPONENT - value: lpx - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_NAMESPACE - value: test-node-local-v2-lpu-only - - name: DYN_PARENT_DGD_K8S_NAME - value: node-local-v2-lpu-only - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: test - - name: ETCD_ENDPOINTS - value: etcd-address - - name: LPU_RUN_SYSTEM_INIT - value: "true" - - name: NATS_SERVER - value: nats-address - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: example.invalid/lpu-runtime:test - name: agent - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: 2222 - timeoutSeconds: 1 - resources: - limits: - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" - requests: - cpu: "62" - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" - securityContext: - privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /nfs - name: model-storage - - mountPath: /configs - name: config - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys - - mountPath: /ssh-pk - name: ssh-secret - readOnly: true - - mountPath: /dev/hugepages - name: hugepages - - mountPath: /dev/shm - name: shared-memory - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - restartPolicy: Always - schedulerName: lpx-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - tolerations: - - key: lpu.nvidia.com/node - operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - emptyDir: - medium: HugePages - name: hugepages - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lgoyrwtj-lpu-62ffb8651b497e5f - name: config - replicas: 4 - roleName: lpu-wkr-m-0 + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -406,10 +406,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 cliqueNames: - - lpu-ldr - - lpu-wkr-m-0 + - cond + - agt minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 4c59040fffb5..e06d1f8a99f4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -7,19 +7,19 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt0] available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 - [datacenters.racks.lpu-wkr-m-1] + [datacenters.racks.agt1] available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-1-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 - [datacenters.racks.lpu-wkr-m-2] + [datacenters.racks.agt2] available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-2-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt2-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 model_config.toml: | draft_to_target_port = 23456 @@ -122,7 +122,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: lmppuelh-lpu-29156c984c942fc4 + name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 namespace: test --- metadata: @@ -133,7 +133,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode - name: lmppuelh + name: node-local-v2-specdecode-63df namespace: test spec: replicas: 1 @@ -143,71 +143,30 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e + nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 - scheduling.lpu.nvidia.com/role: conductor + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 + scheduling.lpu.nvidia.com/model: draft0 + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode nvidia.com/dynamo-namespace: test-node-local-v2-specdecode nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v2-specdecode-lpu - name: lpu-ldr + nvidia.com/selector: node-local-v2-specdecode-draft + name: agt0 spec: - minAvailable: 1 + minAvailable: 4 podSpec: containers: - args: - - --agent-path - - /bin/agent - - --agent-wd - - /tmp - - --conductor-bind-host - - 0.0.0.0 - - --conductor-hostname - - $(POD_IP) - - --control-server-addr - - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - - --model-config-filepath - - /configs/model_config.toml - - --json-logs - - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - - --mpi-use-tcp - - --allocation - - lpu-wkr-m-0:lpu-wkr-m-1:lpu-wkr-m-2 - - --agent-disable-auto-restart - - --agent-server-max-retries - - "60" - - --agent-server-retry-sleep-dur - - 2s - - --controlplane-start-port - - "12345" - - --graceful-shutdown-timeout - - 0s - - --response-timeout - - 120s - - --frontend-type - - dynamo - - --endpoint-id - - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - - --store-backend - - kubernetes + - --partition-metadata command: - - /bin/nova + - /bin/quasar-entrypoint env: - name: DYN_COMPONENT value: lpx @@ -221,15 +180,12 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address + - name: LPU_MODEL_NAME + value: draft0 + - name: LPU_RUN_SYSTEM_INIT + value: "true" - name: NATS_SERVER value: nats-address - - name: OMPI_ALLOW_RUN_AS_ROOT - value: "1" - - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM - value: "1" - - name: OMPI_MCA_plm_rsh_args - value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key - name: POD_IP valueFrom: fieldRef: @@ -248,19 +204,25 @@ spec: fieldRef: fieldPath: metadata.uid image: example.invalid/lpu-runtime:test - name: conductor + name: agent readinessProbe: failureThreshold: 3 - httpGet: - path: /readyz - port: 51045 - scheme: HTTP initialDelaySeconds: 10 periodSeconds: 5 successThreshold: 1 + tcpSocket: + port: 2222 timeoutSeconds: 1 - resources: {} + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" securityContext: + privileged: true runAsGroup: 0 runAsNonRoot: false runAsUser: 0 @@ -269,67 +231,68 @@ spec: name: model-storage - mountPath: /configs name: config - - mountPath: /tmp/dynamo-lpu-ssh - name: single-v2-ssh-key - - mountPath: /dev/shm - name: shared-memory - initContainers: - - args: - - -m - - "0600" - - /ssh-pk/private.key - - /tmp/dynamo-lpu-ssh/private.key - command: - - /bin/install - image: example.invalid/lpu-runtime:test - name: prepare-ssh-key - resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /tmp/dynamo-lpu-ssh - name: single-v2-ssh-key + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys - mountPath: /ssh-pk name: ssh-secret readOnly: true + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - emptyDir: {} - name: single-v2-ssh-key + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory - - name: ssh-secret - secret: - defaultMode: 384 - secretName: ssh-secret - configMap: - name: lmppuelh-lpu-29156c984c942fc4 + name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 name: config - replicas: 1 - roleName: lpu-ldr - startsAfter: - - lpu-wkr-m-0 - - lpu-wkr-m-1 - - lpu-wkr-m-2 + replicas: 4 + roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e + nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 - scheduling.lpu.nvidia.com/model: draft0 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b + scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: @@ -341,7 +304,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v2-specdecode nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v2-specdecode-draft - name: lpu-wkr-m-0 + name: agt1 spec: minAvailable: 4 podSpec: @@ -364,7 +327,7 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: LPU_MODEL_NAME - value: draft0 + value: draft1 - name: LPU_RUN_SYSTEM_INIT value: "true" - name: NATS_SERVER @@ -465,29 +428,29 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: lmppuelh-lpu-29156c984c942fc4 + name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 name: config replicas: 4 - roleName: lpu-wkr-m-0 + roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e + nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b - scheduling.lpu.nvidia.com/model: draft1 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e + scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/stage: draft - nvidia.com/dynamo-component: draft + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode nvidia.com/dynamo-namespace: test-node-local-v2-specdecode nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v2-specdecode-draft - name: lpu-wkr-m-1 + nvidia.com/selector: node-local-v2-specdecode-lpu + name: agt2 spec: minAvailable: 4 podSpec: @@ -510,7 +473,7 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: LPU_MODEL_NAME - value: draft1 + value: target - name: LPU_RUN_SYSTEM_INIT value: "true" - name: NATS_SERVER @@ -611,21 +574,20 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: lmppuelh-lpu-29156c984c942fc4 + name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 name: config replicas: 4 - roleName: lpu-wkr-m-1 + roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 29156c984c942fc48e98a2a12220ad6fd7ae1f2dedf9eae6214c1507949a114e + nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e - scheduling.lpu.nvidia.com/model: target - scheduling.lpu.nvidia.com/role: agent - scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -633,15 +595,57 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v2-specdecode nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v2-specdecode-lpu - name: lpu-wkr-m-2 + name: cond spec: - minAvailable: 4 + minAvailable: 1 podSpec: containers: - args: - - --partition-metadata + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - agt0:agt1:agt2 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --controlplane-start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes command: - - /bin/quasar-entrypoint + - /bin/nova env: - name: DYN_COMPONENT value: lpx @@ -655,12 +659,15 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPU_MODEL_NAME - value: target - - name: LPU_RUN_SYSTEM_INIT - value: "true" - name: NATS_SERVER value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key - name: POD_IP valueFrom: fieldRef: @@ -679,25 +686,19 @@ spec: fieldRef: fieldPath: metadata.uid image: example.invalid/lpu-runtime:test - name: agent + name: conductor readinessProbe: failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP initialDelaySeconds: 10 periodSeconds: 5 successThreshold: 1 - tcpSocket: - port: 2222 timeoutSeconds: 1 - resources: - limits: - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" - requests: - cpu: "62" - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" + resources: {} securityContext: - privileged: true runAsGroup: 0 runAsNonRoot: false runAsUser: 0 @@ -706,61 +707,60 @@ spec: name: model-storage - mountPath: /configs name: config - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory + initContainers: + - args: + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key - mountPath: /ssh-pk name: ssh-secret readOnly: true - - mountPath: /dev/hugepages - name: hugepages - - mountPath: /dev/shm - name: shared-memory - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch terminationGracePeriodSeconds: 60 - tolerations: - - key: lpu.nvidia.com/node - operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - emptyDir: - medium: HugePages - name: hugepages + - emptyDir: {} + name: single-v2-ssh-key - emptyDir: medium: Memory sizeLimit: 8Gi name: shared-memory + - name: ssh-secret + secret: + defaultMode: 384 + secretName: ssh-secret - configMap: - name: lmppuelh-lpu-29156c984c942fc4 + name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 name: config - replicas: 4 - roleName: lpu-wkr-m-2 + replicas: 1 + roleName: cond + startsAfter: + - agt0 + - agt1 + - agt2 headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -770,12 +770,12 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 cliqueNames: - - lpu-ldr - - lpu-wkr-m-0 - - lpu-wkr-m-1 - - lpu-wkr-m-2 + - cond + - agt0 + - agt1 + - agt2 minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index bfadf80f9656..3fc16b8bef5e 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -7,9 +7,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 gas_dir: /nfs/node-local-v3-hx-connected-lpx model_config.toml: | @@ -46,7 +46,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: lxymegpm-lpu-40e022e2d45905fb + name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 namespace: test --- metadata: @@ -57,7 +57,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid - name: lxymegpm + name: node-local-v3-hx-hybrid-be18 namespace: test spec: replicas: 1 @@ -67,148 +67,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 - scheduling.lpu.nvidia.com/role: cyborg-worker - labels: - lpx.nvidia.com/execution-role: gpu - lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid - nvidia.com/dynamo-namespace: test-node-local-v3-hx-hybrid - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v3-hx-hybrid-lpu - name: lpu-engine-gpu - spec: - minAvailable: 1 - podSpec: - containers: - - args: - - serve - command: - - /opt/gpu-runtime - env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - - name: CYBORG_FPGA_GPI_REPLICA_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: DYN_COMPONENT - value: decode - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_FORWARDPASS_METRIC_PORT - value: "20380" - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: DYN_NAMESPACE - value: test-node-local-v3-hx-hybrid - - name: DYN_PARENT_DGD_K8S_NAME - value: node-local-v3-hx-hybrid - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: test - - name: DYN_SYSTEM_ENABLED - value: "true" - - name: DYN_SYSTEM_PORT - value: "9090" - - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS - value: '["generate"]' - - name: ETCD_ENDPOINTS - value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/node-local-v3-hx-connected-lpx/manifest.v2.capnp.bin - - name: NATS_SERVER - value: nats-address - - name: NIXL_TELEMETRY_ENABLE - value: "n" - - name: NIXL_TELEMETRY_EXPORTER - value: prometheus - - name: NIXL_TELEMETRY_PROMETHEUS_PORT - value: "19090" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: example.invalid/gpu-runtime:test - livenessProbe: - failureThreshold: 1 - httpGet: - path: /live - port: system - periodSeconds: 5 - timeoutSeconds: 4 - name: main - ports: - - containerPort: 9090 - name: system - protocol: TCP - - containerPort: 19090 - name: nixl - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /health - port: system - periodSeconds: 10 - timeoutSeconds: 4 - resources: - claims: - - name: gpu - startupProbe: - failureThreshold: 720 - httpGet: - path: /live - port: system - periodSeconds: 10 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /nfs - name: model-storage - - mountPath: /dev/shm - name: shared-memory - resourceClaims: - - name: gpu - resourceClaimTemplateName: decode-gpu - restartPolicy: Always - schedulerName: default-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - replicas: 1 - roleName: lpu-engine-gpu - startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 40e022e2d45905fb8bce9ead674fe7e847e78a5785b481ef8dbff443218d20e5 + nvidia.com/extra-resources-hash: 7443e572ed6c9124e2bfdacd89de30a0515735f36cfb677a99e1be70668ca8d0 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 scheduling.lpu.nvidia.com/model: default @@ -223,7 +82,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v3-hx-hybrid nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v3-hx-hybrid-lpu - name: lpu-wkr-m-0 + name: agt spec: minAvailable: 1 podSpec: @@ -249,7 +108,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: lxymegpm-lpu-40e022e2d45905fb + name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -283,7 +142,7 @@ spec: valueFrom: configMapKeyRef: key: topologies - name: lxymegpm-lpu-40e022e2d45905fb + name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -376,10 +235,151 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: lxymegpm-lpu-40e022e2d45905fb + name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 name: config replicas: 1 - roleName: lpu-wkr-m-0 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/execution-role: gpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid + nvidia.com/dynamo-namespace: test-node-local-v3-hx-hybrid + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-hybrid-lpu + name: cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + command: + - /opt/gpu-runtime + env: + - name: CYBORG_BATCH_SIZE + value: "1" + - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT + value: "1" + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: CYBORG_SWA_CACHE_IDS + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-hybrid + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v3-hx-connected-lpx/manifest.v2.capnp.bin + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/gpu-runtime:test + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + claims: + - name: gpu + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /dev/shm + name: shared-memory + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 1 + roleName: cond + startsAfter: + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -389,10 +389,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 cliqueNames: - - lpu-wkr-m-0 - - lpu-engine-gpu + - agt + - cond minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 6cb2ea731e3b..a2cf685e1376 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -7,9 +7,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 model_config.toml: | type = 'Single' @@ -45,7 +45,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - name: l2q6k2r6-lpu-68b42744ed35823d + name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f namespace: test --- metadata: @@ -56,7 +56,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only - name: l2q6k2r6 + name: node-local-v3-hx-lpu-only-d43c namespace: test spec: replicas: 1 @@ -66,7 +66,184 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 68b42744ed35823df3d0ff18f87c164e682cf5681ce0d76ab40c410b19c997d2 + nvidia.com/extra-resources-hash: b388a698f7dcc08f8fd26e78b06feab8d6992ea55a77555aacb9c70d9262f77a + scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only + nvidia.com/dynamo-namespace: test-node-local-v3-hx-lpu-only + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-lpu-only-lpu + name: agt + spec: + minAvailable: 1 + podSpec: + containers: + - command: + - /bin/quasar-entrypoint + env: + - name: BMC_SSH_ENABLED + value: "false" + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-lpu-only + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: "false" + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: "false" + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: "0" + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: "0" + - name: GROQ_V2_RESET_ON_OPEN + value: "0" + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: "false" + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: REDFISH_BMC_ENABLED + value: "false" + image: example.invalid/lpu-runtime:test + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + nvidia.com/lpu: "16" + requests: + nvidia.com/lpu: "16" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /configs + name: config + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + resources: {} + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f + name: config + replicas: 1 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: b388a698f7dcc08f8fd26e78b06feab8d6992ea55a77555aacb9c70d9262f77a scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/role: conductor @@ -80,7 +257,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v3-hx-lpu-only nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v3-hx-lpu-only-lpu - name: lpu-ldr + name: cond spec: minAvailable: 1 podSpec: @@ -111,7 +288,7 @@ spec: -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - --allocation - - lpu-wkr-m-0 + - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" @@ -278,189 +455,12 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: l2q6k2r6-lpu-68b42744ed35823d + name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f name: config replicas: 1 - roleName: lpu-ldr + roleName: cond startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 68b42744ed35823df3d0ff18f87c164e682cf5681ce0d76ab40c410b19c997d2 - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 - scheduling.lpu.nvidia.com/model: default - scheduling.lpu.nvidia.com/role: agent - scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly - labels: - lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only - nvidia.com/dynamo-namespace: test-node-local-v3-hx-lpu-only - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v3-hx-lpu-only-lpu - name: lpu-wkr-m-0 - spec: - minAvailable: 1 - podSpec: - containers: - - command: - - /bin/quasar-entrypoint - env: - - name: BMC_SSH_ENABLED - value: "false" - - name: DYN_COMPONENT - value: lpx - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_NAMESPACE - value: test-node-local-v3-hx-lpu-only - - name: DYN_PARENT_DGD_K8S_NAME - value: node-local-v3-hx-lpu-only - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: test - - name: ETCD_ENDPOINTS - value: etcd-address - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: "false" - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: "false" - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: "0" - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: "0" - - name: GROQ_V2_RESET_ON_OPEN - value: "0" - - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY - value: "false" - - name: LPU_FPGA_RECONFIGURE_METHOD - value: cvp - - name: NATS_SERVER - value: nats-address - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: REDFISH_BMC_ENABLED - value: "false" - image: example.invalid/lpu-runtime:test - name: agent - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: 2222 - timeoutSeconds: 1 - resources: - limits: - nvidia.com/lpu: "16" - requests: - nvidia.com/lpu: "16" - securityContext: - privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /nfs - name: model-storage - - mountPath: /configs - name: config - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys - - mountPath: /ssh-pk - name: ssh-secret - readOnly: true - - mountPath: /dev/shm - name: shared-memory - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - initContainers: - - command: - - /bin/lp30-agent-v2-init - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: example.invalid/lpu-runtime:test - name: prepare-lp30 - resources: {} - securityContext: - privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /sys - name: host-sys - - mountPath: /dev - name: host-dev - restartPolicy: Always - schedulerName: lpx-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - tolerations: - - key: lpu.nvidia.com/node - operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: l2q6k2r6-lpu-68b42744ed35823d - name: config - replicas: 1 - roleName: lpu-wkr-m-0 + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -470,10 +470,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 cliqueNames: - - lpu-ldr - - lpu-wkr-m-0 + - cond + - agt minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index b44d7b0ace29..f60b2710c1df 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -7,19 +7,19 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt0] available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 - [datacenters.racks.lpu-wkr-m-1] + [datacenters.racks.agt1] available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-1-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 - [datacenters.racks.lpu-wkr-m-2] + [datacenters.racks.agt2] available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-2-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt2-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 model_config.toml: | draft_to_target_port = 23456 @@ -101,7 +101,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - name: ld55picp-lpu-3c4a4d4af295f6cc + name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b namespace: test --- metadata: @@ -112,7 +112,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 labels: nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode - name: ld55picp + name: node-local-v3-hx-specdecode-1f7a namespace: test spec: replicas: 1 @@ -122,230 +122,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 - scheduling.lpu.nvidia.com/role: conductor - labels: - lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode - nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: node-local-v3-hx-specdecode-lpu - name: lpu-ldr - spec: - minAvailable: 1 - podSpec: - containers: - - args: - - --instance-model-name - - test/v3-specdecode - - --agent-path - - /bin/agent - - --agent-wd - - /tmp - - --conductor-bind-host - - 0.0.0.0 - - --conductor-hostname - - $(POD_IP) - - --control-server-addr - - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - - --model-config-filepath - - /configs/model_config.toml - - --json-logs - - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - - --mpi-use-tcp - - --allocation - - lpu-wkr-m-0:lpu-wkr-m-1:lpu-wkr-m-2 - - --agent-disable-auto-restart - - --agent-server-max-retries - - "60" - - --agent-server-retry-sleep-dur - - 2s - - --controlplane-start-port - - "12345" - - --graceful-shutdown-timeout - - 0s - - --response-timeout - - 120s - - --frontend-type - - dynamo - - --endpoint-id - - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - - --store-backend - - kubernetes - - --agent-env-vars - - BMC_SSH_ENABLED=false - - --agent-env-vars - - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 - - --agent-env-vars - - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false - - --agent-env-vars - - GROQ_HAL_V2_BACKGROUND_METRICS=false - - --agent-env-vars - - GROQ_PG613_PORT_CONFIG=EB3 - - --agent-env-vars - - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false - - --agent-env-vars - - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 - - --agent-env-vars - - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 - - --agent-env-vars - - GROQ_V2_RESET_ON_OPEN=0 - - --agent-env-vars - - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false - - --agent-env-vars - - LPU_FPGA_RECONFIGURE_METHOD=cvp - - --agent-env-vars - - LPU_PARTITION_RUNNER_DATAPLANE=pcie - - --agent-env-vars - - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 - - --agent-env-vars - - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 - - --agent-env-vars - - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true - - --agent-env-vars - - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 - - --agent-env-vars - - REDFISH_BMC_ENABLED=false - command: - - /bin/nova - env: - - name: DYN_COMPONENT - value: lpx - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_NAMESPACE - value: test-node-local-v3-hx-specdecode - - name: DYN_PARENT_DGD_K8S_NAME - value: node-local-v3-hx-specdecode - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: test - - name: ETCD_ENDPOINTS - value: etcd-address - - name: NATS_SERVER - value: nats-address - - name: OMPI_ALLOW_RUN_AS_ROOT - value: "1" - - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM - value: "1" - - name: OMPI_MCA_plm_rsh_args - value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: example.invalid/lpu-runtime:test - name: conductor - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: 51045 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 1 - resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /nfs - name: model-storage - - mountPath: /configs - name: config - - mountPath: /tmp/dynamo-lpu-ssh - name: single-v2-ssh-key - - mountPath: /dev/shm - name: shared-memory - initContainers: - - args: - - -m - - "0600" - - /ssh-pk/private.key - - /tmp/dynamo-lpu-ssh/private.key - command: - - /bin/install - image: example.invalid/lpu-runtime:test - name: prepare-ssh-key - resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /tmp/dynamo-lpu-ssh - name: single-v2-ssh-key - - mountPath: /ssh-pk - name: ssh-secret - readOnly: true - restartPolicy: Always - schedulerName: default-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - emptyDir: {} - name: single-v2-ssh-key - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - name: ssh-secret - secret: - defaultMode: 384 - secretName: ssh-secret - - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc - name: config - replicas: 1 - roleName: lpu-ldr - startsAfter: - - lpu-wkr-m-0 - - lpu-wkr-m-1 - - lpu-wkr-m-2 - - annotations: - lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 + nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea scheduling.lpu.nvidia.com/model: draft0 @@ -360,7 +137,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v3-hx-specdecode-draft - name: lpu-wkr-m-0 + name: agt0 spec: minAvailable: 1 podSpec: @@ -517,14 +294,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc + name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b name: config replicas: 1 - roleName: lpu-wkr-m-0 + roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 + nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed scheduling.lpu.nvidia.com/model: draft1 @@ -539,7 +316,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v3-hx-specdecode-draft - name: lpu-wkr-m-1 + name: agt1 spec: minAvailable: 1 podSpec: @@ -696,14 +473,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc + name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b name: config replicas: 1 - roleName: lpu-wkr-m-1 + roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 3c4a4d4af295f6cc8c3a32de03d15e2dc07d279fa7d93d95a908841d032ff842 + nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a scheduling.lpu.nvidia.com/model: target @@ -718,7 +495,7 @@ spec: nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode nvidia.com/metrics-enabled: "true" nvidia.com/selector: node-local-v3-hx-specdecode-lpu - name: lpu-wkr-m-2 + name: agt2 spec: minAvailable: 1 podSpec: @@ -875,10 +652,233 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: ld55picp-lpu-3c4a4d4af295f6cc + name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b name: config replicas: 1 - roleName: lpu-wkr-m-2 + roleName: agt2 + - annotations: + lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode + nvidia.com/dynamo-namespace: test-node-local-v3-hx-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: node-local-v3-hx-specdecode-lpu + name: cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - --instance-model-name + - test/v3-specdecode + - --agent-path + - /bin/agent + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key + - --mpi-use-tcp + - --allocation + - agt0:agt1:agt2 + - --agent-disable-auto-restart + - --agent-server-max-retries + - "60" + - --agent-server-retry-sleep-dur + - 2s + - --controlplane-start-port + - "12345" + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --request-plane + - tcp + - --store-backend + - kubernetes + - --agent-env-vars + - BMC_SSH_ENABLED=false + - --agent-env-vars + - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 + - --agent-env-vars + - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false + - --agent-env-vars + - GROQ_HAL_V2_BACKGROUND_METRICS=false + - --agent-env-vars + - GROQ_PG613_PORT_CONFIG=EB3 + - --agent-env-vars + - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false + - --agent-env-vars + - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 + - --agent-env-vars + - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 + - --agent-env-vars + - GROQ_V2_RESET_ON_OPEN=0 + - --agent-env-vars + - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false + - --agent-env-vars + - LPU_FPGA_RECONFIGURE_METHOD=cvp + - --agent-env-vars + - LPU_PARTITION_RUNNER_DATAPLANE=pcie + - --agent-env-vars + - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 + - --agent-env-vars + - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 + - --agent-env-vars + - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true + - --agent-env-vars + - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 + - --agent-env-vars + - REDFISH_BMC_ENABLED=false + command: + - /bin/nova + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: node-local-v3-hx-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: OMPI_ALLOW_RUN_AS_ROOT + value: "1" + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: "1" + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-runtime:test + name: conductor + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /configs + name: config + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /dev/shm + name: shared-memory + initContainers: + - args: + - -m + - "0600" + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + resources: {} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: {} + name: single-v2-ssh-key + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - name: ssh-secret + secret: + defaultMode: 384 + secretName: ssh-secret + - configMap: + name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: config + replicas: 1 + roleName: cond + startsAfter: + - agt0 + - agt1 + - agt2 headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -888,12 +888,12 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 cliqueNames: - - lpu-ldr - - lpu-wkr-m-0 - - lpu-wkr-m-1 - - lpu-wkr-m-2 + - cond + - agt0 + - agt1 + - agt2 minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 1f7b7c4b0480..641e793e44c1 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -7,9 +7,9 @@ data: [datacenters] [datacenters.racks] - [datacenters.racks.lpu-wkr-m-0] + [datacenters.racks.agt] available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-lpu-wkr-m-0-{node}.${GROVE_HEADLESS_SERVICE}' + node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' preferred_start = 0 model_config.toml: | type = 'Single' @@ -129,7 +129,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - name: lhkmp4xg-lpu-1575485f621480a4 + name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa namespace: inference-engine --- metadata: @@ -140,7 +140,7 @@ metadata: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 labels: nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 - name: lhkmp4xg + name: dgd-single-v2-3a98 namespace: inference-engine spec: replicas: 1 @@ -150,7 +150,155 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 1575485f621480a46d343ddb9e07a6e00472a124e8ba24f535befac176ae9936 + nvidia.com/extra-resources-hash: cc95c2008c398eaa60b329a696fa0f2faeff6c094df93b3d88d73cde3e6966a4 + scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/execution-role: lpu + lpx.nvidia.com/stage: lpu + nvidia.com/dynamo-component: lpu + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 + nvidia.com/dynamo-namespace: inference-engine-dgd-single-v2 + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: dgd-single-v2-lpu + name: agt + spec: + minAvailable: 17 + podSpec: + containers: + - args: + - --partition-metadata + command: + - /bin/quasar-entrypoint + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: inference-engine-dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAME + value: dgd-single-v2 + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: inference-engine + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPU_RUN_SYSTEM_INIT + value: "true" + - name: MPI_BTL_TCP_IF_INCLUDE + value: 7.241.144.0/24 + - name: NATS_SERVER + value: nats-address + - name: OMPI_MCA_btl_tcp_if_include + value: 7.241.144.0/24 + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + name: agent + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + resources: + limits: + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + requests: + cpu: "62" + hugepages-2Mi: 4Gi + lpu.nvidia.com/lpu: "8" + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /nfs + name: model-storage + - mountPath: /configs + name: config + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true + - mountPath: /dev/hugepages + name: hugepages + - mountPath: /dev/shm + name: shared-memory + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + restartPolicy: Always + schedulerName: lpx-scheduler + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + terminationGracePeriodSeconds: 60 + tolerations: + - key: lpu.nvidia.com/node + operator: Exists + - key: lpu.nvidia.com/lpu + operator: Exists + - key: lpu.nvidia.com/node-v2 + operator: Exists + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - hostPath: + path: /dev + type: Directory + name: host-dev + - hostPath: + path: /sys + type: Directory + name: host-sys + - name: ssh-secret + secret: + defaultMode: 420 + secretName: ssh-secret + - emptyDir: + medium: HugePages + name: hugepages + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa + name: config + replicas: 17 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: dgd-single-v2 + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: cc95c2008c398eaa60b329a696fa0f2faeff6c094df93b3d88d73cde3e6966a4 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/role: conductor @@ -164,7 +312,7 @@ spec: nvidia.com/dynamo-namespace: inference-engine-dgd-single-v2 nvidia.com/metrics-enabled: "true" nvidia.com/selector: dgd-single-v2-lpu - name: lpu-ldr + name: cond spec: minAvailable: 1 podSpec: @@ -195,7 +343,7 @@ spec: -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - --allocation - - lpu-wkr-m-0 + - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" @@ -328,160 +476,12 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: lhkmp4xg-lpu-1575485f621480a4 + name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa name: config replicas: 1 - roleName: lpu-ldr + roleName: cond startsAfter: - - lpu-wkr-m-0 - - annotations: - lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 1575485f621480a46d343ddb9e07a6e00472a124e8ba24f535befac176ae9936 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 - scheduling.lpu.nvidia.com/model: default - scheduling.lpu.nvidia.com/role: agent - scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly - labels: - lpx.nvidia.com/execution-role: lpu - lpx.nvidia.com/stage: lpu - nvidia.com/dynamo-component: lpu - nvidia.com/dynamo-component-type: lpx - nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 - nvidia.com/dynamo-namespace: inference-engine-dgd-single-v2 - nvidia.com/metrics-enabled: "true" - nvidia.com/selector: dgd-single-v2-lpu - name: lpu-wkr-m-0 - spec: - minAvailable: 17 - podSpec: - containers: - - args: - - --partition-metadata - command: - - /bin/quasar-entrypoint - env: - - name: DYN_COMPONENT - value: lpx - - name: DYN_DISCOVERY_BACKEND - value: kubernetes - - name: DYN_NAMESPACE - value: inference-engine-dgd-single-v2 - - name: DYN_PARENT_DGD_K8S_NAME - value: dgd-single-v2 - - name: DYN_PARENT_DGD_K8S_NAMESPACE - value: inference-engine - - name: ETCD_ENDPOINTS - value: etcd-address - - name: LPU_RUN_SYSTEM_INIT - value: "true" - - name: MPI_BTL_TCP_IF_INCLUDE - value: 7.241.144.0/24 - - name: NATS_SERVER - value: nats-address - - name: OMPI_MCA_btl_tcp_if_include - value: 7.241.144.0/24 - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 - name: agent - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: 2222 - timeoutSeconds: 1 - resources: - limits: - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" - requests: - cpu: "62" - hugepages-2Mi: 4Gi - lpu.nvidia.com/lpu: "8" - securityContext: - privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - mountPath: /nfs - name: model-storage - - mountPath: /configs - name: config - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys - - mountPath: /ssh-pk - name: ssh-secret - readOnly: true - - mountPath: /dev/hugepages - name: hugepages - - mountPath: /dev/shm - name: shared-memory - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - restartPolicy: Always - schedulerName: lpx-scheduler - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - terminationGracePeriodSeconds: 60 - tolerations: - - key: lpu.nvidia.com/node - operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - hostPath: - path: /dev - type: Directory - name: host-dev - - hostPath: - path: /sys - type: Directory - name: host-sys - - name: ssh-secret - secret: - defaultMode: 420 - secretName: ssh-secret - - emptyDir: - medium: HugePages - name: hugepages - - emptyDir: - medium: Memory - sizeLimit: 8Gi - name: shared-memory - - configMap: - name: lhkmp4xg-lpu-1575485f621480a4 - name: config - replicas: 17 - roleName: lpu-wkr-m-0 + - agt headlessServiceConfig: publishNotReadyAddresses: true podCliqueScalingGroups: @@ -491,10 +491,10 @@ spec: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 cliqueNames: - - lpu-ldr - - lpu-wkr-m-0 + - cond + - agt minAvailable: 1 - name: lpu + name: lpx replicas: 1 terminationDelay: 15m0s status: From adfebdd278ed691dcbe0902ab7715a93d8edf37c Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:13:58 +0300 Subject: [PATCH 080/196] fix(operator): preserve LPX replica seeds during native scaling (#518) Signed-off-by: Andrew Paprotsky (cherry picked from commit ce3fafadf729560cd13b61ca14aaa7b6ef5b8622) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 2 +- .../controller/lpx/controller_test.go | 28 +++++++++++++++++-- .../operator/internal/controller/lpx/grove.go | 16 +++++++++-- .../internal/controller/lpx/lifecycle_test.go | 27 ++++++++++-------- 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 45be592c8aab..248c0c03fd9f 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -278,7 +278,7 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a return state, ctrl.Result{}, err } } - synced, changed, retiring, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, pcs, desired) + synced, changed, retiring, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, lpx.ServingComponent(source).Replicas, pcs, desired) if err != nil { return state, ctrl.Result{}, err } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index c5ce3a84d59c..a764d9a2c6e3 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -41,13 +41,15 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { for _, tc := range []struct { name string replicas *int32 + seed int32 err error wantUpdates int }{ {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, {name: "unchanged", replicas: ptr.To(int32(9))}, - {name: "omitted"}, + {name: "omitted scale out", seed: 3}, + {name: "omitted scale in", seed: 12}, {name: "conflict", replicas: ptr.To(int32(12)), err: conflict, wantUpdates: 1}, } { t.Run(tc.name, func(t *testing.T) { @@ -56,10 +58,26 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { lpx.ServingComponent(source).Replicas = tc.replicas r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) objects := lpxMaterializedObjects(t, r, child, source, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + if tc.replicas == nil { + pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas = ptr.To(tc.seed) + } + hash, err := commoncontroller.GetSpecHash(pcs, commoncontroller.WithPreservedListOrder()) + require.NoError(t, err) + metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) + metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, commoncontroller.NvidiaAnnotationGenerationKey, "1") group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) group.Spec.Replicas = 9 createLPXTestObjects(t, t.Context(), r.Client, objects...) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + beforePCS := pcs.DeepCopy() + + t.Log("Rebuild scheduler intent from explicit replicas or the current native scale") + selected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) + require.Nil(t, rejected) + require.Equal(t, ptr.Deref(tc.replicas, int32(9)), selected.plan.Replicas) + require.Len(t, selected.requests, int(selected.plan.Replicas)) updates := 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { @@ -78,7 +96,7 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { }) t.Log("Only explicit changes write scale, preserving the observed resource-version precondition") - _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) + _, _, err = r.reconcileWorkload(t.Context(), child, source, selected) require.ErrorIs(t, err, tc.err) require.Equal(t, tc.wantUpdates, updates) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) @@ -87,6 +105,10 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { want = *tc.replicas } require.Equal(t, want, group.Spec.Replicas) + + t.Log("Native scale never rewrites the parent PCS seed or its bookkeeping") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.Equal(t, beforePCS, pcs) }) } } @@ -973,7 +995,7 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, retiring, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, nil, pcs) + _, _, retiring, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, lpx.ServingComponent(source).Replicas, nil, pcs) require.ErrorContains(t, err, "input revision") require.Nil(t, retiring) allPCS := &grovev1alpha1.PodCliqueSetList{} diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go index 0b7a49067bce..0befaf38ab5a 100644 --- a/deploy/operator/internal/controller/lpx/grove.go +++ b/deploy/operator/internal/controller/lpx/grove.go @@ -26,11 +26,12 @@ import ( ) // reconcileGrovePodCliqueSetForLPX converges the already-rendered PodCliqueSet -// through native Grove rollout. Only existingPodCliqueSet may be nil, when -// the controller observed no PCS; all decisions and writes use that observation. +// through native Grove rollout. existingPodCliqueSet may be nil when absent; +// nil replicas leaves scale to Grove. deployment and desired are non-nil. func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + replicas *int32, existingPodCliqueSet *grovev1alpha1.PodCliqueSet, desired *grovev1alpha1.PodCliqueSet, ) (*grovev1alpha1.PodCliqueSet, bool, *lpxRetiring, error) { @@ -45,6 +46,17 @@ func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( } } + // Grove seeds native scale only at creation; preserve that seed when replicas are omitted. + if existingPodCliqueSet != nil && replicas == nil { + group := &desired.Spec.Template.PodCliqueScalingGroupConfigs[0] + for _, existing := range existingPodCliqueSet.Spec.Template.PodCliqueScalingGroupConfigs { + if existing.Name == group.Name { + group.Replicas = existing.Replicas + break + } + } + } + // Match normal PCS synchronization; Grove owns rollout and immutable-field validation. modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired, commoncontroller.WithPreservedListOrder()) return synced, modified, nil, err diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 6e545c7a6a54..55fb92819f13 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -693,7 +693,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { }) t.Log("Only bookkeeping changes; root metadata additions, edits and removals are ignored") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, cached, desired) require.NoError(t, err) require.Nil(t, retiring) require.True(t, modified) @@ -706,7 +706,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { t.Log("Root metadata differences alone are a no-op once bookkeeping is current") updates = 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) require.NoError(t, err) require.Nil(t, retiring) require.False(t, modified) @@ -715,7 +715,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { t.Log("Template metadata still updates through Spec without changing root metadata or native controls") desired.Spec.Template.Cliques[0].Annotations = map[string]string{"test.example/template": "next"} - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) require.NoError(t, err) require.Nil(t, retiring) require.True(t, modified) @@ -735,7 +735,7 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.NoError(t, base.Update(ctx, concurrent)) desired.Spec.Replicas += 2 updates = 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, cached, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, cached, desired) require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) require.Nil(t, retiring) require.Nil(t, synced) @@ -757,7 +757,7 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers = []corev1.Container{ {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, } - observed, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + observed, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) require.NoError(t, err) require.Nil(t, retiring) require.True(t, modified) @@ -773,7 +773,7 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("Apply authored ordering changes but ignore unchanged desired specs") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, observed, desired) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, observed, desired) require.NoError(t, err) require.Nil(t, retiring) require.Equal(t, reorder, modified) @@ -784,7 +784,7 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("The next reconciliation is a no-op") - _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, synced, desired) + _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) require.NoError(t, err) require.Nil(t, retiring) require.False(t, modified) @@ -826,7 +826,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { }) t.Log("A failed publication observation prevents initial creation") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) require.ErrorIs(t, err, fenceErr) require.Nil(t, retiring) require.False(t, modified) @@ -842,7 +842,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { request.Finalizers = []string{"test.example/scheduler"} require.NoError(t, base.Create(ctx, request)) writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) require.NoError(t, err) require.NotNil(t, retiring) require.False(t, modified) @@ -856,7 +856,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { request.Finalizers = nil require.NoError(t, base.Update(ctx, request)) writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) require.NoError(t, err) require.Nil(t, retiring) require.True(t, modified) @@ -875,7 +875,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) t.Log("A stale missing observation does not adopt an already-created PCS") - _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, nil, desired.DeepCopy()) + _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) require.Nil(t, retiring) require.False(t, modified) @@ -888,7 +888,7 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" failFence = true writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, live, desired) + synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, live, desired) require.NoError(t, err) require.Nil(t, retiring) require.True(t, modified) @@ -924,6 +924,7 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { _, _, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( ctx, dgd, + lpx.ServingComponent(source).Replicas, drifted, rendered, ) @@ -975,6 +976,7 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { _, _, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX( ctx, dgd, + lpx.ServingComponent(source).Replicas, drifted, rendered, ) @@ -1046,6 +1048,7 @@ func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { synced, changed, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( ctx, successorDGD, + lpx.ServingComponent(successorSource).Replicas, pcs, successorRendered, ) From 8235aa1cc710126f90d438b136fbef99db234529 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:45:03 +0300 Subject: [PATCH 081/196] refactor(operator)!: let LPX pod templates own runtime startup (#519) * fix(operator): stop overriding Nova defaults Signed-off-by: Andrew Paprotsky * refactor(operator)!: let LPX pod templates own runtime startup Preserve authored commands, worker arguments, init containers and startup hooks instead of generating executable paths and wrapping Cyborg startup. Keep model, allocation, scheduling and runtime bindings in LPX. Update XT/HX render fixtures and verify native image entrypoints and authored startup. BREAKING CHANGE: LPX no longer supplies runtime executables, MPI executable options, SSH-key or LP30 init containers, or Hydra startup/readiness probes. DGD role templates must supply the startup contract required by their images. Signed-off-by: Andrew Paprotsky --------- Signed-off-by: Andrew Paprotsky (cherry picked from commit 1b42714b9d18fe601488a4f63fa0e825d5b0336d) Signed-off-by: Andrew Paprotsky --- .../dynamocomponentdeployment_types.go | 4 +- .../dynamocomponentdeployment_types.go | 4 +- ...nvidia.com_dynamocomponentdeployments.yaml | 8 +- .../nvidia.com_dynamographdeployments.yaml | 8 +- .../dgd_lpx_handoff_envtest_test.go | 12 +- .../controller/lpx/component_roles_test.go | 17 +- .../controller/lpx/controller_test.go | 17 +- .../internal/controller/lpx/dependencies.go | 2 +- .../internal/controller/lpx/lifecycle_test.go | 26 +- .../controller/lpx/modeldownload_test.go | 7 +- .../internal/controller/lpx/render_test.go | 302 +++--- deploy/operator/internal/dynamo/graph_lpx.go | 55 +- .../internal/dynamo/graph_lpx_test.go | 6 +- .../internal/dynamo/lpx/conductor_runtime.go | 133 --- .../lpx/cyborg_launcher_fixture_test.go | 90 -- .../internal/dynamo/lpx/cyborg_render.go | 15 - .../internal/dynamo/lpx/cyborg_render_test.go | 21 +- .../internal/dynamo/lpx/cyborg_runtime.go | 38 +- .../dynamo/lpx/cyborg_runtime_test.go | 57 -- .../dynamo/lpx/hybrid_cyborg_bindings.go | 9 +- .../internal/dynamo/lpx/lpu_commands.go | 71 -- .../operator/internal/dynamo/lpx/pod_roles.go | 1 + .../internal/dynamo/lpx/pod_roles_test.go | 858 ++---------------- deploy/operator/internal/dynamo/lpx/render.go | 62 +- .../internal/dynamo/lpx/render_test.go | 77 +- .../dynamo/lpx/resolve_selected_workload.go | 36 +- .../lpx/resolve_selected_workload_test.go | 67 +- .../internal/dynamo/lpx/runtime_agent.go | 365 +------- .../internal/dynamo/lpx/runtime_ssh.go | 82 -- .../internal/dynamo/lpx/runtime_ssh_test.go | 117 --- .../internal/dynamo/lpx/runtime_startup.go | 51 -- .../dynamo/lpx/runtime_startup_test.go | 183 ++-- .../internal/dynamo/lpx/selection_test.go | 22 +- .../dynamo/lpx/selection_validation.go | 96 +- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 115 ++- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 62 +- .../node-local-v2-hybrid.input.yaml | 58 ++ .../from_dgd_yaml/node-local-v2-hybrid.yaml | 6 +- .../node-local-v2-lpu-only.input.yaml | 88 ++ .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 22 +- .../node-local-v2-specdecode.input.yaml | 123 ++- .../node-local-v2-specdecode.yaml | 22 +- .../node-local-v3-hx-hybrid.input.yaml | 55 +- .../node-local-v3-hx-lpu-only.input.yaml | 181 +++- .../node-local-v3-hx-lpu-only.yaml | 22 +- .../node-local-v3-hx-specdecode.input.yaml | 210 ++++- .../node-local-v3-hx-specdecode.yaml | 22 +- .../from_dgd_yaml/single_v2.input.yaml | 105 ++- .../testdata/from_dgd_yaml/single_v2.yaml | 36 +- .../single_v2_manifest_defaults.input.yaml | 55 -- .../validation/dynamographdeployment.go | 11 + ...hdeployment_lpx_validation_envtest_test.go | 150 ++- .../model-deployment/deploy-with-dgd.md | 64 +- .../additional-resources/api-reference-k8s.md | 8 +- .../dynamo-component-deployment.mdx | 10 +- .../kubernetes-api/full-api-reference.mdx | 8 +- 56 files changed, 1679 insertions(+), 2673 deletions(-) delete mode 100644 deploy/operator/internal/dynamo/lpx/conductor_runtime.go delete mode 100644 deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go delete mode 100644 deploy/operator/internal/dynamo/lpx/lpu_commands.go delete mode 100644 deploy/operator/internal/dynamo/lpx/runtime_ssh.go delete mode 100644 deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go delete mode 100644 deploy/operator/internal/dynamo/lpx/runtime_startup.go delete mode 100644 deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 8e4448d59451..80a484d2eb30 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -174,7 +174,9 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components require an agent role and may declare a conductor role. + // LPX components each require an agent role. All LPX components in a DGD + // must declare exactly one conductor role in total. Every LPX role requires + // its own podTemplate. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 6296d9c59840..1add1a5b10ed 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -178,7 +178,9 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components require an agent role and may declare a conductor role. + // LPX components each require an agent role. All LPX components in a DGD + // must declare exactly one conductor role in total. Every LPX role requires + // its own podTemplate. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 02c73dadbe95..e236aac33573 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -11465,7 +11465,9 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require an agent role and may declare a conductor role. + LPX components each require an agent role. All LPX components in a DGD + must declare exactly one conductor role in total. Every LPX role requires + its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -28653,7 +28655,9 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require an agent role and may declare a conductor role. + LPX components each require an agent role. All LPX components in a DGD + must declare exactly one conductor role in total. Every LPX role requires + its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 3c3c43245827..b738bf03cda8 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11857,7 +11857,9 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require an agent role and may declare a conductor role. + LPX components each require an agent role. All LPX components in a DGD + must declare exactly one conductor role in total. Every LPX role requires + its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -29422,7 +29424,9 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components require an agent role and may declare a conductor role. + LPX components each require an agent role. All LPX components in a DGD + must declare exactly one conductor role in total. Every LPX role requires + its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 8ccbcb125b5a..4b5474d41eee 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -165,7 +165,17 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "lpx", ComponentType: v1beta1.ComponentTypeLPX, LPX: &v1beta1.LPXConfig{BuildID: (&url.URL{Scheme: "file", Path: buildDir}).String()}, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXConductor}, { + Roles: []v1beta1.ComponentRoleSpec{{ + Name: v1beta1.ComponentRoleLPXConductor, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: consts.MainContainerName, Image: "example/conductor:1.4.0", + Command: []string{"/custom-conductor"}, Args: []string{"--workers", "$(LPX_ALLOCATION)"}, + VolumeMounts: []corev1.VolumeMount{{Name: consts.ModelStorageVolumeName, MountPath: "/models"}}, + }}, + Volumes: []corev1.Volume{{Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + }}, + }, { Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{ diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go index 5c0aaf578277..99319e2c38ac 100644 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -18,25 +18,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestLPXRoleDependenciesIncludeDraftAndConductorFallback(t *testing.T) { - t.Log("Give both components consumed Agent claims while the target conductor uses its Agent fallback") +func TestLPXRoleDependenciesIncludeDraftAndConductor(t *testing.T) { + t.Log("Give both components consumed Agent claims and the conductor its own dependency") source := newLPXSpecDecodeTestSource() - lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil for _, component := range lpx.Components(source) { pod := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(component.ComponentName + "-gpu")}} pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} } - require.Nil(t, lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate) + conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor) + conductor.PodTemplate.Spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("conductor-gpu")}} + conductor.PodTemplate.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} before := source.DeepCopy() - require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu"}, lpxDRAClaimReferences(true)(source)) - require.Equal(t, before, source) - t.Log("An independent conductor template contributes its own consumed dependency") - conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor) - conductor.PodTemplate = lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.DeepCopy() - conductor.PodTemplate.Spec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("conductor-gpu") + t.Log("Index each authored role's consumed claim without modifying the source") require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu", "conductor-gpu"}, lpxDRAClaimReferences(true)(source)) + require.Equal(t, before, source) } func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index a764d9a2c6e3..0271affd27dd 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -680,7 +680,7 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { t.Log("Publish a speculative engine with long deployment and component names") _, source, registry := newLPXSpecDecodeTestDGD(t) - source.Name = "apaprotskyi-gpt-oss-20b-lp20-b300" + source.Name = "test-models-gpt-oss-20b-lp20-b300" source.Spec.Components[0].ComponentName = "serving-component-with-name" source.Spec.Components[1].ComponentName = "draft-component-with-name" source.Spec.Components[1].Replicas = ptr.To(int32(1)) @@ -872,6 +872,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { for _, scenario := range []struct { name, componentName string sharedDraft bool + missingConductor string messages []string }{ {name: "invalid intent", messages: []string{ @@ -882,6 +883,10 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { "spec.topologyConstraint: Forbidden:", "spec.components[0].topologyConstraint: Forbidden:", }}, + {name: "missing conductor role", missingConductor: "role", messages: []string{"LPX components must declare exactly one conductor role"}}, + {name: "missing conductor template", missingConductor: "podTemplate", messages: []string{"LPX conductor requires an explicit podTemplate"}}, + {name: "shared draft missing conductor role", sharedDraft: true, missingConductor: "role", messages: []string{"LPX components must declare exactly one conductor role"}}, + {name: "shared draft missing conductor template", sharedDraft: true, missingConductor: "podTemplate", messages: []string{"LPX conductor requires an explicit podTemplate"}}, {name: "disabled Grove", messages: []string{"Grove is disabled"}}, {name: componentProvider, messages: []string{"requires the Grove workload provider"}}, {name: "long serving name", componentName: strings.Repeat("serving-", 7) + "engine"}, @@ -904,6 +909,14 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { if scenario.componentName != "" { component.ComponentName = scenario.componentName } + switch scenario.missingConductor { + case "role": + component.Roles = slices.DeleteFunc(component.Roles, func(role v1beta1.ComponentRoleSpec) bool { + return role.Name == v1beta1.ComponentRoleLPXConductor + }) + case "podTemplate": + component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil + } switch scenario.name { case "invalid intent": source.Spec.ProviderOverride = &v1beta1.ProviderOverride{ @@ -940,6 +953,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { } t.Log("Reject unsupported intent before downloads and let valid names reach the download gate") + before := source.DeepCopy() _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) if scenario.name == componentProvider { require.ErrorContains(t, err, "requires the Grove workload provider") @@ -979,6 +993,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { require.NoError(t, r.List(t.Context(), requests)) require.Empty(t, pcs.Items) require.Empty(t, requests.Items) + require.Equal(t, before, source) }) } } diff --git a/deploy/operator/internal/controller/lpx/dependencies.go b/deploy/operator/internal/controller/lpx/dependencies.go index a0fdee1c2ab3..ea8e02605a7a 100644 --- a/deploy/operator/internal/controller/lpx/dependencies.go +++ b/deploy/operator/internal/controller/lpx/dependencies.go @@ -41,7 +41,7 @@ func lpxTopologyBindingReferences(obj client.Object) []string { // Extraction uses only source intent, including references not yet created. func lpxDRAClaimReferences(template bool) client.IndexerFunc { return func(obj client.Object) []string { - // Index every authored role, including Agent templates used by conductor fallback. + // Index dependencies from every independently authored role. source := obj.(*nvidiacomv1beta1.DynamoGraphDeployment) references := sets.New[string]() for index := range source.Spec.Components { diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 55fb92819f13..917ff1b66eae 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -529,13 +529,26 @@ func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { pcsBefore := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, pcsBefore)) - t.Log("Fail the actual snapshot dependency or remove required runtime SSH configuration") - message := "node-local LPU runtime requires an MPI SSH secret name" + t.Log("Fail the actual snapshot dependency or break cross-role model storage") + message := "must use the Conductor model-storage mount path" if test.snapshotError != nil { reconciler.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: test.snapshotError} message = test.snapshotError.Error() } else { - reconciler.Config.MPI.SSHSecretName = "" + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(source), source)) + conductor := source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) + for index := range conductor.PodTemplate.Spec.Containers[0].VolumeMounts { + mount := &conductor.PodTemplate.Spec.Containers[0].VolumeMounts[index] + if mount.Name == consts.ModelStorageVolumeName { + mount.MountPath = "/different-model-storage" + } + } + require.NoError(t, reconciler.Update(ctx, source)) + require.NoError(t, reconciler.Get(ctx, key, dgd)) + revision, revisionErr := dynamo.LPXInputRevision(source, "") + require.NoError(t, revisionErr) + dgd.Spec.InputRevision = revision + require.NoError(t, reconciler.Update(ctx, dgd)) } _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) pcs := &grovev1alpha1.PodCliqueSet{} @@ -2441,6 +2454,12 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D }}, }, } + // Each role authors its own startup; the conductor cannot inherit the Agent command. + agentTemplate := source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent).PodTemplate + agentTemplate.Spec.Containers[0].Command = []string{"/bin/quasar-entrypoint"} + conductorTemplate := agentTemplate.DeepCopy() + conductorTemplate.Spec.Containers[0].Command = []string{"/bin/nova"} + source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate = conductorTemplate if pipeline == lpx.PipelineLPX { *source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) = nvidiacomv1beta1.ComponentRoleSpec{ Name: nvidiacomv1beta1.ComponentRoleLPXConductor, @@ -2626,7 +2645,6 @@ func newLPXTestReconciler( recorder := events.NewFakeRecorder(100) config := &configv1alpha1.OperatorConfiguration{ LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, } runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, DRA: true, LPX: true}} return &graphReconciler{ diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 79416cea01f7..7d8acfecc140 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -370,7 +370,12 @@ func newModelDownloadDGD(buildIDs ...string) *v1beta1.DynamoGraphDeployment { }}}}, } if i == len(buildIDs)-1 { - component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}) + component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{ + Name: v1beta1.ComponentRoleLPXConductor, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime", Command: []string{"/bin/nova"}}}, + }}, + }) } else { component.ComponentName = "draft" } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index ada3e08523ef..4ef715589788 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -63,7 +63,7 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { require.NoError(t, err) plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) - config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} + config := &configv1alpha1.OperatorConfiguration{} kube := newTestLPXClient(t) runtimeConfig := &controller_common.RuntimeConfig{} pcs, _, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, @@ -121,7 +121,7 @@ func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { require.NoError(t, err) plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) - config := &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}} + config := &configv1alpha1.OperatorConfiguration{} pcs, _, err := renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) require.NoError(t, err) @@ -173,12 +173,11 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { {"node-local-v2-specdecode", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, {"node-local-v3-hx-lpu-only", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, {"node-local-v3-hx-hybrid", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, - {"node-local-v3-hx-hybrid-inherited", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, {"node-local-v3-hx-specdecode", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, } { t.Run(test.name, func(t *testing.T) { t.Log("Resolve the authored engine and its exact child without changing either input") - fixture := strings.TrimSuffix(test.name, "-inherited") + fixture := test.name payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + fixture + ".input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} @@ -215,12 +214,6 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { component.ModelRef = &v1beta1.ModelReference{Name: "test/model"} component.MinAvailable = ptr.To(int32(1)) hybrid := strings.HasSuffix(fixture, "-hybrid") - inherited := fixture != test.name - if inherited { - t.Log("Let the HX GPU conductor inherit the Agent template and its authored config mount") - component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil - component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = resource.MustParse("1") - } singleXT := test.name == "node-local-v2-lpu-only" if hybrid { component.Replicas = ptr.To(int32(3)) @@ -252,7 +245,6 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{ - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, } @@ -397,7 +389,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, "from-component", clique.Labels["shared-label"]) require.Equal(t, "false", clique.Annotations[commonconsts.KubeAnnotationEnableMetrics]) require.Equal(t, "kubernetes", clique.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend]) - if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != lpxv1alpha1.PodRoleCyborgWorker || test.wantFamily == lpxv1alpha1.TargetFamilyXt8888 || inherited { + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != lpxv1alpha1.PodRoleCyborgWorker || test.wantFamily == lpxv1alpha1.TargetFamilyXt8888 { t.Log("Bind the generated immutable ConfigMap alongside authored volumes and mounts") configIndex := slices.IndexFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "config" }) require.GreaterOrEqual(t, configIndex, 0) @@ -459,89 +451,74 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { } func TestLPXSpecDecodeConductorTemplate(t *testing.T) { - t.Log("Create immutable native builds for the explicit and inherited conductor cases") + t.Log("Give the speculative components distinct authored Agent and conductor commands") registry := newTestDataModelRegistry(t, t.TempDir()) + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") + require.NoError(t, err) + source := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, source)) + for _, component := range lpx.Components(source) { + agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate + agent.Spec.Containers[0].Image = component.ComponentName + "-runtime" + agent.Spec.Containers[0].Command = []string{"/bin/quasar-entrypoint"} + agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} + } + target := lpx.ServingComponent(source) + template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + template.Spec.Containers[0].Image = "independent-conductor-runtime" + template.Spec.Containers[0].Command = []string{"/bin/nova"} + template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} + template.Labels = map[string]string{"owner": "explicit-conductor"} + template.Spec.NodeSelector = map[string]string{"runtime-role": "conductor"} + template.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"conductor"}}}, + }}}, + }} + before := source.DeepCopy() + selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) + require.NoError(t, err) - for _, explicit := range []bool{false, true} { - t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) { - t.Log("Give the speculative components distinct templates and select the target conductor") - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") - require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - for _, component := range lpx.Components(source) { - agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate - agent.Spec.Containers[0].Image = component.ComponentName + "-runtime" - agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} - } - target := lpx.ServingComponent(source) - wantImage := target.ComponentName + "-runtime" - target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil - template := target.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate - if explicit { - template = template.DeepCopy() - wantImage = "independent-conductor-runtime" - template.Spec.Containers[0].Image = wantImage - template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} - template.Labels = map[string]string{"owner": "explicit-conductor"} - template.Spec.NodeSelector = map[string]string{"runtime-role": "conductor"} - template.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"conductor"}}}, - }}}, - }} - target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = template - } - template.Spec.Containers[0].VolumeMounts = append(template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}) - template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) - before := source.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) - require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) - require.NoError(t, err) - - t.Log("Render one shared conductor without changing either component's Agent template") - pcs, _, err := renderPodCliqueSet(t.Context(), source, - &configv1alpha1.OperatorConfiguration{MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}}, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, source)) - require.NoError(t, err) + t.Log("Render one shared conductor without changing either component's Agent template") + pcs, _, err := renderPodCliqueSet(t.Context(), source, + &configv1alpha1.OperatorConfiguration{}, + &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, source)) + require.NoError(t, err) - t.Log("Preserve independent role fields while retaining the Nova launch") - require.Len(t, pcs.Spec.Template.Cliques, len(plan.Agents)+1) - conductors := 0 - for _, clique := range pcs.Spec.Template.Cliques { - container := clique.Spec.PodSpec.Containers[0] - for _, mount := range container.VolumeMounts { - require.True(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { - return volume.Name == mount.Name - }), "%s mount %s requires a volume", clique.Name, mount.Name) - } - if clique.Name == plan.ConductorTemplate { - conductors++ - require.Equal(t, wantImage, container.Image) - require.Equal(t, []string{"/bin/nova"}, container.Command) - configFlag := slices.Index(container.Args, "--datacenter-config-filepath") - require.GreaterOrEqual(t, configFlag, 0) - require.Equal(t, "/configs/datacenter.toml", container.Args[configFlag+1]) - if explicit { - template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate - require.Equal(t, "explicit-conductor", clique.Labels["owner"]) - require.Equal(t, template.Spec.NodeSelector, clique.Spec.PodSpec.NodeSelector) - require.Equal(t, template.Spec.Affinity, clique.Spec.PodSpec.Affinity) - require.Contains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) - require.NotContains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) - } - } else { - component := clique.Labels[commonconsts.KubeLabelDynamoComponent] - require.Equal(t, component+"-runtime", container.Image) - require.Contains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) - require.NotContains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) - } - } - require.Equal(t, 1, conductors) - require.Equal(t, before, source) - }) + t.Log("Keep Nova separate from Quasar and preserve each role's metadata and storage") + require.Len(t, pcs.Spec.Template.Cliques, len(plan.Agents)+1) + conductors := 0 + for _, clique := range pcs.Spec.Template.Cliques { + container := clique.Spec.PodSpec.Containers[0] + for _, mount := range container.VolumeMounts { + require.True(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { + return volume.Name == mount.Name + }), "%s mount %s requires a volume", clique.Name, mount.Name) + } + if clique.Name == plan.ConductorTemplate { + conductors++ + require.Equal(t, "independent-conductor-runtime", container.Image) + require.Equal(t, []string{"/bin/nova"}, container.Command) + configFlag := slices.Index(container.Args, "--datacenter-config-filepath") + require.GreaterOrEqual(t, configFlag, 0) + require.Equal(t, "/configs/datacenter.toml", container.Args[configFlag+1]) + require.Equal(t, "explicit-conductor", clique.Labels["owner"]) + require.Equal(t, template.Spec.NodeSelector, clique.Spec.PodSpec.NodeSelector) + require.Equal(t, template.Spec.Affinity, clique.Spec.PodSpec.Affinity) + require.Contains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) + require.NotContains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) + } else { + component := clique.Labels[commonconsts.KubeLabelDynamoComponent] + require.Equal(t, component+"-runtime", container.Image) + require.Equal(t, []string{"/bin/quasar-entrypoint"}, container.Command) + require.Contains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) + require.NotContains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) + } } + require.Equal(t, 1, conductors) + require.Equal(t, before, source) } func newTestDataModelRegistry(t *testing.T, registryRoot string) *lpx.ModelRegistry { @@ -821,7 +798,6 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { Orchestrators: configv1alpha1.OrchestratorConfiguration{ Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, }, - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, } const buildID = "single-v2-manifest-defaults/build_manifest_defaults" registryRoot := t.TempDir() @@ -836,7 +812,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { t.Log("Render manifest-derived, equivalent explicit, and overridden runtime settings") configs := make([]renderedConfig, 0, 3) for _, settingsJSON := range []string{"", `{"batch_size":1,"swa":{"chunked":false}}`, `{"sequence_length":65536}`} { - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml") + payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/single_v2.input.yaml") require.NoError(t, err) var deployment v1beta1.DynamoGraphDeployment @@ -844,6 +820,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { require.Len(t, deployment.Spec.Components, 1) require.NotNil(t, deployment.Spec.Components[0].LPX) require.Len(t, deployment.Spec.Components[0].Roles, 2) + deployment.Spec.Components[0].LPX = &v1beta1.LPXConfig{BuildID: buildID} if settingsJSON != "" { deployment.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(settingsJSON)} } @@ -997,7 +974,6 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { }, }, LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, } registryRoot := t.TempDir() @@ -1116,7 +1092,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { } func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *testing.T) { - t.Log("Load the pre-scheduler V2 hybrid DGD shape without scheduler intent or an image-owned command") + t.Log("Load the pre-scheduler V2 hybrid DGD shape without scheduler intent and with an authored runtime command") payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) require.NoError(t, err) dgd := &v1beta1.DynamoGraphDeployment{} @@ -1144,7 +1120,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test sidecar := corev1.Container{Name: "cyborg", Image: "sidecar", Args: []string{"observe"}} cyborgRole.PodTemplate.Spec.Containers = append(cyborgRole.PodTemplate.Spec.Containers, sidecar) - t.Log("Customize Agent metadata and placement while leaving its launch to the runtime") + t.Log("Customize Agent metadata and placement with an image ENTRYPOINT and authored arguments") agentTemplate := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate agentMain := &agentTemplate.Spec.Containers[0] require.NotEmpty(t, agentMain.Command) @@ -1193,7 +1169,6 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test controllerConfig := &configv1alpha1.OperatorConfiguration{ Infrastructure: configv1alpha1.InfrastructureConfiguration{ETCDAddress: "etcd", NATSAddress: "nats"}, - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, LPX: configv1alpha1.LPXConfiguration{Enabled: true}, } @@ -1203,7 +1178,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test ) require.NoError(t, err) - t.Log("Verify LPX places the worker while the generated backend owns the complete process contract") + t.Log("Verify LPX places the worker while retaining the authored process contract") var agent, cyborg *grovev1alpha1.PodCliqueTemplateSpec serving := 0 for _, clique := range pcs.Spec.Template.Cliques { @@ -1234,9 +1209,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test main := agent.Spec.PodSpec.Containers[0] require.Equal(t, "agent", main.Name) - require.Equal(t, []string{"/bin/hydra-entrypoint"}, main.Command) - require.Equal(t, []string{"start"}, main.Args) - require.NotContains(t, main.Args, "--agent-env-vars") + require.Nil(t, main.Command) + require.Equal(t, agentMain.Args, main.Args) require.NotNil(t, main.SecurityContext) require.True(t, *main.SecurityContext.Privileged) require.False(t, main.Stdin) @@ -1293,8 +1267,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Contains(t, cyborg.Spec.PodSpec.Containers, sidecar) require.NotEmpty(t, cyborg.Spec.PodSpec.Containers) cyborgMain := cyborg.Spec.PodSpec.Containers[0] - require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, cyborgMain.Command) - require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--", "/opt/gpu-runtime", "serve"}, cyborgMain.Args) + require.Equal(t, authoredCyborgMain.Command, cyborgMain.Command) + require.Equal(t, authoredCyborgMain.Args, cyborgMain.Args) require.Contains(t, cyborgMain.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) @@ -1362,19 +1336,22 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) } -func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testing.T) { +func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing.T) { t.Log("Share the immutable build registry across entrypoint cases") registry := newTestDataModelRegistry(t, t.TempDir()) kubeClient := newTestLPXClient(t) modes := []struct { - name string - file string - hybrid bool - single bool + name string + file string + hybrid bool + conductorConfig bool + hx bool }{ - {name: "Single V2", file: "single_v2.input.yaml", single: true}, - {name: "V2 LPU-only", file: "node-local-v2-lpu-only.input.yaml", single: true}, - {name: "V2 hybrid", file: "node-local-v2-hybrid.input.yaml", hybrid: true}, + {name: "Single V2", file: "single_v2.input.yaml", conductorConfig: true}, + {name: "V2 LPU-only", file: "node-local-v2-lpu-only.input.yaml", conductorConfig: true}, + {name: "V2 hybrid", file: "node-local-v2-hybrid.input.yaml", hybrid: true, conductorConfig: true}, + {name: "HX LPU-only", file: "node-local-v3-hx-lpu-only.input.yaml", conductorConfig: true, hx: true}, + {name: "HX hybrid", file: "node-local-v3-hx-hybrid.input.yaml", hybrid: true, hx: true}, } intents := []struct { name string @@ -1386,8 +1363,10 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin configPath *string configRole int envFrom bool + customProbes bool }{ {name: "image entrypoint and image command"}, + {name: "custom exec health probes", command: []string{"/opt/custom-runtime"}, customProbes: true}, {name: "default startup without config mount", configPath: ptr.To(""), configRole: 1}, {name: "default startup with misplaced config mount", configPath: ptr.To("/custom"), configRole: 1}, {name: "default Agent without config mount", configPath: ptr.To("")}, @@ -1396,6 +1375,11 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin {name: "image entrypoint with user arguments", args: []string{"serve"}}, {name: "explicit executable with image arguments", command: []string{"/opt/custom-runtime"}}, {name: "explicit executable and arguments", command: []string{"/opt/custom-runtime"}, args: []string{"serve"}}, + {name: "allocation flag in arguments", args: []string{"--allocation", "template-target"}}, + {name: "allocation flag in command", command: []string{"/opt/custom-runtime", "--allocation=template-target"}}, + {name: "allocation environment expansion", args: []string{"--allocation", "$(LPX_ALLOCATION)"}}, + {name: "argument terminator", command: []string{"/opt/custom-runtime"}, args: []string{"--", "literal argument"}}, + {name: "shell command", command: []string{"/bin/sh", "-c"}, args: []string{"exec /opt/custom-runtime --allocation $LPX_ALLOCATION"}}, { name: "explicit Cyborg shell remains user-owned", command: []string{"/opt/custom-runtime"}, @@ -1424,7 +1408,7 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin require.NoError(t, err) for _, intent := range intents { - if intent.hybridOnly && !mode.hybrid { + if intent.hybridOnly && !mode.hybrid || intent.configPath != nil && intent.configRole == 1 && !mode.conductorConfig { continue } t.Run(intent.name, func(t *testing.T) { @@ -1446,6 +1430,22 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin } main.Command = slices.Clone(command) main.Args = slices.Clone(args) + main.StartupProbe, main.LivenessProbe, main.ReadinessProbe = nil, nil, nil + if intent.customProbes { + main.StartupProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "started"}}}, + FailureThreshold: 12, PeriodSeconds: 4, TimeoutSeconds: 2, + } + main.LivenessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "live"}}}, + FailureThreshold: 4, PeriodSeconds: 6, TimeoutSeconds: 3, + } + main.ReadinessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "ready"}}}, + FailureThreshold: 2, PeriodSeconds: 5, TimeoutSeconds: 1, + } + } + main.Env = append(main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) if intent.envFrom && roleIndex == 0 { main.EnvFrom = []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated-env"}}}} } @@ -1461,14 +1461,13 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin } } + before := dgd.DeepCopy() selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) require.NoError(t, err) pcs, _, err := renderPodCliqueSet( t.Context(), dgd, - &configv1alpha1.OperatorConfiguration{ - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, - }, + &configv1alpha1.OperatorConfiguration{}, &controller_common.RuntimeConfig{}, kubeClient, nil, @@ -1476,65 +1475,38 @@ func TestGenerateGrovePodCliqueSet_V2NodeLocalPreservesImageEntrypoint(t *testin mustPlanSelectedLPX(t, dgd, selected), newLPXRenderDeployment(t, dgd), ) - if intent.configPath != nil { - require.ErrorContains(t, err, `requires a volume mounted at "/configs"`) - return - } - if mode.single && len(intent.command) == 0 && len(intent.args) > 0 { - require.ErrorContains(t, err, "only supports --instance-model-name") + require.Equal(t, before, dgd) + if mode.hx && intent.configPath != nil && *intent.configPath != "" { + require.ErrorContains(t, err, `reserves volume "config" at "/configs"`) return } require.NoError(t, err) - t.Log("Verify explicit commands are preserved and direct Agents receive their launch defaults") + t.Log("Verify every role preserves authored startup and health probes, including omission") var agents, conductors, cyborgs int for _, clique := range pcs.Spec.Template.Cliques { main := clique.Spec.PodSpec.Containers[0] + authored := templates[1].Spec.Containers[0] + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { + authored = templates[0].Spec.Containers[0] + } + require.Equal(t, authored.Command, main.Command, "%s command", clique.Name) + require.True(t, slices.Equal(authored.Args, main.Args), "%s args", clique.Name) + require.Equal(t, authored.StartupProbe, main.StartupProbe, "%s startup", clique.Name) + require.Equal(t, authored.LivenessProbe, main.LivenessProbe, "%s liveness", clique.Name) + require.Equal(t, authored.ReadinessProbe, main.ReadinessProbe, "%s readiness", clique.Name) switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { case lpxv1alpha1.PodRoleAgent: agents++ require.False(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) - if len(intent.command)+len(intent.args) == 0 { - if mode.hybrid { - require.Equal(t, []string{"/bin/hydra-entrypoint"}, main.Command) - require.Equal(t, []string{"start"}, main.Args) - } else { - require.Equal(t, []string{"/bin/quasar-entrypoint"}, main.Command) - require.Equal(t, []string{"--partition-metadata"}, main.Args) - } - } else { - wantCommand := intent.command - require.Equal(t, wantCommand, main.Command) - require.True(t, slices.Equal(intent.args, main.Args)) - } case lpxv1alpha1.PodRoleConductor: conductors++ - command := intent.command - if len(command) == 0 { - command = []string{"/bin/nova"} - } - require.Equal(t, command, main.Command) - require.True(t, slices.Equal(intent.args, main.Args[:len(intent.args)])) - allocation := slices.Index(main.Args, "--allocation") - require.GreaterOrEqual(t, allocation, 0) - require.Equal(t, strings.Join(clique.Spec.StartsAfter, ","), main.Args[allocation+1]) + require.Contains(t, main.Env, corev1.EnvVar{ + Name: "LPX_ALLOCATION", Value: strings.Join(clique.Spec.StartsAfter, ","), + }) + require.NotContains(t, main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) case lpxv1alpha1.PodRoleCyborgWorker: cyborgs++ - wantCommand := intent.command - wantArgs := intent.args - if intent.cyborgCommand != nil { - wantCommand = intent.cyborgCommand - wantArgs = intent.cyborgArgs - } - if len(wantCommand) == 0 { - wantCommand = []string{"/usr/local/bin/cyborg"} - } - require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, main.Command) - require.Equal(t, []string{"--expand-hosts", "--"}, main.Args[:2]) - require.Equal(t, append(slices.Clone(wantCommand), wantArgs...), main.Args[2:]) - require.NotNil(t, main.LivenessProbe) - require.NotNil(t, main.ReadinessProbe) - require.NotNil(t, main.StartupProbe) require.NotEmpty(t, main.Ports) } } @@ -1602,9 +1574,7 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { require.NoError(t, err) plan := mustPlanSelectedLPX(t, source, selected) pcs, _, err := renderPodCliqueSet(t.Context(), source, - &configv1alpha1.OperatorConfiguration{ - MPI: configv1alpha1.MPIConfiguration{SSHSecretName: "ssh-secret"}, - }, + &configv1alpha1.OperatorConfiguration{}, &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) @@ -1637,9 +1607,7 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { require.Equal(t, "9", env["TOTAL_REPLICAS"]) require.Equal(t, "/custom-servers", env["SERVER_HOSTS_FILE"]) - t.Log("Select only SWA initialization while preserving the authored command and host-file override") - require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, main.Command) - wantArgs := append([]string{"--swa-batch-ids", "--"}, leader.Spec.Containers[0].Command...) - wantArgs = append(wantArgs, leader.Spec.Containers[0].Args...) - require.Equal(t, wantArgs, main.Args) + t.Log("Preserve the authored command and host-file override without adding a launcher") + require.Equal(t, leader.Spec.Containers[0].Command, main.Command) + require.Equal(t, leader.Spec.Containers[0].Args, main.Args) } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 2dc91d7cbc49..cea496aec603 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -276,9 +276,8 @@ func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment) string { func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, plan *dynamolpx.MaterializationPlan) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { // Pass runtime inputs; deployment identity is stamped only on final resources. input := &dynamolpx.RenderInput{ - MinAvailable: p.component.MinAvailable, - Stages: make(map[string]corev1.PodTemplateSpec), - SSHSecretName: p.operatorConfig.MPI.SSHSecretName, + MinAvailable: p.component.MinAvailable, + Stages: make(map[string]corev1.PodTemplateSpec), } // Resolve preserved alpha metadata once for all independently rendered roles. @@ -292,7 +291,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p alphaComponent := alphaComponents[component.ComponentName] agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) - lpuDefaults := &imageEntrypointComponentDefaults{ComponentDefaults: &BaseComponentDefaults{}} + lpuDefaults := &podTemplateRuntimeDefaults{ComponentDefaults: &BaseComponentDefaults{}} lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, lpuDefaults, nil) if err != nil { @@ -306,34 +305,28 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p continue } - // Non-hybrid conductors use their own template, or the Agent fallback in LPX lowering. + // The serving role owns its startup independently of the Agent template. if workload.Pipeline() != dynamolpx.PipelineLPX { - if conductor != nil && conductor.PodTemplate != nil { - role := lpxRoleComponent(component, conductor.PodTemplate, p.dynamoDeployment, p.discoveryBackend) - input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, lpuDefaults, nil) - if err != nil { - return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) - } - input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName - input.Conductor.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" + role := lpxRoleComponent(component, conductor.PodTemplate, p.dynamoDeployment, p.discoveryBackend) + input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, + p.discoveryContext, lpuDefaults, nil) + if err != nil { + return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) } + input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName + input.Conductor.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" continue } - // Keep the existing hybrid execution path; only its authored template location changes. - template, replicas := agent.PodTemplate, int32(1) - if conductor != nil { - replicas = ptr.Deref(conductor.Replicas, 1) - if conductor.PodTemplate != nil { - template = conductor.PodTemplate - } - } + // Hybrid conductors use their authored GPU template and replica count. + template, replicas := conductor.PodTemplate, ptr.Deref(conductor.Replicas, 1) role := lpxRoleComponent(component, template, p.dynamoDeployment, p.discoveryBackend) role.ComponentType = v1beta1.ComponentTypeDecode role.Replicas = ptr.To(replicas) role.MinAvailable = nil - defaults := ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)) + var defaults ComponentDefaults = &podTemplateRuntimeDefaults{ + ComponentDefaults: ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)), + } if workload.BuildFamily() == dynamolpx.BuildFamilyXT { input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan, lpuTemplate.Spec) if err != nil { @@ -343,8 +336,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p ComponentDefaults: defaults, workload: workload, configMapName: input.CyborgConfigMap.Name, replicas: *role.Replicas, lpxPodSpec: lpuTemplate.Spec, } - } else { - defaults = &imageEntrypointComponentDefaults{ComponentDefaults: defaults} } gpu := p gpu.component = role @@ -386,18 +377,23 @@ func lpxRoleComponent(source *v1beta1.DynamoComponentDeploymentSharedSpec, templ return role } -// imageEntrypointComponentDefaults preserves all role-specific defaults while -// leaving the executable to the workload image unless the user supplied one. -type imageEntrypointComponentDefaults struct { +// podTemplateRuntimeDefaults leaves startup and health checks to the authored role +// while retaining shared infrastructure bindings. +type podTemplateRuntimeDefaults struct { ComponentDefaults } -func (d *imageEntrypointComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { +func (d *podTemplateRuntimeDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { container, err := d.ComponentDefaults.GetBaseContainer(context) if err != nil { return corev1.Container{}, err } + + // Omitted template settings must not inherit another runtime's launch or health contract. container.Command = nil + container.StartupProbe = nil + container.LivenessProbe = nil + container.ReadinessProbe = nil return container, nil } @@ -416,7 +412,6 @@ func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentCont if err != nil { return corev1.Container{}, err } - container.Command = nil if err := dynamolpx.ApplySelectedCyborgContainerDefaults( &container, d.workload, diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index d2a44b367204..ffb7f2a84c1c 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -97,7 +97,7 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { t.Log("Render the role before runtime-specific lowering") template, err := renderSelectedLPXRole(component, source, nil, &configv1alpha1.OperatorConfiguration{}, &mockSecretsRetriever{}, DiscoveryContext{}, - &imageEntrypointComponentDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) + &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) require.NoError(t, err) t.Log("Retain authored security context and default only an absent one") @@ -153,8 +153,8 @@ func TestLPXPCSNamePreservesReadablePrefixWithinGroveBudget(t *testing.T) { }{ {name: "c", prefix: "c"}, {name: "gpt-oss-120b-production", prefix: "gpt-oss-120b-production"}, - {name: "apaprotskyi-llama3-8b-lp20", prefix: "apaprotskyi-llama3-8b-lp20"}, - {name: "apaprotskyi-gpt-oss-20b-lp20-b300", prefix: "apaprotskyi-gpt-oss-20b-lp20-b300"}, + {name: "test-models-llama3-8b-lp20", prefix: "test-models-llama3-8b-lp20"}, + {name: "test-models-gpt-oss-20b-lp20-b300", prefix: "test-models-gpt-oss-20b-lp20-b300"}, {name: "chat.example", prefix: "chat-example"}, {name: "120b-chat", prefix: "lpx-120b-chat"}, {name: strings.Repeat("a", 33), prefix: strings.Repeat("a", 33)}, diff --git a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go b/deploy/operator/internal/dynamo/lpx/conductor_runtime.go deleted file mode 100644 index 0bef27f2eac2..000000000000 --- a/deploy/operator/internal/dynamo/lpx/conductor_runtime.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" -) - -const ( - lpuModelConfigFilepath = lpuConfigMountPath + "/model_config.toml" - lpuDatacenterConfigFilepath = lpuConfigMountPath + "/datacenter.toml" -) - -func updateLPUConductorContainer( - container *corev1.Container, - allocation string, -) { - // Nova reads the original configuration and expands hostname templates natively. - container.Name = "conductor" - if len(container.Command) == 0 { - container.Command = []string{"/bin/nova"} - } - - // Nova and OpenMPI require root in the supported direct-DGD runtime. - if container.SecurityContext == nil { - container.SecurityContext = &corev1.SecurityContext{} - } - container.SecurityContext.RunAsUser = ptr.To(int64(0)) - container.SecurityContext.RunAsGroup = ptr.To(int64(0)) - container.SecurityContext.RunAsNonRoot = ptr.To(false) - - // Append the complete operator-owned Nova launch contract after user arguments. - container.Args = append(container.Args, - "--agent-path", - "/bin/agent", - "--agent-wd", - runtimeTemporaryStorageMountPath, - "--conductor-bind-host", - "0.0.0.0", - "--conductor-hostname", - "$(POD_IP)", - "--control-server-addr", - "$(POD_IP):51045", - - // configmap - "--datacenter-config-filepath", - lpuDatacenterConfigFilepath, - "--model-config-filepath", - lpuModelConfigFilepath, - - // logs - "--json-logs", - "--disable-tracing", - - // mpirun - "--mpirun-path", - "/bin/mpirun", - "--mpi-ssh-cmd", - fmt.Sprintf( - "ssh -p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s", - LPUSSHPort, - conductorSSHPrivateKeyPath, - ), - "--mpi-use-tcp", - - // alloc - "--allocation", allocation, - - // conductor/agent lifecycle - "--agent-disable-auto-restart", - "--agent-server-max-retries", - "60", - "--agent-server-retry-sleep-dur", - "2s", - "--controlplane-start-port", - "12345", - "--graceful-shutdown-timeout", - "0s", - - // misc - "--response-timeout", - "120s", - - // dyn - "--frontend-type", - "dynamo", - "--endpoint-id", - "dyn://$(DYN_NAMESPACE).backend.generate", - "--request-plane", - "tcp", - "--store-backend", - "kubernetes", - ) - - // Allow root OpenMPI to reach the SSH workers with the copied private key. - container.Env = append(container.Env, - corev1.EnvVar{Name: "OMPI_ALLOW_RUN_AS_ROOT", Value: "1"}, - corev1.EnvVar{Name: "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM", Value: "1"}, - corev1.EnvVar{ - Name: "OMPI_MCA_plm_rsh_args", - Value: fmt.Sprintf( - "-p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=%s", - LPUSSHPort, - conductorSSHPrivateKeyPath, - ), - }, - ) - - // Default readiness to Nova's health endpoint while preserving an explicit probe. - if container.ReadinessProbe == nil { - container.ReadinessProbe = &corev1.Probe{ - FailureThreshold: 3, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - SuccessThreshold: 1, - TimeoutSeconds: 1, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromInt(51045), - Scheme: corev1.URISchemeHTTP, - }, - }, - } - } -} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go deleted file mode 100644 index f47ab5685fb1..000000000000 --- a/deploy/operator/internal/dynamo/lpx/cyborg_launcher_fixture_test.go +++ /dev/null @@ -1,90 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -// Image and command from inference-engine-instances at 334110c9676479224544a3879ebf8723d04ab3c7, -// kustomize/overlays/hou2-prod1/deployments/gpt-oss-20b-lp20-b300.yaml. -// The command is /bin/bash -lc; this is its complete, unmodified argument. -const pinnedCyborgImage = "us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:0.157.0@sha256:e13b373f9a08ab9dc1d7d4c618180815aa3bbac471f0ca25426da79e8437f908" - -const pinnedCyborgLauncher = `set -euo pipefail - -if [[ -z "${GBUILD_MANIFEST_PATH:-}" ]]; then - echo "GBUILD_MANIFEST_PATH was not provided by the Dynamo operator" >&2 - exit 1 -fi -if [[ -z "${CYBORG_BATCH_SIZE:-}" ]]; then - echo "CYBORG_BATCH_SIZE was not provided by the Dynamo operator" >&2 - exit 1 -fi -MODEL_RUNTIME_DIR="${GBUILD_MANIFEST_PATH%/*}" -. /usr/local/bin/k8s_gpu_detect.sh -set_cuda_visible_devices_from_nvidia_mounts - -trap 'exit 0' TERM INT - -ulimit -l unlimited - -LPU_RDMA_ADDR_FILE="$(mktemp)" -/usr/local/bin/render_grove_lpu_addr_files \ - --server-hosts-file "${SERVER_HOSTS_FILE}" \ - --rdma-addr-file "${LPU_RDMA_ADDR_FILE}" - -cyborg_cmd=( - /usr/local/bin/cyborg - --lpu-rdma-addr-file "${LPU_RDMA_ADDR_FILE}" \ - --gpu 0 \ - --model-family gpt-oss \ - --max-inflight-tasks 1 \ - --num-layers 12 \ - --num-heads-q 64 \ - --head-dim-qk 64 \ - --head-dim-v 64 \ - --num-heads-kv 8 \ - --seqlen-kv 131072 \ - --tokens-per-page 32 \ - --hidden-size 2880 \ - --stop-tokens 200002 \ - --swa-block-size 128 \ - --swa-padding-size 128 \ - --lpu-tail-top-k 40 \ - --standalone-post-decoder \ - --embedding-path "${MODEL_RUNTIME_DIR}/runtime/text_embeddings.npz" \ - --attention-sinks-path "${MODEL_RUNTIME_DIR}/cuda/attention_sinks.npz" \ - --batch-size "${CYBORG_BATCH_SIZE}" \ - --nic auto \ - --dtype bf16 \ - --backend fpga_gpi \ - --dynamo-listen-port 9124 \ - --dynamo-readiness-port 9090 \ - --dynamo-model-name "${MODEL_NAME}" \ - --tokenizer-dir "${TOKENIZER_DIR}" \ - --dynamo-namespace "${DYN_NAMESPACE:-${DYN_PARENT_DGD_K8S_NAMESPACE:-inference-engine}}" \ - --dynamo-component "${DYN_COMPONENT:-decode}" \ - --dynamo-tool-call-parser harmony \ - --dynamo-reasoning-parser gpt_oss \ - --openai-http-port "${OPENAI_HTTP_PORT}" \ - --openai-http-head-host "${POD_IP}" -) - -cyborg_nsys_enable="${CYBORG_NSYS_ENABLE:-false}" -if [[ "${cyborg_nsys_enable,,}" == "true" || "${cyborg_nsys_enable}" == "1" ]]; then - nsys_session_name="${NSYS_SESSION_NAME:-cyborg}" - nsys_profile_dir="${NSYS_PROFILE_DIR:-/profiles}" - nsys_trace="${NSYS_TRACE:-cuda-sw,nvtx}" - nsys_cuda_graph_trace="${NSYS_CUDA_GRAPH_TRACE:-node}" - command -v nsys >/dev/null - mkdir -p "${nsys_profile_dir}" - exec nsys launch \ - --session-new "${nsys_session_name}" \ - --trace="${nsys_trace}" \ - --cuda-graph-trace="${nsys_cuda_graph_trace}" \ - --resolve-symbols=false \ - "${cyborg_cmd[@]}" -fi - -exec "${cyborg_cmd[@]}" -` diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index f0fd3b9ff2d7..d294e665a2e6 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -7,8 +7,6 @@ package lpx import ( "fmt" - "slices" - "strconv" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -46,24 +44,11 @@ func configureHybridCyborg( if err != nil { return err } - if projection.configuredBuild.Family == BuildFamilyXT { - setContainerEnv(container, false, corev1.EnvVar{Name: "RDMA_PORT", Value: strconv.Itoa(lpuRDMAPort)}) - } applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) applyCyborgSWACacheIDs(container, cyborgBatchSize) if err := applyCyborgManifestPath(container, projection, modelStorage.mount.MountPath); err != nil { return err } - configFile := "" - if cyborgConfigMap != nil && slices.ContainsFunc(container.Env, func(variable corev1.EnvVar) bool { - return variable.Name == selectedCyborgServerHostsFileEnv && variable.Value == runtimeTemporaryStorageMountPath+"/lpu_servers" && variable.ValueFrom == nil - }) { - configFile = "lpu_servers" - if err := validateRuntimeConfigStorage(&cyborg.Spec.PodSpec, container, configFile); err != nil { - return err - } - } - wrapCyborgStartup(container, cyborgBatchSize, configFile) cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 98c0e6741c89..85bfbe4708d2 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -55,8 +55,7 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { cyborg.Spec.Replicas = test.replicas cyborg.Spec.MinAvailable = ptr.To(test.replicas) _, err := RenderSelectedNodeLocal(pcs, workload, plan, RenderInput{ - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, - SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, }) if test.wantError { require.ErrorContains(t, err, "materialized Cyborg Pod hostname") @@ -106,14 +105,15 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { decode.Spec.PodSpec.Containers[0].Resources.Limits = corev1.ResourceList{ corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), } - decode.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} + decode.Spec.PodSpec.Containers[0].Command = []string{"/custom-cyborg", "--wrapper-option"} + decode.Spec.PodSpec.Containers[0].Args = []string{"argument with spaces", "literal $HOME", ""} decode.Spec.PodSpec.Containers[0].Env = append( decode.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "RDMA_PORT", Value: "12345"}, corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) input := RenderInput{ - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, } rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, input) require.NoError(t, err) @@ -136,12 +136,12 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { Name: "GBUILD_MANIFEST_PATH", Value: "/models/model-build/manifest.v2.capnp.bin", }) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: "RDMA_PORT", Value: "19877", + Name: "RDMA_PORT", Value: "12345", }) require.Contains(t, cyborg.Spec.PodSpec.Volumes, renderTestPodSpec().Volumes[0]) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, renderTestPodSpec().Containers[0].VolumeMounts[0]) - require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, cyborg.Spec.PodSpec.Containers[0].Command) - require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--", "/usr/local/bin/dynamo_main"}, cyborg.Spec.PodSpec.Containers[0].Args) + require.Equal(t, []string{"/custom-cyborg", "--wrapper-option"}, cyborg.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"argument with spaces", "literal $HOME", ""}, cyborg.Spec.PodSpec.Containers[0].Args) t.Log("Preserve an image-owned entrypoint") imageEntrypointPCS := renderTestPCS(true) @@ -154,11 +154,12 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) - t.Log("Render the agreed Cyborg binary when the command is omitted") + t.Log("Leave the image ENTRYPOINT selected when command is omitted") input.Stages = map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}} _, err = renderSelectedForTest(imageEntrypointPCS, []*ModelProjection{projection}, input) require.NoError(t, err) - require.Equal(t, []string{"/usr/local/bin/cyborg", "serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args[3:]) + require.Nil(t, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Command) + require.Equal(t, []string{"serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args) t.Log("Reject incomplete endpoint and fanout coverage of the same split-I/O runtime") @@ -178,7 +179,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { cyborg.Spec.MinAvailable = ptr.To(test.replicas) cyborg.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} _, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, }) require.ErrorContains(t, err, test.wantError) }) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index 82aa6d27e2a1..d3b73974a784 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -30,7 +30,7 @@ func applyCyborgManifestPath(container *corev1.Container, projection *ModelProje if err != nil { return fmt.Errorf("resolve GBuild manifest path: %w", err) } - setContainerEnv(container, false, corev1.EnvVar{ + setContainerEnv(container, corev1.EnvVar{ Name: gbuildManifestPathEnv, Value: filepath.Join(buildRoot, gbuildManifestV2CapnpFile), }) @@ -39,7 +39,7 @@ func applyCyborgManifestPath(container *corev1.Container, projection *ModelProje // applyCyborgRuntimeIO projects the resolved split-I/O fanout into one Cyborg container. func applyCyborgRuntimeIO(container *corev1.Container, cyborgBatchSize int, ioFPGACount int32) { - setContainerEnv(container, false, + setContainerEnv(container, corev1.EnvVar{Name: cyborgFpgaGpiIOFPGACountEnv, Value: strconv.FormatInt(int64(ioFPGACount), 10)}, corev1.EnvVar{ Name: cyborgFpgaGpiReplicaIndexEnv, @@ -54,7 +54,7 @@ func applyCyborgRuntimeIO(container *corev1.Container, cyborgBatchSize int, ioFP // applyCyborgSWACacheIDs binds the replica index for an unbatched Cyborg container. func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { if cyborgBatchSize == 1 { - setContainerEnv(container, false, corev1.EnvVar{ + setContainerEnv(container, corev1.EnvVar{ Name: cyborgSwaCacheIDsEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ FieldPath: grovePodCliquePodIndexPath, @@ -63,30 +63,6 @@ func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { } } -// wrapCyborgStartup prepares engine-local hosts and SWA IDs in one wrapper. -func wrapCyborgStartup(container *corev1.Container, cyborgBatchSize int, configFile string) { - if cyborgBatchSize == 1 && configFile == "" { - return - } - - args := make([]string, 0, 3+len(container.Command)+len(container.Args)) - if configFile != "" { - args = append(args, "--expand-hosts") - } - if cyborgBatchSize > 1 { - args = append(args, "--swa-batch-ids") - } - args = append(args, "--") - if len(container.Command) == 0 { - args = append(args, "/usr/local/bin/cyborg") - } else { - args = append(args, container.Command...) - } - args = append(args, container.Args...) - container.Command = []string{"/usr/local/bin/cyborg-entrypoint"} - container.Args = args -} - // cyborgRuntimeIO requires a nonnil normalized build and validates its Cyborg replica domain. func cyborgRuntimeIO(build *Build, replicas int32) (int, int32, error) { // Require every physical endpoint and client-owned transaction in this replica domain. @@ -125,15 +101,13 @@ func configuredCyborgBatchSize(container *corev1.Container, derived int) (int, e return derived, nil } -// setContainerEnv updates variables in order while optionally preserving the first existing value. -func setContainerEnv(container *corev1.Container, preserveExisting bool, variables ...corev1.EnvVar) { +// setContainerEnv updates the first matching variable or appends a new one. +func setContainerEnv(container *corev1.Container, variables ...corev1.EnvVar) { variables: for _, variable := range variables { for index := range container.Env { if container.Env[index].Name == variable.Name { - if !preserveExisting { - container.Env[index] = variable - } + container.Env[index] = variable continue variables } } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go index 48c6f470b0ef..e02514f8d6bc 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go @@ -12,54 +12,6 @@ import ( corev1 "k8s.io/api/core/v1" ) -func TestWrapCyborgDecodeForSwaBatchSkipsBatchOne(t *testing.T) { - t.Parallel() - - t.Log("Construct a single-item Cyborg container entrypoint") - container := &corev1.Container{ - Command: []string{"/usr/local/bin/dynamo_main"}, - Args: []string{"--model", "test"}, - } - original := container.DeepCopy() - - t.Log("Apply the batch-one runtime contract") - wrapCyborgStartup(container, 1, "") - - t.Log("Verify batch one leaves the container unchanged") - require.Equal(t, original, container) -} - -func TestWrapCyborgStartupPreservesArgumentVector(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - batchSize int - configFile string - wantFlags []string - }{ - {"hosts", 1, "lpu_servers", []string{"--expand-hosts", "--"}}, - {"batch", 2, "", []string{"--swa-batch-ids", "--"}}, - {"hosts and batch", 2, "lpu_servers", []string{"--expand-hosts", "--swa-batch-ids", "--"}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - t.Log("Select image helper options while preserving a caller-owned shell and literal arguments") - command := []string{"/bin/sh", "-c"} - args := []string{`printf '%s\n' "$@"`, "--", "argument with spaces", "literal '$HOME'", ""} - container := &corev1.Container{Command: command, Args: args} - want := append(append(append([]string{}, test.wantFlags...), command...), args...) - wrapCyborgStartup(container, test.batchSize, test.configFile) - - t.Log("Keep the wrapped invocation independent of subsequent caller slice mutations") - command[0], args[2] = "mutated-command", "mutated-argument" - require.Equal(t, []string{"/usr/local/bin/cyborg-entrypoint"}, container.Command) - require.Equal(t, want, container.Args) - }) - } -} - func TestConfiguredCyborgBatchSizeRejectsInvalidEnvironment(t *testing.T) { t.Log("Define invalid literal and field-sourced Cyborg batch-size variables") tests := []corev1.EnvVar{ @@ -73,12 +25,3 @@ func TestConfiguredCyborgBatchSizeRejectsInvalidEnvironment(t *testing.T) { require.Error(t, err) } } - -func TestPinnedCyborgLauncher(t *testing.T) { - t.Log("Preserve the linked deployment's exact image and complete bash argument") - container := &corev1.Container{Image: pinnedCyborgImage, Command: []string{"/bin/bash", "-lc"}, Args: []string{pinnedCyborgLauncher}} - wrapCyborgStartup(container, 2, "lpu_servers") - require.Equal(t, pinnedCyborgImage, container.Image) - require.Equal(t, []string{"/bin/bash", "-lc", pinnedCyborgLauncher}, container.Args[3:]) - require.Equal(t, []string{"--expand-hosts", "--swa-batch-ids", "--"}, container.Args[:3]) -} diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 0ce8eea46352..3d2e6c9f209c 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -18,9 +18,8 @@ import ( ) const ( - selectedCyborgServerHostsFileEnv = "SERVER_HOSTS_FILE" - selectedCyborgTokenizerDirEnv = "TOKENIZER_DIR" - selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" + selectedCyborgTokenizerDirEnv = "TOKENIZER_DIR" + selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" ) // ApplySelectedCyborgContainerDefaults installs the operator-owned V2 Cyborg @@ -68,10 +67,6 @@ func ApplySelectedCyborgContainerDefaults( if err := applyCyborgManifestPath(container, projection, modelStoragePath); err != nil { return err } - container.Env = append(container.Env, corev1.EnvVar{ - Name: selectedCyborgServerHostsFileEnv, - Value: runtimeTemporaryStorageMountPath + "/lpu_servers", - }) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_commands.go b/deploy/operator/internal/dynamo/lpx/lpu_commands.go deleted file mode 100644 index 4ab2488b66b0..000000000000 --- a/deploy/operator/internal/dynamo/lpx/lpu_commands.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - // LPUSSHPort is the SSH port used by Nova and LPU Agent workers. - LPUSSHPort = commonconsts.MpiRunSshPort - // LPUReadinessPort is the HTTP readiness port used by the LPU runtime. - LPUReadinessPort = 19878 - lpuRDMAPort = 19877 -) - -const ( - lpuV2HTTPStartupFailureThreshold int32 = 60 - lpuV2ExecStartupFailureThreshold int32 = 240 -) - -func lpuV2StartupProbe(useHTTP bool) *corev1.Probe { - probe := lpuV2ReadinessProbe(useHTTP) - if useHTTP { - probe.FailureThreshold = lpuV2HTTPStartupFailureThreshold - } else { - probe.FailureThreshold = lpuV2ExecStartupFailureThreshold - } - return probe -} - -func lpuV2ReadinessProbe(useHTTP bool) *corev1.Probe { - probe := &corev1.Probe{ - FailureThreshold: 6, - PeriodSeconds: 5, - SuccessThreshold: 1, - TimeoutSeconds: 1, - } - if useHTTP { - probe.HTTPGet = &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromInt(LPUReadinessPort), - Scheme: corev1.URISchemeHTTP, - } - } else { - probe.Exec = &corev1.ExecAction{ - Command: []string{"/bin/hydra-entrypoint", "ready"}, - } - } - return probe -} - -func lpuSSHProbe() *corev1.Probe { - return &corev1.Probe{ - FailureThreshold: 3, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - SuccessThreshold: 1, - TimeoutSeconds: 1, - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(LPUSSHPort), - }, - }, - } -} diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index 6bc4b6cd9914..84149cc3cf21 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -22,6 +22,7 @@ const ( v3LPUResourceName corev1.ResourceName = "nvidia.com/lpu" lpuModelNameEnvVar = "LPU_MODEL_NAME" + allocationEnvVar = "LPX_ALLOCATION" ) // configureAgentScheduling consumes a fresh, nonnil Agent PodSpec containing main. diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 4df87480cdef..0ebd58e7cb22 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" ) @@ -154,780 +153,100 @@ func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { return lists } -func TestStripNovaOnlyArgs(t *testing.T) { - t.Parallel() - - t.Log("Define mixed Nova-leader and worker argument shapes") - for _, test := range []struct { - name string - args []string - want []string - wantNovaOnly bool - }{ - { - name: "worker args only", - args: []string{"custom-agent-command"}, - want: []string{"custom-agent-command"}, - }, - { - name: "split leader arg", - args: []string{"--instance-model-name", "custom-model-name", "custom-agent-command"}, - want: []string{"custom-agent-command"}, - }, - { - name: "equals leader arg", - args: []string{"--instance-model-name=custom-model-name", "custom-agent-command"}, - want: []string{"custom-agent-command"}, - }, - { - name: "leader args only", - args: []string{"--instance-model-name", "custom-model-name"}, - want: []string{}, wantNovaOnly: true, - }, - { - name: "Nova environment args", - args: []string{"--agent-env-vars", "FIRST=1", "--agent-env-vars=SECOND=2", "custom-agent-command"}, - want: []string{"custom-agent-command"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - t.Log("Strip Nova-only arguments from a copied input") - args := append([]string(nil), test.args...) - got, novaOnly := stripNovaOnlyArgs(args) - - t.Log("Verify worker arguments, classification, and input immutability") - require.Equal(t, test.want, got) - require.Equal(t, test.wantNovaOnly, novaOnly) - require.Equal(t, test.args, args) - }) - } -} - -func TestConfigureNodeLocalLPURuntimeRoles(t *testing.T) { - t.Log("Start from the supported image-only V3 PodSpec") - base := corev1.PodSpec{ - HostUsers: ptr.To(false), - Containers: []corev1.Container{{ - Name: commonconsts.MainContainerName, - Image: "runtime:latest", - Args: []string{"--instance-model-name", "test/model", "--agent-env-vars", "PIPELINE_ENV=kept"}, - VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", ReadOnly: true}}, - Env: []corev1.EnvVar{ - {Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, - {Name: "DYN_KUBE_DISCOVERY_MODE", Value: "container"}, - {Name: "GROQ_V2_RESET_ON_OPEN", Value: "custom"}, - { - Name: "MAIN_CPU", - ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: commonconsts.MainContainerName, - Resource: "limits.cpu", - }}, - }, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, - Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, - }, - LivenessProbe: testExecProbe("check-live"), - StartupProbe: testExecProbe("check-started"), - Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"stop-main"}}}, - }, - }}, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{}, - }, - }, - { - Name: "resources", - VolumeSource: corev1.VolumeSource{ +func TestConfigureNodeLocalRuntimeBindings(t *testing.T) { + for _, role := range []string{"conductor", "agent"} { + t.Run(role, func(t *testing.T) { + t.Log("Author main-container references and HX resource requirements") + pod := corev1.PodSpec{ + HostUsers: ptr.To(false), + Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, + Env: []corev1.EnvVar{ + {Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, + {Name: "MAIN_CPU", ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, Resource: "limits.cpu", + }}}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }}, + Volumes: []corev1.Volume{{Name: "resources", VolumeSource: corev1.VolumeSource{ DownwardAPI: &corev1.DownwardAPIVolumeSource{Items: []corev1.DownwardAPIVolumeFile{{ - Path: "main-memory", - ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: commonconsts.MainContainerName, - Resource: "limits.memory", + Path: "main-memory", ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, Resource: "limits.memory", }, }}}, - }, - }, - {Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "read-only-tmp"}}}, - {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, - {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, - {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret", DefaultMode: ptr.To(int32(0644))}}}, - }, - } - base.Containers[0].VolumeMounts = append(base.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) - conductor := *base.DeepCopy() - agent := *base.DeepCopy() - - t.Log("Lower the image into the Nova conductor and privileged SSH worker roles") - require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret"), "writable storage") - require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), "writable storage") - conductor = *base.DeepCopy() - agent = *base.DeepCopy() - conductor.Containers[0].VolumeMounts[0].ReadOnly = false - conductor.Volumes[2].VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} - conductor.Volumes[5].Secret.DefaultMode = ptr.To(int32(0600)) - conductor.Volumes = append(conductor.Volumes, corev1.Volume{Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) - conductor.Containers[0].VolumeMounts = append(conductor.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir, ReadOnly: true, - }) - agent.Containers[0].VolumeMounts[0].ReadOnly = false - agent.Volumes[2].VolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} - agent.Containers[0].VolumeMounts = append(agent.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: "host-dev", MountPath: "/dev"}, - corev1.VolumeMount{Name: "host-sys", MountPath: "/sys"}, - corev1.VolumeMount{Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, - ) - conductorMounts := slices.Clone(conductor.Containers[0].VolumeMounts) - agentMounts := slices.Clone(agent.Containers[0].VolumeMounts) - require.NoError(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret")) - require.NoError(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret")) - - t.Log("Verify the conductor starts Nova with immutable placement and source behavior") - require.Len(t, conductor.Containers, 1) - conductorContainer := conductor.Containers[0] - require.Equal(t, conductorMounts, conductorContainer.VolumeMounts) - require.Equal(t, "conductor", conductorContainer.Name) - require.Equal(t, []string{"/bin/nova"}, conductorContainer.Command) - requireFlagValue(t, conductorContainer.Args, "--datacenter-config-filepath", "/configs/datacenter.toml") - require.Contains(t, conductorContainer.Args, "--instance-model-name") - requireFlagValue(t, conductorContainer.Args, "--allocation", "agt") - require.GreaterOrEqual(t, len(conductorContainer.Args), len(nodeLocalHXAgentEnvironmentArgs)) - staticSuffixIndex := len(conductorContainer.Args) - len(nodeLocalHXAgentEnvironmentArgs) - require.Equal(t, nodeLocalHXAgentEnvironmentArgs, conductorContainer.Args[staticSuffixIndex:]) - pipelineEnvIndex := slices.Index(conductorContainer.Args, "PIPELINE_ENV=kept") - require.GreaterOrEqual(t, pipelineEnvIndex, 0) - require.Less(t, pipelineEnvIndex, staticSuffixIndex) - require.Equal(t, "1", testContainerEnvValue(conductorContainer.Env, "OMPI_ALLOW_RUN_AS_ROOT")) - require.Equal(t, "conductor", testContainerEnvValue(conductorContainer.Env, "CONTAINER_NAME")) - require.Equal(t, "conductor", testContainerEnvSource(conductorContainer.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) - require.Equal(t, "conductor", conductor.Volumes[1].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) - require.Equal(t, base.Containers[0].LivenessProbe, conductorContainer.LivenessProbe) - require.Equal(t, base.Containers[0].StartupProbe, conductorContainer.StartupProbe) - require.Equal(t, base.Containers[0].Lifecycle, conductorContainer.Lifecycle) - require.False(t, *conductor.HostUsers) - require.NotNil(t, testContainerEnvSource(conductorContainer.Env, "POD_IP")) - require.Contains(t, conductorContainer.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) - require.NotNil(t, testVolumeByName(t, conductor.Volumes, "tmp").EmptyDir) - require.Contains(t, testVolumeMountPaths(conductorContainer.VolumeMounts), conductorSSHKeyDir) - require.Len(t, conductor.InitContainers, 1) - require.Equal(t, conductorSSHKeyInitContainerName, conductor.InitContainers[0].Name) - require.Equal(t, int32(0600), *testVolumeByName(t, conductor.Volumes, runtimeSSHVolumeName).Secret.DefaultMode) - require.Equal(t, "ssh-secret", testVolumeByName(t, conductor.Volumes, runtimeSSHVolumeName).Secret.SecretName) - - t.Log("Verify the Agent hosts SSHD with local devices and the source resource profile") - require.Len(t, agent.Containers, 1) - agentContainer := agent.Containers[0] - require.Equal(t, agentMounts, agentContainer.VolumeMounts) - require.Equal(t, lpuAgentContainerName, agentContainer.Name) - require.Equal(t, []string{"/bin/quasar-entrypoint"}, agentContainer.Command) - require.Empty(t, agentContainer.Args) - require.Empty(t, testContainerEnvValue(agentContainer.Env, "LPU_RUN_SYSTEM_INIT")) - require.Equal(t, "cvp", testContainerEnvValue(agentContainer.Env, "LPU_FPGA_RECONFIGURE_METHOD")) - require.Equal(t, "custom", testContainerEnvValue(agentContainer.Env, "GROQ_V2_RESET_ON_OPEN")) - require.Equal(t, "0", testContainerEnvValue(agentContainer.Env, "GROQ_PG613_SKIP_HOST_LINK_PROFILE")) - require.Equal(t, lpuAgentContainerName, testContainerEnvValue(agentContainer.Env, "CONTAINER_NAME")) - require.Equal(t, lpuAgentContainerName, testContainerEnvSource(agentContainer.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) - require.Equal(t, lpuAgentContainerName, agent.Volumes[1].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) - require.Equal(t, &corev1.SecurityContext{ - Privileged: ptr.To(true), - RunAsUser: ptr.To(int64(0)), - RunAsGroup: ptr.To(int64(0)), - RunAsNonRoot: ptr.To(false), - }, agentContainer.SecurityContext) - require.False(t, agentContainer.TTY) - require.False(t, agentContainer.Stdin) - require.Nil(t, agentContainer.LivenessProbe) - require.Nil(t, agentContainer.StartupProbe) - require.Nil(t, agentContainer.Lifecycle) - require.Nil(t, agent.HostUsers) - require.True(t, agent.HostIPC) - require.True(t, agent.HostNetwork) - require.Equal(t, corev1.DNSClusterFirstWithHostNet, agent.DNSPolicy) - require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/dev") - require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/sys") - require.Contains(t, testVolumeMountPaths(agentContainer.VolumeMounts), "/ssh-pk") - require.Contains(t, agentContainer.VolumeMounts, corev1.VolumeMount{Name: "tmp", MountPath: "/tmp"}) - require.NotNil(t, testVolumeByName(t, agent.Volumes, "tmp").EmptyDir) - require.Equal(t, resource.MustParse("2"), agentContainer.Resources.Requests[corev1.ResourceCPU]) - require.Equal(t, resource.MustParse("4Gi"), agentContainer.Resources.Limits[corev1.ResourceMemory]) - require.NotContains(t, testVolumeNames(agent.Volumes), "hugepages") - require.Equal(t, int32(0644), *testVolumeByName(t, agent.Volumes, runtimeSSHVolumeName).Secret.DefaultMode) - require.Len(t, agent.InitContainers, 1) - initContainer := agent.InitContainers[0] - require.Equal(t, lp30InitContainerName, initContainer.Name) - require.Equal(t, "runtime:latest", initContainer.Image) - require.Equal(t, []string{lp30InitPath}, initContainer.Command) - require.True(t, *initContainer.SecurityContext.Privileged) - require.Equal(t, "spec.nodeName", testContainerEnvSource(initContainer.Env, "NODE_NAME").FieldRef.FieldPath) - require.Contains(t, testVolumeMountPaths(initContainer.VolumeMounts), "/dev") - require.Contains(t, testVolumeMountPaths(initContainer.VolumeMounts), "/sys") -} - -func TestConfigureNodeLocalXTConductorSSHInitUsesMainImage(t *testing.T) { - t.Log("Place an unrelated sidecar before the XT conductor main container") - podSpec := renderTestPodSpec() - podSpec.Containers[0].Image = "lpu-runtime" - podSpec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent - podSpec.Containers = append([]corev1.Container{{Name: "metrics", Image: "metrics-sidecar", ImagePullPolicy: corev1.PullAlways}}, podSpec.Containers...) - - t.Log("Configure the conductor without deriving its runtime identity from list position") - require.NoError(t, configureNodeLocalConductorRuntime(&podSpec, BuildFamilyXT, "agt", "ssh-secret")) - - t.Log("Run SSH initialization with the resolved conductor image and preserve the sidecar") - require.Equal(t, "metrics", podSpec.Containers[0].Name) - require.Equal(t, "metrics-sidecar", podSpec.Containers[0].Image) - require.Equal(t, "conductor", podSpec.Containers[1].Name) - require.Len(t, podSpec.InitContainers, 1) - require.Equal(t, "lpu-runtime", podSpec.InitContainers[0].Image) - require.Equal(t, corev1.PullIfNotPresent, podSpec.InitContainers[0].ImagePullPolicy) -} - -func TestXTSSHSecretNameIsNotUsedAsVolumeName(t *testing.T) { - const sshSecretName = "mpi.ssh" - - t.Log("Configure each XT runtime role with a valid dotted SSH Secret name") - base := renderTestPodSpec() - testVolumeByName(t, base.Volumes, runtimeSSHVolumeName).Secret.SecretName = sshSecretName - configureAgentScheduling(&base, BuildFamilyXT) - direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() - require.NoError(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", sshSecretName, false)) - require.NoError(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, sshSecretName)) - require.NoError(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "agt", sshSecretName)) - - for _, test := range []struct { - name string - pod *corev1.PodSpec - mounts []corev1.VolumeMount - }{ - {"direct hybrid agent", direct, direct.Containers[0].VolumeMounts}, - {"node-local agent", agent, agent.Containers[0].VolumeMounts}, - {"node-local conductor", conductor, conductor.InitContainers[0].VolumeMounts}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Keep the configured Secret separate from the fixed volume and mount name") - sshVolume := testVolumeByName(t, test.pod.Volumes, "ssh-secret") - require.Equal(t, sshSecretName, sshVolume.Secret.SecretName) - require.NotContains(t, testVolumeNames(test.pod.Volumes), sshSecretName) - require.Contains(t, test.mounts, corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}) - }) - } -} - -func TestGeneratedAgentLaunchCommands(t *testing.T) { - for _, test := range []struct { - name string - family BuildFamily - direct, multiNode bool - command []string - args []string - }{ - {"XT SSH worker", BuildFamilyXT, false, false, []string{"/bin/quasar-entrypoint"}, []string{"--partition-metadata"}}, - {"HX SSH worker", BuildFamilyHX, false, false, []string{"/bin/quasar-entrypoint"}, nil}, - {"direct single-node", BuildFamilyXT, true, false, []string{"/bin/hydra-entrypoint"}, []string{"start"}}, - {"direct multi-node", BuildFamilyXT, true, true, []string{"/bin/hydra-entrypoint"}, []string{"start"}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Configure a runtime role without an authored startup command") - pod := renderTestPodSpec() - pod.Containers[0].Command, pod.Containers[0].Args = nil, nil - configureAgentScheduling(&pod, test.family) - if test.direct { - require.NoError(t, configureDirectHybridAgentRuntime(&pod, "graph-lpu", "ssh-secret", test.multiNode)) - } else { - require.NoError(t, configureNodeLocalAgentRuntime(&pod, test.family, false, "ssh-secret")) + }}}, } + before := pod.DeepCopy() - t.Log("Execute the installed launcher after all role defaults have been applied") - agent := pod.Containers[0] - require.Equal(t, test.command, agent.Command) - require.Equal(t, test.args, agent.Args) - if test.direct { - require.Equal(t, []string{"/bin/hydra-entrypoint", "ready"}, agent.StartupProbe.Exec.Command) - require.Equal(t, []string{"/bin/hydra-entrypoint", "ready"}, agent.ReadinessProbe.Exec.Command) - require.EqualValues(t, 240, agent.StartupProbe.FailureThreshold) - require.EqualValues(t, 6, agent.ReadinessProbe.FailureThreshold) - require.EqualValues(t, 5, agent.ReadinessProbe.PeriodSeconds) - require.EqualValues(t, 1, agent.ReadinessProbe.TimeoutSeconds) + t.Log("Bind the selected role and retarget every authored main-container reference") + if role == "conductor" { + configureNodeLocalConductorRuntime(&pod, BuildFamilyHX, "agt") } else { - require.Equal(t, intstr.FromInt(LPUSSHPort), agent.ReadinessProbe.TCPSocket.Port) - } - }) - } -} - -func TestGeneratedAgentSSHMountRequirements(t *testing.T) { - for _, runtime := range []struct { - name string - family BuildFamily - direct, multiNode bool - }{ - {"XT", BuildFamilyXT, false, false}, - {"HX", BuildFamilyHX, false, false}, - {"direct single-node", BuildFamilyXT, true, false}, - {"direct multi-node", BuildFamilyXT, true, true}, - } { - for _, test := range []struct { - name string - mount corev1.VolumeMount - command []string - args []string - wantErr bool - }{ - {name: "missing", wantErr: true}, - {name: "wrong volume", mount: corev1.VolumeMount{Name: "other", MountPath: "/ssh-pk"}, wantErr: true}, - {name: "misplaced", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/elsewhere"}, wantErr: true}, - {name: "subPath", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", SubPath: "private.key"}, wantErr: true}, - {name: "subPathExpr", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk", SubPathExpr: "$(KEY)"}, wantErr: true}, - {name: "writable", mount: corev1.VolumeMount{Name: "ssh-secret", MountPath: "/ssh-pk"}}, - {name: "read-only", mount: corev1.VolumeMount{Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}}, - {name: "custom command", command: []string{"/custom-agent"}}, - {name: "custom args", args: []string{"custom-agent"}}, - } { - custom := len(test.command) != 0 || len(test.args) != 0 - if runtime.family == BuildFamilyHX && custom { - continue + configureNodeLocalAgentRuntime(&pod, BuildFamilyHX) } - t.Run(runtime.name+"/"+test.name, func(t *testing.T) { - t.Log("Author the Agent startup and selected key mount on a fresh runtime fixture") - pod := renderTestPodSpec() - main := &pod.Containers[0] - main.Command, main.Args = test.command, test.args - main.Resources = corev1.ResourceRequirements{} - main.SecurityContext = &corev1.SecurityContext{ReadOnlyRootFilesystem: ptr.To(true)} - main.VolumeMounts = nil - if test.mount.Name != "" { - main.VolumeMounts = append(main.VolumeMounts, test.mount) - } - if runtime.family == BuildFamilyXT && !custom { - main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) - } - before := pod.DeepCopy() - configureAgentScheduling(&pod, runtime.family) - - t.Log("Require SSH only for generated startup that consumes the key directory") - var err error - if runtime.direct { - err = configureDirectHybridAgentRuntime(&pod, "graph-lpu", "ssh-secret", runtime.multiNode) - } else { - err = configureNodeLocalAgentRuntime(&pod, runtime.family, false, "ssh-secret") - } - if test.wantErr && (!runtime.direct || runtime.multiNode) { - require.ErrorContains(t, err, `requires volume "ssh-secret" mounted at "/ssh-pk" without subPath`) - } else { - require.NoError(t, err) - require.False(t, ptr.Deref(pod.Containers[0].SecurityContext.ReadOnlyRootFilesystem, false)) - } - require.Equal(t, before.Containers[0].VolumeMounts, pod.Containers[0].VolumeMounts) - require.Equal(t, before.Volumes, pod.Volumes) - if custom { - require.Equal(t, test.command, pod.Containers[0].Command) - require.Equal(t, test.args, pod.Containers[0].Args) - } - }) - } - } -} - -func TestGeneratedAgentConfigMountOverrides(t *testing.T) { - for _, test := range []struct { - name string - env []corev1.EnvVar - wantErr bool - }{ - {name: "default", wantErr: true}, - {name: "empty", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR"}}, wantErr: true}, - {name: "canonical", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", Value: "/configs"}}, wantErr: true}, - {name: "custom", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", Value: "/custom"}}}, - {name: "ValueFrom", env: []corev1.EnvVar{{Name: "LPU_CONFIG_DIR", ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "custom"}, Key: "config-dir"}, - }}}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Require /configs unless an explicit variable overrides it; EnvFrom alone is insufficient") - container := corev1.Container{Env: test.env, EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated"}, - }}}} - before := container.DeepCopy() - err := validateGeneratedAgentMounts(&container, true, false) - if test.wantErr { - require.ErrorContains(t, err, `requires a volume mounted at "/configs"`) + container := pod.Containers[0] + require.Equal(t, role, container.Name) + require.Equal(t, role, testContainerEnvValue(container.Env, "CONTAINER_NAME")) + require.Equal(t, role, testContainerEnvSource(container.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) + require.Equal(t, role, pod.Volumes[0].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) + require.Equal(t, "status.podIP", testContainerEnvSource(container.Env, "POD_IP").FieldRef.FieldPath) + require.Equal(t, before.Containers[0].Resources, container.Resources) + + t.Log("Apply worker networking only to the Agent and placement only to the conductor") + if role == "conductor" { + require.Equal(t, "agt", testContainerEnvValue(container.Env, "LPX_ALLOCATION")) + require.Equal(t, before.HostUsers, pod.HostUsers) + require.False(t, pod.HostIPC) + require.False(t, pod.HostNetwork) } else { - require.NoError(t, err) + require.Nil(t, pod.HostUsers) + require.True(t, pod.HostIPC) + require.True(t, pod.HostNetwork) + require.Equal(t, corev1.DNSClusterFirstWithHostNet, pod.DNSPolicy) } - require.Equal(t, *before, container) }) } } -func TestHostDeviceVolumeValidationIsLimitedToLP30Init(t *testing.T) { - for _, test := range []struct { - name string - volume corev1.Volume - wantErr bool - }{ - {"ConfigMap", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "application"}}, - }}, true}, - {"Secret", corev1.Volume{Name: "host-sys", VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: "application"}, - }}, true}, - {"wrong host directory", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/application"}, - }}, true}, - {"wrong host type", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathFile)}, - }}, true}, - {"compatible host directory", corev1.Volume{Name: "host-dev", VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}, - }}, false}, - {"compatible host type", corev1.Volume{Name: "host-sys", VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: ptr.To(corev1.HostPathDirectoryOrCreate)}, - }}, false}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Give an authored sidecar a volume name used by LP30 initialization") - base := corev1.PodSpec{ - Containers: []corev1.Container{ - {Name: commonconsts.MainContainerName, Image: "lpu-runtime"}, - {Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: test.volume.Name, MountPath: "/application"}}}, - }, - Volumes: []corev1.Volume{ - test.volume, - {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, - }, - } - base.Containers[0].VolumeMounts = []corev1.VolumeMount{ - {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, - {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, - } - configureAgentScheduling(&base, BuildFamilyXT) - direct, agent := base.DeepCopy(), base.DeepCopy() - - t.Log("Leave authored device bindings to the DGD creator in both XT runtimes") - errors := []error{ - configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret", false), - configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), - } - for _, err := range errors { - require.NoError(t, err) - } - - t.Log("Never rebind the authored volume or its sidecar mount") - for _, pod := range []*corev1.PodSpec{direct, agent} { - require.Equal(t, base.Volumes, pod.Volumes) - require.Equal(t, test.volume.VolumeSource, testVolumeByName(t, pod.Volumes, test.volume.Name)) - require.Equal(t, base.Containers[1], pod.Containers[1]) - } - - t.Log("Validate only the host volumes mounted by the generated LP30 init") - lp30 := renderTestPodSpec() - index := slices.IndexFunc(lp30.Volumes, func(volume corev1.Volume) bool { return volume.Name == test.volume.Name }) - lp30.Volumes[index] = test.volume - before := lp30.DeepCopy() - err := addLP30InitContainer(&lp30, &lp30.Containers[0]) - if test.wantErr { - require.ErrorContains(t, err, "must use directory hostPath") - require.Equal(t, *before, lp30) - } else { - require.NoError(t, err) - require.Equal(t, before.Volumes, lp30.Volumes) - require.Equal(t, before.Containers, lp30.Containers) - } - }) - } - - for _, name := range []string{"host-dev", "host-sys"} { - t.Run("LP30 missing "+name, func(t *testing.T) { - t.Log("Reject a missing generated init binding without changing the Pod") - pod := renderTestPodSpec() - pod.Volumes = slices.DeleteFunc(pod.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }) - before := pod.DeepCopy() - err := addLP30InitContainer(&pod, &pod.Containers[0]) - require.ErrorContains(t, err, "requires podTemplate volume") - require.ErrorContains(t, err, name) - require.Equal(t, *before, pod) - }) - } -} - -func TestXTRuntimesValidateTemporaryStorage(t *testing.T) { +func TestConfigureDirectHybridAgentRuntimePreservesPodSecurity(t *testing.T) { for _, test := range []struct { name string - source corev1.VolumeSource - readOnly bool - wantErr bool - }{ - {"emptyDir", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, false, false}, - {"read-only mount", corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, true, true}, - {"hostPath", corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/scratch"}}, false, false}, - {"PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "scratch"}}, false, false}, - {"CSI", corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{Driver: "storage.example.com"}}, false, false}, - {"AzureDisk", corev1.VolumeSource{AzureDisk: &corev1.AzureDiskVolumeSource{DiskName: "scratch", DataDiskURI: "disk-uri"}}, false, false}, - {"ConfigMap", corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}, false, true}, - {"Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{}}, false, true}, - {"DownwardAPI", corev1.VolumeSource{DownwardAPI: &corev1.DownwardAPIVolumeSource{}}, false, true}, - {"Projected", corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{}}, false, true}, - {"Image", corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "runtime:latest"}}, false, true}, - {"read-only PVC", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ReadOnly: true}}, false, true}, - {"read-only NFS", corev1.VolumeSource{NFS: &corev1.NFSVolumeSource{ReadOnly: true}}, false, true}, - {"read-only CSI", corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{ReadOnly: ptr.To(true)}}, false, true}, - {"read-only ISCSI", corev1.VolumeSource{ISCSI: &corev1.ISCSIVolumeSource{ReadOnly: true}}, false, true}, - {"read-only FC", corev1.VolumeSource{FC: &corev1.FCVolumeSource{ReadOnly: true}}, false, true}, - {"read-only FlexVolume", corev1.VolumeSource{FlexVolume: &corev1.FlexVolumeSource{ReadOnly: true}}, false, true}, - {"read-only GCE", corev1.VolumeSource{GCEPersistentDisk: &corev1.GCEPersistentDiskVolumeSource{ReadOnly: true}}, false, true}, - {"read-only AWS", corev1.VolumeSource{AWSElasticBlockStore: &corev1.AWSElasticBlockStoreVolumeSource{ReadOnly: true}}, false, true}, - {"read-only Cinder", corev1.VolumeSource{Cinder: &corev1.CinderVolumeSource{ReadOnly: true}}, false, true}, - {"read-only AzureFile", corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{ReadOnly: true}}, false, true}, - {"read-only AzureDisk", corev1.VolumeSource{AzureDisk: &corev1.AzureDiskVolumeSource{ReadOnly: ptr.To(true)}}, false, true}, - {"read-only Portworx", corev1.VolumeSource{PortworxVolume: &corev1.PortworxVolumeSource{ReadOnly: true}}, false, true}, - } { - for _, name := range []string{"tmp", "scratch"} { - t.Run(test.name+"/"+name, func(t *testing.T) { - t.Log("Use either a named tmp volume or an existing /tmp mount") - base := corev1.PodSpec{ - Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, - Volumes: []corev1.Volume{ - {Name: name, VolumeSource: test.source}, - {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, - }, - } - base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: name, MountPath: "/tmp", ReadOnly: test.readOnly}, {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}} - if name != "tmp" { - base.Volumes = append(base.Volumes, corev1.Volume{Name: "tmp", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "unused"}}}) - } - base.Containers[0].VolumeMounts = append(base.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) - configureAgentScheduling(&base, BuildFamilyXT) - direct, agent := base.DeepCopy(), base.DeepCopy() - - t.Log("Require writable temporary storage through both XT runtime entry points") - for _, err := range []error{ - configureDirectHybridAgentRuntime(direct, "graph-lpu", "ssh-secret", false), - configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "ssh-secret"), - } { - if test.wantErr { - require.ErrorContains(t, err, "requires writable storage") - } else { - require.NoError(t, err) - } - } - - t.Log("Preserve authored storage and the selected mount") - for _, pod := range []*corev1.PodSpec{direct, agent} { - require.Equal(t, base.Containers[0].VolumeMounts, pod.Containers[0].VolumeMounts) - require.Equal(t, base.Volumes, pod.Volumes) - if !test.wantErr { - require.Contains(t, pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: name, MountPath: "/tmp"}) - } - } - }) - } - } -} - -func TestRuntimeTemporaryStorageBindings(t *testing.T) { - for _, test := range []struct { - name string - mounts []corev1.VolumeMount - err string + security *corev1.PodSecurityContext }{ - {name: "container-local storage"}, - {name: "missing tmp source", mounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, err: `references missing volume "tmp"`}, - {name: "missing custom source", mounts: []corev1.VolumeMount{{Name: "scratch", MountPath: "/tmp"}}, err: `references missing volume "scratch"`}, + {name: "omitted"}, + {name: "authored", security: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(1000)), + RunAsGroup: ptr.To(int64(1000)), + RunAsNonRoot: ptr.To(true), + FSGroup: ptr.To(int64(1000)), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + SupplementalGroups: []int64{2000}, + Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }}, } { t.Run(test.name, func(t *testing.T) { - t.Log("Use container-local storage or validate the authored mount without inserting storage") - pod := corev1.PodSpec{Containers: []corev1.Container{{VolumeMounts: test.mounts}}} - before := pod.DeepCopy() - err := validateRuntimeTemporaryStorage(&pod, &pod.Containers[0]) - if test.err != "" { - require.ErrorContains(t, err, test.err) - } else { - require.NoError(t, err) + t.Log("Configure a direct worker with the authored Pod security context") + pod := corev1.PodSpec{ + Containers: []corev1.Container{{Name: commonconsts.MainContainerName}}, + SecurityContext: test.security.DeepCopy(), } - require.Equal(t, *before, pod) - }) - } -} + configureAgentScheduling(&pod, BuildFamilyXT) + configureDirectHybridAgentRuntime(&pod, "graph-lpu") -func TestConfigureDirectHybridAgentRuntimePreservesPodOverrides(t *testing.T) { - t.Log("Define custom entrypoint, probe and Pod security preservation cases") - customStartup := testExecProbe("custom-startup") - customReadiness := testExecProbe("custom-ready") - tests := []struct { - name string - container corev1.Container - podSecurity *corev1.PodSecurityContext - wantCommand []string - wantArgs []string - wantStartup *corev1.Probe - wantReadiness *corev1.Probe - }{ - { - name: "args", - container: corev1.Container{Name: commonconsts.MainContainerName, Args: []string{"custom-agent"}}, - wantCommand: nil, - wantArgs: []string{"custom-agent"}, - wantStartup: lpuV2StartupProbe(true), - wantReadiness: lpuV2ReadinessProbe(true), - }, - { - name: "command", - container: corev1.Container{Name: commonconsts.MainContainerName, Command: []string{"custom-entrypoint"}}, - podSecurity: &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(1000)), - RunAsGroup: ptr.To(int64(1000)), - RunAsNonRoot: ptr.To(true), - FSGroup: ptr.To(int64(1000)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), - SupplementalGroups: []int64{2000}, - Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, - SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, - }, - wantCommand: []string{"custom-entrypoint"}, - wantStartup: lpuV2StartupProbe(true), - wantReadiness: lpuV2ReadinessProbe(true), - }, - { - name: "user probes", - container: corev1.Container{ - Name: commonconsts.MainContainerName, - Args: []string{"custom-agent"}, - StartupProbe: customStartup, - ReadinessProbe: customReadiness, - }, - wantCommand: nil, - wantArgs: []string{"custom-agent"}, - wantStartup: customStartup, - wantReadiness: customReadiness, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Configure the direct LPU agent runtime from a user-owned entrypoint") - podSpec := corev1.PodSpec{ - Containers: []corev1.Container{*test.container.DeepCopy()}, - SecurityContext: test.podSecurity.DeepCopy(), - Volumes: []corev1.Volume{ - {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, - }, - } - podSpec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", SubPath: "workspace"}} - authoredVolumes := slices.Clone(podSpec.Volumes) - configureAgentScheduling(&podSpec, BuildFamilyXT) - err := configureDirectHybridAgentRuntime(&podSpec, "graph-lpu", "ssh-secret", false) - require.NoError(t, err) - - t.Log("Preserve the entrypoint and user probes while supplying defaults when absent") - agent := podSpec.Containers[0] - require.Equal(t, test.wantCommand, agent.Command) - require.Equal(t, test.wantArgs, agent.Args) - require.Equal(t, test.wantStartup, agent.StartupProbe) - require.Equal(t, test.wantReadiness, agent.ReadinessProbe) - require.Equal(t, []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp", SubPath: "workspace"}}, agent.VolumeMounts) - require.Equal(t, authoredVolumes, podSpec.Volumes) - - t.Log("Preserve orthogonal Pod security settings while using the root runtime identity") - wantSecurity := test.podSecurity.DeepCopy() - if wantSecurity == nil { - wantSecurity = &corev1.PodSecurityContext{} + t.Log("Change the worker identity to root while preserving every other security field") + want := test.security.DeepCopy() + if want == nil { + want = &corev1.PodSecurityContext{} } - wantSecurity.RunAsUser = ptr.To(int64(0)) - wantSecurity.RunAsGroup = ptr.To(int64(0)) - wantSecurity.RunAsNonRoot = ptr.To(false) - require.Equal(t, wantSecurity, podSpec.SecurityContext) - }) - } -} - -func TestLPXInitContainerNames(t *testing.T) { - t.Log("Reserve a generated name only in the Pod receiving that init container") - for _, test := range []struct { - name string - add func(*corev1.PodSpec, *corev1.Container) error - }{ - {name: "prepare-lp30", add: addLP30InitContainer}, - {name: "prepare-ssh-key", add: func(spec *corev1.PodSpec, main *corev1.Container) error { - return addConductorSSHKey(spec, main, "ssh-secret") - }}, - } { - for _, name := range []string{"prepare-lp30", "prepare-ssh-key"} { - for _, list := range []string{"containers", "initContainers"} { - t.Run(test.name+"/"+name+"/"+list, func(t *testing.T) { - t.Log("Author an independent container with an image and command to preserve") - spec := renderTestPodSpec() - authored := corev1.Container{Name: name, Image: "custom", Command: []string{"/custom-init"}} - if list == "containers" { - spec.Containers = append(spec.Containers, authored) - } else { - spec.InitContainers = []corev1.Container{authored} - } - before := spec.DeepCopy() - - t.Log("Reject only the actual generated name, before any mutation") - err := test.add(&spec, &spec.Containers[0]) - if name == test.name { - require.ErrorContains(t, err, list) - require.ErrorContains(t, err, name) - require.Equal(t, *before, spec) - return - } - - t.Log("Preserve a name belonging to the other role and add the intended init") - require.NoError(t, err) - require.Equal(t, before.Containers, spec.Containers) - require.Len(t, spec.InitContainers, len(before.InitContainers)+1) - for i, container := range before.InitContainers { - require.Equal(t, container, spec.InitContainers[i]) - } - require.Equal(t, test.name, spec.InitContainers[len(before.InitContainers)].Name) - }) - } - } - } -} - -func TestConfigureNodeLocalLPURuntimeRolesRejectsUnsupportedShape(t *testing.T) { - t.Log("Define unsupported node-local PodSpec shapes") - tests := []struct { - name string - mutate func(*corev1.PodSpec) - errorText string - }{ - { - name: "explicit command", - mutate: func(spec *corev1.PodSpec) { - spec.Containers[0].Command = []string{"/custom"} - }, - errorText: "image-only main container without command", - }, - { - name: "sidecar", - mutate: func(spec *corev1.PodSpec) { - spec.Containers = append(spec.Containers, corev1.Container{Name: "sidecar"}) - }, - errorText: `requires exactly one "main" container`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Reject input outside the supported image-only V3 contract") - base := corev1.PodSpec{Containers: []corev1.Container{{Name: commonconsts.MainContainerName}}} - test.mutate(&base) - conductor := *base.DeepCopy() - agent := *base.DeepCopy() - - require.ErrorContains(t, configureNodeLocalConductorRuntime(&conductor, BuildFamilyHX, "agt", "ssh-secret"), test.errorText) - require.ErrorContains(t, configureNodeLocalAgentRuntime(&agent, BuildFamilyHX, false, "ssh-secret"), test.errorText) + want.RunAsUser = ptr.To(int64(0)) + want.RunAsGroup = ptr.To(int64(0)) + want.RunAsNonRoot = ptr.To(false) + require.Equal(t, want, pod.SecurityContext) }) } } @@ -938,14 +257,6 @@ func testExecProbe(command string) *corev1.Probe { } } -func requireFlagValue(t *testing.T, args []string, flag, value string) { - t.Helper() - index := slices.Index(args, flag) - require.GreaterOrEqual(t, index, 0) - require.Less(t, index+1, len(args)) - require.Equal(t, value, args[index+1]) -} - func testContainerEnvValue(env []corev1.EnvVar, name string) string { index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) if index < 0 { @@ -961,26 +272,3 @@ func testContainerEnvSource(env []corev1.EnvVar, name string) *corev1.EnvVarSour } return env[index].ValueFrom } - -func testVolumeMountPaths(mounts []corev1.VolumeMount) []string { - paths := make([]string, 0, len(mounts)) - for _, mount := range mounts { - paths = append(paths, mount.MountPath) - } - return paths -} - -func testVolumeByName(t *testing.T, volumes []corev1.Volume, name string) corev1.VolumeSource { - t.Helper() - index := slices.IndexFunc(volumes, func(volume corev1.Volume) bool { return volume.Name == name }) - require.GreaterOrEqual(t, index, 0) - return volumes[index].VolumeSource -} - -func testVolumeNames(volumes []corev1.Volume) []string { - names := make([]string, 0, len(volumes)) - for _, volume := range volumes { - names = append(names, volume.Name) - } - return names -} diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 49c3c653c3ee..29cea99c0813 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -40,11 +40,9 @@ type RenderInput struct { // Stages contains an independently merged LPU template for every projected stage. Stages map[string]corev1.PodTemplateSpec // Conductor supplies a fresh, independently merged template for the shared LPU - // conductor. Nil reuses the serving component's agent template. Hybrid pipelines - // do not emit this conductor. + // conductor. Required for non-hybrid pipelines; hybrid pipelines leave it nil + // and supply their independently rendered Cyborg clique in pcs. Conductor *corev1.PodTemplateSpec - // SSHSecretName names the Secret containing runtime SSH credentials. - SSHSecretName string } // RenderSelectedNodeLocal consumes fresh graph and render inputs. pcs, workload, @@ -84,11 +82,15 @@ func RenderSelectedNodeLocal( // The serving component owns conductor metadata and storage independently of model order. conductorStage := projections[len(projections)-1].stage - conductorTemplate := input.Stages[conductorStage] - if input.Conductor != nil { - conductorTemplate = *input.Conductor + conductorTemplate := input.Conductor + storageTemplate := input.Stages[conductorStage] + if !hybrid { + if conductorTemplate == nil { + return nil, fmt.Errorf("LPX rendering requires an explicit conductor template") + } + storageTemplate = *conductorTemplate } - modelStorage, err := lpuModelStorageBinding(conductorTemplate.Spec) + modelStorage, err := lpuModelStorageBinding(storageTemplate.Spec) if err != nil { return nil, err } @@ -116,14 +118,9 @@ func RenderSelectedNodeLocal( configHash := LPUConfigMapHash(configMap) - // Consume an explicit conductor; separate an inherited one before consuming its Agent. + // Consume the independently rendered conductor without copying Agent startup or placement. var conductor *grovev1alpha1.PodCliqueTemplateSpec if conductorTemplateName != "" { - conductorSpec := conductorTemplate.Spec - if input.Conductor == nil { - conductorSpec = *conductorSpec.DeepCopy() - conductorSpec.Affinity = nil - } annotations := roleAnnotations(conductorTemplate.Annotations, lpxv1alpha1.PodRoleConductor, workloadDigest) annotations[commonconsts.AnnotationExtraResourcesHash] = configHash conductor = &grovev1alpha1.PodCliqueTemplateSpec{ @@ -132,7 +129,7 @@ func RenderSelectedNodeLocal( Annotations: annotations, Spec: grovev1alpha1.PodCliqueSpec{ RoleName: conductorTemplateName, - PodSpec: conductorSpec, + PodSpec: conductorTemplate.Spec, Replicas: 1, MinAvailable: ptr.To(int32(1)), StartsAfter: agentTemplateNames, @@ -158,7 +155,7 @@ func RenderSelectedNodeLocal( if stage == conductorStage && conductor != nil { conductorSpec = &conductor.Spec.PodSpec } - if err := configureLPURolePods(&template.Spec, conductorSpec, workload, configMap.Name, allocation, input.SSHSecretName); err != nil { + if err := configureLPURolePods(&template.Spec, conductorSpec, workload, configMap.Name, allocation); err != nil { return nil, fmt.Errorf("stage %s: %w", stage, err) } } @@ -209,7 +206,7 @@ func RenderSelectedNodeLocal( return nil, err } - // HX Cyborg may inherit its Agent's configuration mount. + // Bind the authored HX Cyborg configuration mount to the generated ConfigMap. container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) if cyborgConfigMap == nil && slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == lpuConfigVolumeName }) { @@ -249,7 +246,7 @@ func RenderSelectedNodeLocal( // configureLPURolePods consumes fresh, independently owned Agent and conductor // specs. Agent and workload are nonnil; nil conductor means no emitted launcher. -func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, workload *SelectedWorkload, configMapName, allocation, sshSecretName string) error { +func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, workload *SelectedWorkload, configMapName, allocation string) error { if err := withLPUConfigVolume(agentPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { return err } @@ -264,37 +261,16 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo } if workload.Pipeline() == PipelineSpecDecode || workload.Pipeline() == PipelineLPX { agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) - setContainerEnv(agent, false, corev1.EnvVar{Name: lpuModelNameEnvVar}) + setContainerEnv(agent, corev1.EnvVar{Name: lpuModelNameEnvVar}) } if workload.Pipeline() == PipelineLPX { - // Extra Agents beyond one per runtime partition imply multi-node SSH setup. - projection := workload.modelProjections[0] - partitionCount := len(projection.partitions) - if workload.BuildFamily() == BuildFamilyXT { - partitionCount = len(projection.configuredBuild.Partitions) - } - if err := configureDirectHybridAgentRuntime( - agentPodSpec, - configMapName, - sshSecretName, - projection.agentReplicas > partitionCount, - ); err != nil { - return err - } + configureDirectHybridAgentRuntime(agentPodSpec, configMapName) } else { - // Both node-local roles use the operator-managed key, including an agent-only draft. - if strings.TrimSpace(sshSecretName) == "" { - return fmt.Errorf("node-local LPU runtime requires an MPI SSH secret name") - } if conductorPodSpec != nil { - if err := configureNodeLocalConductorRuntime(conductorPodSpec, workload.BuildFamily(), allocation, sshSecretName); err != nil { - return err - } - } - if err := configureNodeLocalAgentRuntime(agentPodSpec, workload.BuildFamily(), workload.Pipeline() == PipelineSingle, sshSecretName); err != nil { - return err + configureNodeLocalConductorRuntime(conductorPodSpec, workload.BuildFamily(), allocation) } + configureNodeLocalAgentRuntime(agentPodSpec, workload.BuildFamily()) } ensureLPUNodeTolerations(agentPodSpec) diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 5768928b933d..2e85f6bc852a 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -7,7 +7,6 @@ package lpx import ( "encoding/json" - "slices" "strings" "testing" @@ -30,44 +29,6 @@ const ( testTargetStageName = "target" ) -func TestHybridSSHMountFollowsRuntimePartitionSize(t *testing.T) { - for _, test := range []struct { - name string - family BuildFamily - physical int - runtime int - agents int - wantErr bool - }{ - {"XT independent single-node", BuildFamilyXT, 2, 2, 2, false}, - {"XT multi-node", BuildFamilyXT, 1, 1, 2, true}, - {"XT collapsed chain", BuildFamilyXT, 2, 1, 2, true}, - {"HX independent single-node", BuildFamilyHX, 2, 0, 2, false}, - {"HX multi-node", BuildFamilyHX, 1, 0, 2, true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Supply resolved runtime counts and an Agent without an SSH mount") - workload := &SelectedWorkload{modelProjections: []*ModelProjection{{ - pipeline: PipelineLPX, agentReplicas: test.agents, - partitions: make([]BuildPartition, test.physical), - configuredBuild: Build{Family: test.family, Partitions: make([]BuildPartition, test.runtime)}, - }}} - pod := renderTestPodSpec() - pod.Containers[0].VolumeMounts = slices.DeleteFunc(pod.Containers[0].VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == runtimeSSHVolumeName - }) - - t.Log("Require SSH only when a runtime partition spans multiple Agents") - err := configureLPURolePods(&pod, nil, workload, "config", "allocation", "ssh-secret") - if test.wantErr { - require.ErrorContains(t, err, `requires volume "ssh-secret" mounted at "/ssh-pk"`) - } else { - require.NoError(t, err) - } - }) - } -} - func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*ModelProjection, input RenderInput) (*grovev1alpha1.PodCliqueSet, error) { // Attach test projections to their authored stage without inventing templates. for _, projection := range projections { @@ -148,8 +109,8 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Log("Render into the fresh PCS without retaining stale runtime identity") rendered, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, - SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: template}, + Conductor: template.DeepCopy(), }) require.NoError(t, err) require.Same(t, pcs, rendered) @@ -228,20 +189,20 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { t.Log("Render conductor and Agent roles from the base PodSpec") rendered, err := renderSelectedForTest(renderTestPCS(false), []*ModelProjection{projection}, RenderInput{ - Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, - SSHSecretName: "ssh-secret", + Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: conductorPodSpec}}, + Conductor: &corev1.PodTemplateSpec{Spec: *conductorPodSpec.DeepCopy()}, }) require.NoError(t, err) t.Log("Verify conductor-owned placement and entrypoint behavior") conductor := namedClique(t, rendered, "cond") - require.Nil(t, conductor.Spec.PodSpec.Affinity) + require.Equal(t, conductorPodSpec.Affinity, conductor.Spec.PodSpec.Affinity) require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) - requireFlagValue(t, conductor.Spec.PodSpec.Containers[0].Args, "--allocation", "agt") + require.Equal(t, "agt", testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, allocationEnvVar)) require.Equal(t, []string{"agt"}, conductor.Spec.StartsAfter) t.Log("Verify Agent-owned affinity and logical model binding") @@ -267,7 +228,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { targetFixture.pipelineName = runtimeModelTarget targetSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, targetFixture)) - t.Log("Keep independent draft and target templates, with the conductor seeded by target") + t.Log("Keep independent draft, target and conductor templates") draft := testLPXComponent(draftStageName, "draft-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) @@ -277,7 +238,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { "setup":{"agent_connect_timeout":"30s","agent_setup_timeout":"180s"} }`)} target := testLPXComponent(testTargetStageName, "target-build", - v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) source := newSelectedTestDGD(t, "specdecode", draft, target) @@ -289,6 +250,10 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { template.Spec.Containers[0].Image = component.ComponentName + "-runtime" stages[component.ComponentName] = *template.DeepCopy() } + conductorTemplate := source.Spec.Components[1].ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + conductorTemplate.Labels = map[string]string{"owner": "conductor"} + conductorTemplate.Annotations = map[string]string{"owner": "conductor"} + conductorTemplate.Spec.Containers[0].Image = "conductor-runtime" before := source.DeepCopy() t.Log("Resolve and render the authored speculative workload") @@ -300,7 +265,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { plan, err := selected.PlanNodeLocalMaterialization(pcs.Name) require.NoError(t, err) extraResources, err := RenderSelectedNodeLocal(pcs, selected, plan, RenderInput{ - Stages: stages, SSHSecretName: "ssh-secret", + Stages: stages, Conductor: conductorTemplate.DeepCopy(), }) require.NoError(t, err) @@ -316,9 +281,9 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { firstAgent := namedClique(t, pcs, plan.Agents[0].TemplateName) secondAgent := namedClique(t, pcs, plan.Agents[1].TemplateName) conductor := namedClique(t, pcs, plan.ConductorTemplate) - require.Equal(t, "target-runtime", conductor.Spec.PodSpec.Containers[0].Image) - require.Equal(t, testTargetStageName, conductor.Labels["owner"]) - require.Equal(t, testTargetStageName, conductor.Annotations["owner"]) + require.Equal(t, "conductor-runtime", conductor.Spec.PodSpec.Containers[0].Image) + require.Equal(t, "conductor", conductor.Labels["owner"]) + require.Equal(t, "conductor", conductor.Annotations["owner"]) secondAgentBefore := secondAgent.Spec.PodSpec.DeepCopy() conductorBefore := conductor.Spec.PodSpec.DeepCopy() require.NotEmpty(t, firstAgent.Spec.PodSpec.Containers[0].Env) @@ -343,7 +308,7 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { require.Equal(t, before, source) t.Log("Bind conductor allocation and Nova hostnames to the renamed Agent cliques") - requireFlagValue(t, conductor.Spec.PodSpec.Containers[0].Args, "--allocation", "agt0:agt1:agt2") + require.Equal(t, "agt0:agt1:agt2", testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, allocationEnvVar)) require.Equal(t, []string{"agt0", "agt1", "agt2"}, conductor.Spec.StartsAfter) var datacenter struct { Datacenters struct { @@ -399,7 +364,7 @@ func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { {Name: "model-storage", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}}}, }, Containers: []corev1.Container{{Name: "main", Image: "cyborg", Env: []corev1.EnvVar{ - {Name: selectedCyborgServerHostsFileEnv, Value: "/tmp/lpu_servers"}, + {Name: "SERVER_HOSTS_FILE", Value: "/tmp/lpu_servers"}, }, VolumeMounts: []corev1.VolumeMount{ {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, {Name: "model-storage", MountPath: "/models"}, @@ -422,7 +387,7 @@ func renderTestPodSpec() corev1.PodSpec { ClaimName: "model-storage", }}, }, - {Name: conductorSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "ssh-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret"}}}, {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, @@ -432,8 +397,8 @@ func renderTestPodSpec() corev1.PodSpec { VolumeMounts: []corev1.VolumeMount{ {Name: "model-storage", MountPath: "/models"}, {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, - {Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}, - {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, + {Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}, + {Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}, }, Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{ corev1.ResourceName("lpu.nvidia.com/devices"): resource.MustParse("1"), diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index 00e2eb69b08c..25f1d088d2a5 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -148,10 +148,7 @@ func validateSelectedConductor( if pipeline == PipelineSpecDecode { return fmt.Errorf("%w: the shared speculative runtime requires LPU-only builds", ErrUnsupportedRuntime) } - template := component.ComponentRole(dynamov1beta1.ComponentRoleLPXAgent).PodTemplate - if conductor != nil && conductor.PodTemplate != nil { - template = conductor.PodTemplate - } + template := conductor.PodTemplate container := common.FindContainerByName(template.Spec.Containers, commonconsts.MainContainerName) count, err := EffectiveCyborgGPUCount(container.Resources) if err != nil { @@ -168,47 +165,24 @@ func validateSelectedConductor( return nil } - // An omitted conductor role or template inherits the serving agent's template. - role := conductor - if role == nil || role.PodTemplate == nil { - role = component.ComponentRole(dynamov1beta1.ComponentRoleLPXAgent) - } - - // Preserve the authored component and role path when reporting inherited collisions. + // Preserve authored indices when reporting conductor name collisions. componentIndex := slices.IndexFunc(dgd.Spec.Components, func(candidate dynamov1beta1.DynamoComponentDeploymentSharedSpec) bool { return candidate.ComponentName == component.ComponentName }) roleIndex := slices.IndexFunc(component.Roles, func(candidate dynamov1beta1.ComponentRoleSpec) bool { - return candidate.Name == role.Name + return candidate.Name == conductor.Name }) podSpecPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec") // Check both container lists before a selected workload can be published. - if err := validateRolePodSpecContainerNames(&role.PodTemplate.Spec, podSpecPath, "conductor").ToAggregate(); err != nil { + if err := validateRolePodSpecContainerNames(&conductor.PodTemplate.Spec, podSpecPath, dynamov1beta1.ComponentRoleLPXConductor).ToAggregate(); err != nil { return err } - // Preserve explicit conductor replica and launch validation after checking inherited names. - if conductor == nil { - return nil - } + // An LPU-only conductor is a singleton even when the engine replica count is larger. if ptr.Deref(conductor.Replicas, 1) != 1 { return fmt.Errorf("%w: component %q conductor replicas must be one for LPU-only execution", ErrUnsupportedRuntime, component.ComponentName) } - if conductor.PodTemplate == nil { - return nil - } - // Explicit LPU conductors receive the same appended launch arguments as inherited templates. - for containerIndex := range conductor.PodTemplate.Spec.Containers { - container := &conductor.PodTemplate.Spec.Containers[containerIndex] - if container.Name != commonconsts.MainContainerName { - continue - } - containerPath := podSpecPath.Child("containers").Index(containerIndex) - if err := validateAllocationInjectionTargetFields(container, containerPath).ToAggregate(); err != nil { - return err - } - } return nil } diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index c84f34a09753..ba2b4182d22a 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -23,7 +23,7 @@ import ( func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { t.Log("Create one LPX component beside an unrelated conventional decode") - dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")})) + dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor-runtime")})) dgd.Annotations = nil dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "ordinary-decode", ComponentType: v1beta1.ComponentTypeDecode, @@ -56,7 +56,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin } dgd.Spec.Components[0].Replicas = nil - t.Log("Require hybrid conductor resources in either the explicit or inherited template") + t.Log("Require resources on the independently authored hybrid conductor") fixture := newV2CompilerFixture() fixture.compilationMode = manifestcapnp.CompilationMode_lpx fixture.selectedPropSyncChains = nil @@ -65,7 +65,6 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin source := staticBuildSnapshotSource{"build": snapshot} _, err = ResolveSelectedWorkload(t.Context(), dgd, source) require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") - dgd.Spec.Components[0].Roles = append(dgd.Spec.Components[0].Roles, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}) conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) conductor.PodTemplate = testLPXPodTemplate("cyborg-runtime") _, err = ResolveSelectedWorkload(t.Context(), dgd, source) @@ -100,16 +99,6 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Equal(t, plan, scheduledPlan) require.Empty(t, scheduledPlan.ConductorTemplate) - t.Log("Use the same agent template when the standalone conductor role is omitted") - agent := *dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent) - agent.PodTemplate = conductor.PodTemplate.DeepCopy() - dgd.Spec.Components[0].Roles = []v1beta1.ComponentRoleSpec{agent} - before := dgd.DeepCopy() - implicit, err := ResolveSelectedWorkload(t.Context(), dgd, source) - require.NoError(t, err) - require.Equal(t, xt, implicit) - require.Same(t, &dgd.Spec.Components[0], ServingComponent(dgd)) - require.Equal(t, before, dgd) } func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { @@ -145,7 +134,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Build a selected SpecDecode DGD for the fixture's manifest generation") dgd := newSelectedTestDGD(t, "specdecode", - testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), + testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), testLPXComponent("small", "draft-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), ) dgd.Spec.Components[0].LPX.Settings = settings @@ -278,26 +267,29 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { } } -func TestResolveConductorLaunchErrorUsesAuthoredRolePath(t *testing.T) { - t.Log("Place the target first and author a conductor argument owned by the operator") +func TestResolveSelectedWorkloadPreservesAuthoredLaunch(t *testing.T) { + t.Log("Author independent wrappers without exposing launch syntax to the operator") conductor := testLPXPodTemplate("conductor-runtime") - conductor.Spec.Containers[0].Args = []string{"--allocation=forged"} - dgd := newSelectedTestDGD(t, "specdecode", - testLPXComponent("large", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: conductor}), - testLPXComponent("small", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft-runtime")}), - ) + conductor.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} + conductor.Spec.Containers[0].Args = []string{"exec custom-conductor --workers \"$LPX_ALLOCATION\"", "--"} + agent := testLPXPodTemplate("agent-runtime") + agent.Spec.Containers[0].Command = []string{"/custom-worker", "--"} + agent.Spec.Containers[0].Args = []string{"--allocation=application-owned"} + dgd := newSelectedTestDGD(t, "selected", testLPXComponent("lpx", "build", + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: agent}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: conductor}, + )) snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) before := dgd.DeepCopy() - t.Log("Report the authored target and conductor indices after canonical build selection") + t.Log("Validate placement and role ownership without parsing either command line") _, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": snapshot}) - require.ErrorContains(t, err, "spec.components[0].roles[1].podTemplate.spec.containers[0].args") - require.ErrorContains(t, err, "must not set --allocation") + require.NoError(t, err) require.Equal(t, before, dgd) } func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *testing.T) { - t.Log("Author independent launch and placement errors on both roles") + t.Log("Author placement and container identity errors on both roles") agent, conductor := testLPXPodTemplate("agent"), testLPXPodTemplate("conductor") agent.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} agent.Spec.Containers[0].Args = []string{"--allocation=forged", "--"} @@ -316,9 +308,6 @@ func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *tes t.Log("Aggregate every actionable error at its authored path before acquiring a build") _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) for _, message := range []string{ - "spec.components[0].roles[0].podTemplate.spec.containers[0].args: Forbidden: selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list", - "spec.components[0].roles[0].podTemplate.spec.containers[0].args: Forbidden: selected LPX main container must not terminate arguments before Dynamo appends --allocation", - "spec.components[0].roles[0].podTemplate.spec.containers[0].command: Forbidden: selected LPX main container cannot use a shell because Dynamo appends --allocation", "spec.components[0].roles[0].podTemplate.spec.hostname: Forbidden: LPX owns role addressing and placement", `spec.components[0].roles[0].podTemplate.spec.containers[1].name: Forbidden: LPX reserves "agent" for the materialized role container`, `spec.components[0].roles[0].podTemplate.spec.initContainers[0].name: Forbidden: LPX reserves "agent" for the materialized role container`, @@ -344,7 +333,7 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t hybridFixture.partitions = append(hybridFixture.partitions, testV3CapnpPartition{id: 11, deviceType: manifestcapnp.DeviceType_cuda}) hybridSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, hybridFixture)) - t.Log("Cover explicit, inherited, draft and hybrid template ownership") + t.Log("Cover independent serving, draft and hybrid template ownership") tests := []struct { name string conductor *v1beta1.ComponentRoleSpec @@ -354,12 +343,6 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t hybrid bool wantForbidden bool }{ - {name: "omitted conductor role", wantForbidden: true}, - { - name: "conductor with omitted template", - conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, - wantForbidden: true, - }, { name: "independent conductor template", conductor: &v1beta1.ComponentRoleSpec{ @@ -374,24 +357,12 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t containerInConductor: true, wantForbidden: true, }, - { - name: "target with omitted conductor template", - conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, - addDraft: true, - wantForbidden: true, - }, { name: "draft does not supply the conductor template", - conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, + conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, addDraft: true, containerInDraft: true, }, - {name: "hybrid with omitted conductor role", hybrid: true}, - { - name: "hybrid with omitted conductor template", - conductor: &v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor}, - hybrid: true, - }, { name: "hybrid with explicit conductor template", conductor: &v1beta1.ComponentRoleSpec{ diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 80a07b8ba2d5..960f03ac75c6 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -6,144 +6,25 @@ package lpx import ( - "fmt" "slices" - "strings" + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ) -// Preserve rendered workload identities and image-owned executable paths. -const ( - runtimeTemporaryStorageMountPath = "/tmp" - lp30InitContainerName = "prepare-lp30" - lp30InitPath = "/bin/lp30-agent-v2-init" -) - -var lp30WorkerEnvironment = []corev1.EnvVar{ - {Name: "BMC_SSH_ENABLED", Value: "false"}, - {Name: "GROQ_HAL_V2_BACKGROUND_METRICS", Value: "false"}, - {Name: "GROQ_PG613_PORT_CONFIG", Value: "EB3"}, - {Name: "GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2", Value: "false"}, - {Name: "GROQ_PG613_SKIP_HOST_LINK_PROFILE", Value: "0"}, - {Name: "GROQ_V2_FPGA_BUS_ADDRESS_ID_0", Value: "0000:9a:00.0"}, - {Name: "GROQ_V2_FPGA_BUS_ADDRESS_ID_1", Value: "0000:7e:00.0"}, - {Name: "GROQ_V2_HEALTH_CHECK_ON_OPEN", Value: "0"}, - {Name: "GROQ_V2_RESET_ON_OPEN", Value: "0"}, - {Name: "LPU_FPGA_AUTOMATIC_JTAG_RECOVERY", Value: "false"}, - {Name: "LPU_FPGA_RECONFIGURE_METHOD", Value: "cvp"}, - {Name: "REDFISH_BMC_ENABLED", Value: "false"}, -} - -// nodeLocalHXAgentEnvironmentArgs is the immutable Nova environment suffix for HX. -var nodeLocalHXAgentEnvironmentArgs = []string{ - "--agent-env-vars", "BMC_SSH_ENABLED=false", - "--agent-env-vars", "GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0", - "--agent-env-vars", "GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false", - "--agent-env-vars", "GROQ_HAL_V2_BACKGROUND_METRICS=false", - "--agent-env-vars", "GROQ_PG613_PORT_CONFIG=EB3", - "--agent-env-vars", "GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false", - "--agent-env-vars", "GROQ_PG613_SKIP_HOST_LINK_PROFILE=1", - "--agent-env-vars", "GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0", - "--agent-env-vars", "GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0", - "--agent-env-vars", "GROQ_V2_HEALTH_CHECK_ON_OPEN=0", - "--agent-env-vars", "GROQ_V2_RESET_ON_OPEN=0", - "--agent-env-vars", "LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false", - "--agent-env-vars", "LPU_FPGA_RECONFIGURE_METHOD=cvp", - "--agent-env-vars", "LPU_PARTITION_RUNNER_DATAPLANE=pcie", - "--agent-env-vars", "LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000", - "--agent-env-vars", "LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10", - "--agent-env-vars", "LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true", - "--agent-env-vars", "LPU_VSAP_MAX_TENSORS_PER_CHUNK=4", - "--agent-env-vars", "REDFISH_BMC_ENABLED=false", -} - -// stripNovaOnlyArgs removes conductor-owned Nova CLI flags and reports whether -// every input argument was a well-formed conductor-owned flag. -func stripNovaOnlyArgs(args []string) ([]string, bool) { - if len(args) == 0 { - return args, true - } - - // Parse both accepted flag forms once while retaining worker arguments in order. - const modelNameFlag = "--instance-model-name" - const agentEnvFlag = "--agent-env-vars" - out := make([]string, 0, len(args)) - novaOnly := true - seenModelName := false - for i := 0; i < len(args); i++ { - argument := args[i] - switch argument { - case modelNameFlag, agentEnvFlag: - validValue := i+1 < len(args) && strings.TrimSpace(args[i+1]) != "" - if argument == modelNameFlag { - validValue = validValue && !seenModelName - seenModelName = true - } - novaOnly = novaOnly && validValue - if i+1 < len(args) { - i++ - } - default: - flag, value, hasValue := strings.Cut(argument, "=") - switch flag { - case modelNameFlag: - novaOnly = novaOnly && !seenModelName && hasValue && strings.TrimSpace(value) != "" - seenModelName = true - case agentEnvFlag: - novaOnly = novaOnly && hasValue && strings.TrimSpace(value) != "" - default: - novaOnly = false - out = append(out, argument) - } - } - } - return out, novaOnly -} - // configureDirectHybridAgentRuntime lowers an LPX-scheduled PodSpec into the // direct agent runtime used by conductorless hybrid workloads. func configureDirectHybridAgentRuntime( agentPodSpec *corev1.PodSpec, - lpuConfigMapName, sshSecretName string, - multiNode bool, -) error { - if strings.TrimSpace(sshSecretName) == "" { - return fmt.Errorf("direct hybrid agent runtime requires an MPI SSH secret name") - } - + lpuConfigMapName string, +) { agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) - // Drop Nova-only flags before applying the direct Agent identity and worker defaults. - agent.Args, _ = stripNovaOnlyArgs(agent.Args) - hasCustomStartup := len(agent.Command) != 0 || len(agent.Args) != 0 + // Keep the authored startup while applying the Agent identity and worker resources. agent.Name = lpuAgentContainerName applyLPUWorkerContainerBase(agent) - if !hasCustomStartup { - if err := validateGeneratedAgentMounts(agent, true, multiNode); err != nil { - return err - } - agent.Command = []string{"/bin/hydra-entrypoint"} - agent.Args = []string{"start"} - } - - if !hasCustomStartup || agent.StartupProbe == nil { - agent.StartupProbe = lpuV2StartupProbe(hasCustomStartup) - } - if !hasCustomStartup || agent.ReadinessProbe == nil { - agent.ReadinessProbe = lpuV2ReadinessProbe(hasCustomStartup) - } - - agent.Env = append(agent.Env, - corev1.EnvVar{Name: "GLUE_RDMA_PATH_MTU", Value: "1024"}, - corev1.EnvVar{Name: "NIC_NAME", Value: "mlx5_0"}, - corev1.EnvVar{Name: "UCX_TLS", Value: "tcp"}, - corev1.EnvVar{Name: "RDMA_PORT", Value: fmt.Sprintf("%d", lpuRDMAPort)}, - corev1.EnvVar{Name: "READINESS_PORT", Value: fmt.Sprintf("%d", LPUReadinessPort)}, - ) retargetMainContainerReferences(agentPodSpec, agent) @@ -164,9 +45,6 @@ func configureDirectHybridAgentRuntime( }, ) setNodeLocalPodIPEnv(agent, true) - if err := validateRuntimeTemporaryStorage(agentPodSpec, agent); err != nil { - return err - } updateWorkerPodSpec(agentPodSpec) // Keep volume permissions and other Pod settings while forcing the Agent's root identity. @@ -176,191 +54,42 @@ func configureDirectHybridAgentRuntime( agentPodSpec.SecurityContext.RunAsUser = ptr.To(int64(0)) agentPodSpec.SecurityContext.RunAsGroup = ptr.To(int64(0)) agentPodSpec.SecurityContext.RunAsNonRoot = ptr.To(false) - - return validateSSHVolume(agentPodSpec, sshSecretName) } -// configureNodeLocalConductorRuntime consumes a nonnil conductor PodSpec. -// The caller has validated the runtime SSH secret name. +// configureNodeLocalConductorRuntime consumes a fresh conductor PodSpec with a validated main container. func configureNodeLocalConductorRuntime( conductorPodSpec *corev1.PodSpec, targetFamily BuildFamily, allocation string, - sshSecretName string, -) error { - // Resolve the conductor container before changing its image-only contract. - conductor, err := runtimeMainContainer(conductorPodSpec, targetFamily) - if err != nil { - return fmt.Errorf("node-local LPU conductor: %w", err) - } - - isXT := targetFamily == BuildFamilyXT - if !isXT && len(conductor.Command) != 0 { - return fmt.Errorf("node-local LPU runtime requires an image-only main container without command") - } - - // Materialize the GPC conductor around the image's Nova binary. - setNodeLocalPodIPEnv(conductor, isXT) - if err := validateRuntimeConfigStorage(conductorPodSpec, conductor, ""); err != nil { - return err - } - updateLPUConductorContainer(conductor, allocation) - if targetFamily == BuildFamilyHX { - conductor.Args = append(conductor.Args, nodeLocalHXAgentEnvironmentArgs...) - } +) { + // Bind placement data without interpreting the template's executable or arguments. + conductor := common.FindContainerByName(conductorPodSpec.Containers, commonconsts.MainContainerName) + setNodeLocalPodIPEnv(conductor, targetFamily == BuildFamilyXT) + conductor.Name = dynamov1beta1.ComponentRoleLPXConductor + setContainerEnv(conductor, corev1.EnvVar{Name: allocationEnvVar, Value: allocation}) retargetMainContainerReferences(conductorPodSpec, conductor) - return addConductorSSHKey(conductorPodSpec, conductor, sshSecretName) } -// configureNodeLocalAgentRuntime consumes a nonnil Agent PodSpec. -// The caller has validated the runtime SSH secret name. +// configureNodeLocalAgentRuntime consumes a fresh Agent PodSpec with a validated main container. func configureNodeLocalAgentRuntime( agentPodSpec *corev1.PodSpec, targetFamily BuildFamily, - validateXTSingleArgs bool, - sshSecretName string, -) error { - // Validate the emitted Agent's own shape before changing its startup contract. - agent, err := runtimeMainContainer(agentPodSpec, targetFamily) - if err != nil { - return fmt.Errorf("node-local LPU agent: %w", err) - } +) { + // Apply the local LPU role identity and worker resources. + agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) isXT := targetFamily == BuildFamilyXT - if isXT { - strippedArgs, novaOnly := stripNovaOnlyArgs(agent.Args) - if validateXTSingleArgs && len(agent.Command) == 0 && !novaOnly { - return fmt.Errorf("operator-managed XT single only supports --instance-model-name and --agent-env-vars; set command for custom startup behavior") - } - agent.Args = strippedArgs - } else if len(agent.Command) != 0 { - return fmt.Errorf("node-local LPU runtime requires an image-only main container without command") - } - preserveAgentEntrypoint := isXT && (len(agent.Command) != 0 || len(agent.Args) != 0) - // Materialize the LPU role as the privileged SSH target that Nova launches. setNodeLocalPodIPEnv(agent, isXT) agent.Name = lpuAgentContainerName retargetMainContainerReferences(agentPodSpec, agent) - configureNodeLocalAgentWorkerContainer(agent, isXT, preserveAgentEntrypoint) - if err := validateRuntimeTemporaryStorage(agentPodSpec, agent); err != nil { - return err - } agentPodSpec.HostUsers = nil updateWorkerPodSpec(agentPodSpec) - if err := validateSSHVolume(agentPodSpec, sshSecretName); err != nil { - return err - } - - // Check only the paths consumed by the selected image launcher. - if !preserveAgentEntrypoint { - if err := validateGeneratedAgentMounts(agent, isXT, true); err != nil { - return err - } - } - // Apply the family-specific worker resource and initialization requirements. + // Apply the XT worker's resource and security requirements. if isXT { applyLPUWorkerContainerBase(agent) agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) agent.SecurityContext.RunAsNonRoot = ptr.To(false) - } else { - return addLP30InitContainer(agentPodSpec, agent) - } - - return nil -} - -// runtimeMainContainer consumes a fresh selected role with a validated main container. -func runtimeMainContainer( - podSpec *corev1.PodSpec, - targetFamily BuildFamily, -) (*corev1.Container, error) { - // Keep V3's image-only single-container contract unchanged. - if targetFamily == BuildFamilyHX && len(podSpec.Containers) != 1 { - return nil, fmt.Errorf("PodSpec requires exactly one %q container", commonconsts.MainContainerName) - } - return common.FindContainerByName(podSpec.Containers, commonconsts.MainContainerName), nil -} - -func configureNodeLocalAgentWorkerContainer( - container *corev1.Container, - isXT bool, - preserveEntrypoint bool, -) { - // The Agent pod hosts SSHD; Nova starts /bin/agent in a separate MPI process. - if !preserveEntrypoint { - container.Command = []string{"/bin/quasar-entrypoint"} - container.Args = nil - if isXT { - container.Args = []string{"--partition-metadata"} - } - } - - if isXT { - if !preserveEntrypoint || container.ReadinessProbe == nil { - container.ReadinessProbe = lpuSSHProbe() - } - setContainerEnv(container, true, corev1.EnvVar{Name: "LPU_RUN_SYSTEM_INIT", Value: "true"}) - } else { - container.SecurityContext = &corev1.SecurityContext{ - Privileged: ptr.To(true), - RunAsUser: ptr.To(int64(0)), - RunAsGroup: ptr.To(int64(0)), - RunAsNonRoot: ptr.To(false), - } - - // Remove health and lifecycle behavior owned by the source image process. - container.LivenessProbe = nil - container.StartupProbe = nil - container.Lifecycle = nil - container.ReadinessProbe = lpuSSHProbe() - setContainerEnv(container, true, lp30WorkerEnvironment...) - } -} - -func addLP30InitContainer(podSpec *corev1.PodSpec, agent *corev1.Container) error { - // Reserve this name only when the selected runtime actually adds the init container. - if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), lp30InitContainerName).ToAggregate(); err != nil { - return err - } - - initContainer := corev1.Container{ - Name: lp30InitContainerName, - Image: agent.Image, - ImagePullPolicy: agent.ImagePullPolicy, - Command: []string{lp30InitPath}, - Env: []corev1.EnvVar{{ - Name: "NODE_NAME", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }}, - }}, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - RunAsUser: ptr.To(int64(0)), - RunAsGroup: ptr.To(int64(0)), - RunAsNonRoot: ptr.To(false), - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "host-sys", MountPath: "/sys"}, - {Name: "host-dev", MountPath: "/dev"}, - }, - } - - // Validate only host volumes consumed by the operator-generated init container. - for _, mount := range initContainer.VolumeMounts { - index := slices.IndexFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) - if index < 0 { - return fmt.Errorf("LP30 initialization requires podTemplate volume %q", mount.Name) - } - hostPath := podSpec.Volumes[index].HostPath - if hostPath == nil || hostPath.Path != mount.MountPath || - hostPath.Type != nil && *hostPath.Type != corev1.HostPathUnset && - *hostPath.Type != corev1.HostPathDirectory && *hostPath.Type != corev1.HostPathDirectoryOrCreate { - return fmt.Errorf("LP30 initialization volume %q must use directory hostPath %q", mount.Name, mount.MountPath) - } } - podSpec.InitContainers = append(podSpec.InitContainers, initContainer) - return nil } func setNodeLocalPodIPEnv(container *corev1.Container, isXT bool) { @@ -437,65 +166,3 @@ func retargetResourceFieldReferencesInDownwardAPI(items []corev1.DownwardAPIVolu } } } - -func validateGeneratedAgentMounts(container *corev1.Container, requireConfig, requireSSH bool) error { - // Only an explicit configuration override can waive the default mount. - usesDefaultConfigDir := true - for _, variable := range container.Env { - if variable.Name == "LPU_CONFIG_DIR" { - usesDefaultConfigDir = variable.ValueFrom == nil && (variable.Value == "" || variable.Value == lpuConfigMountPath) - } - } - if requireConfig && usesDefaultConfigDir && !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == lpuConfigMountPath }) { - return fmt.Errorf("LPX runtime configuration requires a volume mounted at %q", lpuConfigMountPath) - } - - // Generated SSH setup reads the Secret's keys directly from this directory. - if requireSSH && !slices.ContainsFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == runtimeSSHVolumeName && mount.MountPath == runtimeSSHSecretMountPath && mount.SubPath == "" && mount.SubPathExpr == "" - }) { - return fmt.Errorf("LPX agent requires volume %q mounted at %q without subPath", runtimeSSHVolumeName, runtimeSSHSecretMountPath) - } - return nil -} - -func validateRuntimeTemporaryStorage(podSpec *corev1.PodSpec, container *corev1.Container) error { - // Agents use writable root filesystems; validate only explicitly mounted workspace storage. - mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.MountPath == runtimeTemporaryStorageMountPath - }) - if mountIndex < 0 { - return nil - } - mount := container.VolumeMounts[mountIndex] - if mount.ReadOnly { - return fmt.Errorf("LPX runtime requires writable storage at %q", mount.MountPath) - } - - // Validate the authored source without creating or replacing it. - volumeIndex := slices.IndexFunc(podSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) - if volumeIndex < 0 { - return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) - } - if volumeIsReadOnly(podSpec.Volumes[volumeIndex].VolumeSource) { - return fmt.Errorf("LPX runtime requires writable storage at %q", mount.MountPath) - } - return nil -} - -func volumeIsReadOnly(source corev1.VolumeSource) bool { - // These sources force a read-only mount even when VolumeMount.ReadOnly is false. - return source.ConfigMap != nil || source.Secret != nil || source.DownwardAPI != nil || source.Projected != nil || source.Image != nil || - source.PersistentVolumeClaim != nil && source.PersistentVolumeClaim.ReadOnly || - source.NFS != nil && source.NFS.ReadOnly || - source.CSI != nil && ptr.Deref(source.CSI.ReadOnly, false) || - source.ISCSI != nil && source.ISCSI.ReadOnly || - source.FC != nil && source.FC.ReadOnly || - source.FlexVolume != nil && source.FlexVolume.ReadOnly || - source.GCEPersistentDisk != nil && source.GCEPersistentDisk.ReadOnly || - source.AWSElasticBlockStore != nil && source.AWSElasticBlockStore.ReadOnly || - source.Cinder != nil && source.Cinder.ReadOnly || - source.AzureFile != nil && source.AzureFile.ReadOnly || - source.AzureDisk != nil && ptr.Deref(source.AzureDisk.ReadOnly, false) || - source.PortworxVolume != nil && source.PortworxVolume.ReadOnly -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh.go deleted file mode 100644 index bc355fb6478d..000000000000 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - "slices" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/ptr" -) - -const ( - runtimeSSHVolumeName = "ssh-secret" - runtimeSSHSecretMountPath = "/ssh-pk" - runtimeSSHSecretPrivateKeyPath = runtimeSSHSecretMountPath + "/private.key" - conductorSSHKeyDir = runtimeTemporaryStorageMountPath + "/dynamo-lpu-ssh" - conductorSSHPrivateKeyPath = conductorSSHKeyDir + "/private.key" - conductorSSHKeyInitContainerName = "prepare-ssh-key" - conductorSSHKeyVolumeName = "single-v2-ssh-key" -) - -func validateSSHVolume(podSpec *corev1.PodSpec, secretName string) error { - // Preserve authored storage and key projections instead of replacing their contents. - for _, volume := range podSpec.Volumes { - if volume.Name == runtimeSSHVolumeName { - if volume.Secret == nil || volume.Secret.SecretName != secretName || len(volume.Secret.Items) != 0 { - return fmt.Errorf("selected LPX podTemplate volume %q is reserved for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) - } - return nil - } - } - return fmt.Errorf("selected LPX podTemplate requires volume %q for MPI SSH Secret %q", runtimeSSHVolumeName, secretName) -} - -func addConductorSSHKey(podSpec *corev1.PodSpec, conductor *corev1.Container, secretName string) error { - // Check both container lists before replacing any authored configuration. - if err := validateRolePodSpecContainerNames(podSpec, field.NewPath("spec"), conductorSSHKeyInitContainerName).ToAggregate(); err != nil { - return err - } - - // Require the authored source Secret used by the generated init container. - if err := validateSSHVolume(podSpec, secretName); err != nil { - return err - } - - // The init container and Nova must share the authored Pod-local key directory. - if !slices.ContainsFunc(podSpec.Volumes, func(volume corev1.Volume) bool { - return volume.Name == conductorSSHKeyVolumeName && volume.EmptyDir != nil - }) { - return fmt.Errorf("LPX conductor requires emptyDir volume %q", conductorSSHKeyVolumeName) - } - if !slices.ContainsFunc(conductor.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == conductorSSHKeyVolumeName && mount.MountPath == conductorSSHKeyDir && mount.SubPath == "" && mount.SubPathExpr == "" - }) { - return fmt.Errorf("LPX conductor requires volume %q mounted at %q without subPath", conductorSSHKeyVolumeName, conductorSSHKeyDir) - } - - // Copy the key with OpenSSH's required mode before the conductor starts. - initContainer := corev1.Container{ - Name: conductorSSHKeyInitContainerName, - Image: conductor.Image, - ImagePullPolicy: conductor.ImagePullPolicy, - Command: []string{"/bin/install"}, - Args: []string{"-m", "0600", runtimeSSHSecretPrivateKeyPath, conductorSSHPrivateKeyPath}, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To(int64(0)), - RunAsGroup: ptr.To(int64(0)), - RunAsNonRoot: ptr.To(false), - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}, - {Name: runtimeSSHVolumeName, MountPath: runtimeSSHSecretMountPath, ReadOnly: true}, - }, - } - podSpec.InitContainers = append(podSpec.InitContainers, initContainer) - return nil -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go b/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go deleted file mode 100644 index 57ebceeabdec..000000000000 --- a/deploy/operator/internal/dynamo/lpx/runtime_ssh_test.go +++ /dev/null @@ -1,117 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "testing" - - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" -) - -func TestValidateSSHVolumePreservesAuthoredStorage(t *testing.T) { - for _, test := range []struct { - name string - source corev1.VolumeSource - wantErr bool - }{ - {"matching Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "mpi.ssh"}}, false}, - {"persistent data", corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}, true}, - {"different Secret", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}, true}, - {"restricted keys", corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ - SecretName: "mpi.ssh", Items: []corev1.KeyToPath{{Key: "public.key", Path: "public.key"}}, - }}, true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Bind the authored storage to the runtime SSH volume name") - pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: test.source}}} - before := pod.DeepCopy() - - t.Log("Require a compatible MPI key binding without changing the authored source") - err := validateSSHVolume(&pod, "mpi.ssh") - if test.wantErr { - require.ErrorContains(t, err, `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"`) - require.Equal(t, *before, pod) - return - } - require.NoError(t, err) - require.Equal(t, *before, pod) - }) - } -} - -func TestAddConductorSSHKeyValidatesAuthoredStorage(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - source corev1.VolumeSource - mount corev1.VolumeMount - err string - }{ - {name: "Secret", source: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "application"}}, err: "requires emptyDir"}, - {name: "PVC", source: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}, err: "requires emptyDir"}, - {name: "missing source", err: "requires emptyDir"}, - {name: "missing mount", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, err: "without subPath"}, - {name: "subPath", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, mount: corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir, SubPath: "keys"}, err: "without subPath"}, - {name: "EmptyDir", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, mount: corev1.VolumeMount{Name: conductorSSHKeyVolumeName, MountPath: conductorSSHKeyDir}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Author the conductor key source, destination and runtime mount") - pod := corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: commonconsts.MainContainerName, Image: "lpu-runtime", - VolumeMounts: []corev1.VolumeMount{test.mount}, - }}, - Volumes: []corev1.Volume{ - {Name: runtimeSSHVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ - SecretName: "mpi.ssh", DefaultMode: ptr.To(int32(0644)), - }}}, - {Name: conductorSSHKeyVolumeName, VolumeSource: test.source}, - }, - } - before := pod.DeepCopy() - - t.Log("Require a shared Pod-local directory without changing authored storage") - err := addConductorSSHKey(&pod, &pod.Containers[0], "mpi.ssh") - if test.err != "" { - require.ErrorContains(t, err, test.err) - require.Equal(t, *before, pod) - return - } - require.NoError(t, err) - require.Equal(t, before.Volumes, pod.Volumes) - require.Equal(t, before.Containers, pod.Containers) - require.Len(t, pod.InitContainers, 1) - require.Equal(t, []string{"/bin/install"}, pod.InitContainers[0].Command) - require.Equal(t, []string{"-m", "0600", "/ssh-pk/private.key", "/tmp/dynamo-lpu-ssh/private.key"}, pod.InitContainers[0].Args) - }) - } -} - -func TestXTRuntimesRejectSSHVolumeCollision(t *testing.T) { - t.Log("Start each XT role with an authored volume using the fixed SSH name") - base := corev1.PodSpec{ - Containers: []corev1.Container{{Name: commonconsts.MainContainerName, Image: "lpu-runtime"}}, - Volumes: []corev1.Volume{{Name: "ssh-secret", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, - } - base.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}} - configureAgentScheduling(&base, BuildFamilyXT) - direct, agent, conductor := base.DeepCopy(), base.DeepCopy(), base.DeepCopy() - - t.Log("Propagate the collision through every XT runtime configuration path") - const wantError = `volume "ssh-secret" is reserved for MPI SSH Secret "mpi.ssh"` - require.ErrorContains(t, configureDirectHybridAgentRuntime(direct, "graph-lpu", "mpi.ssh", false), wantError) - require.ErrorContains(t, configureNodeLocalAgentRuntime(agent, BuildFamilyXT, false, "mpi.ssh"), wantError) - require.ErrorContains(t, configureNodeLocalConductorRuntime(conductor, BuildFamilyXT, "agt", "mpi.ssh"), wantError) - - t.Log("Preserve the authored volume in every rejected runtime") - for _, pod := range []*corev1.PodSpec{direct, agent, conductor} { - require.Equal(t, base.Volumes[0].VolumeSource, testVolumeByName(t, pod.Volumes, "ssh-secret")) - } -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup.go b/deploy/operator/internal/dynamo/lpx/runtime_startup.go deleted file mode 100644 index cc45c0525c16..000000000000 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" -) - -// validateRuntimeConfigStorage requires the config mount and a writable, Pod-local runtime workspace. -// expandedConfigFile is empty when the runtime reads configuration directly. -func validateRuntimeConfigStorage(podSpec *corev1.PodSpec, container *corev1.Container, expandedConfigFile string) error { - // Reject incompatible authored bindings instead of replacing their volumes or mounts. - hasConfigMount := false - var mount corev1.VolumeMount - for _, existing := range container.VolumeMounts { - hasConfigMount = hasConfigMount || existing.MountPath == lpuConfigMountPath - if expandedConfigFile != "" && existing.MountPath == runtimeTemporaryStorageMountPath+"/"+expandedConfigFile { - return fmt.Errorf("LPX runtime configuration cannot overwrite a volume mounted at %q", existing.MountPath) - } - if existing.MountPath == runtimeTemporaryStorageMountPath { - mount = existing - } - } - if !hasConfigMount { - return fmt.Errorf("LPX runtime configuration requires a volume mounted at %q", lpuConfigMountPath) - } - if mount.Name == "" { - if container.SecurityContext != nil && ptr.Deref(container.SecurityContext.ReadOnlyRootFilesystem, false) { - return fmt.Errorf("LPX runtime configuration requires writable storage at %q", runtimeTemporaryStorageMountPath) - } - return nil - } - if mount.ReadOnly { - return fmt.Errorf("LPX runtime configuration requires writable storage at %q", mount.MountPath) - } - for _, volume := range podSpec.Volumes { - if volume.Name == mount.Name { - if volume.EmptyDir == nil { - return fmt.Errorf("LPX runtime configuration requires Pod-local emptyDir storage at %q", mount.MountPath) - } - return nil - } - } - return fmt.Errorf("LPX runtime mount at %q references missing volume %q", mount.MountPath, mount.Name) -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index 2731429ee1cd..aeb4a80a2733 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -6,6 +6,7 @@ package lpx import ( + "slices" "testing" "github.com/stretchr/testify/require" @@ -13,66 +14,134 @@ import ( "k8s.io/utils/ptr" ) -func TestRuntimeConfigStorage(t *testing.T) { - for _, security := range []*corev1.SecurityContext{nil, {ReadOnlyRootFilesystem: ptr.To(false)}, {ReadOnlyRootFilesystem: ptr.To(true)}} { - t.Log("Allow container-local expansion unless the retained root filesystem is read-only") - pod := corev1.PodSpec{} - container := corev1.Container{SecurityContext: security, VolumeMounts: []corev1.VolumeMount{{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}}} - before := container.DeepCopy() - err := validateRuntimeConfigStorage(&pod, &container, "lpu_servers") - if security != nil && ptr.Deref(security.ReadOnlyRootFilesystem, false) { - require.ErrorContains(t, err, "writable storage") - } else { - require.NoError(t, err) - } - require.Empty(t, pod.Volumes) - require.Equal(t, *before, container) - } +func TestRuntimePreservesAuthoredStartup(t *testing.T) { + t.Parallel() - for _, source := range []corev1.VolumeSource{ - {EmptyDir: &corev1.EmptyDirVolumeSource{}}, - {PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "shared"}}, - {Secret: &corev1.SecretVolumeSource{SecretName: "readonly"}}, + for _, role := range []struct { + name string + family BuildFamily + conductor bool + direct bool + }{ + {name: "XT conductor", family: BuildFamilyXT, conductor: true}, + {name: "HX conductor", family: BuildFamilyHX, conductor: true}, + {name: "XT worker", family: BuildFamilyXT}, + {name: "HX worker", family: BuildFamilyHX}, + {name: "XT direct hybrid", family: BuildFamilyXT, direct: true}, + {name: "HX direct hybrid", family: BuildFamilyHX, direct: true}, } { - t.Log("Reuse only a writable Pod-local workspace for expanded runtime configuration") - pod := corev1.PodSpec{Volumes: []corev1.Volume{{Name: "scratch", VolumeSource: source}}} - container := corev1.Container{SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: ptr.To(true)}, VolumeMounts: []corev1.VolumeMount{ - {Name: "scratch", MountPath: "/tmp"}, - {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, - }} - authoredPod, authoredContainer := pod.DeepCopy(), container.DeepCopy() - err := validateRuntimeConfigStorage(&pod, &container, "lpu_servers") - if source.EmptyDir == nil { - require.ErrorContains(t, err, "Pod-local emptyDir") - } else { - require.NoError(t, err) - require.Len(t, pod.Volumes, 1) - } - require.Equal(t, *authoredPod, pod) - require.Equal(t, *authoredContainer, container) - container.VolumeMounts[0].ReadOnly = true - require.ErrorContains(t, validateRuntimeConfigStorage(&pod, &container, "lpu_servers"), "writable storage") - } -} + for _, startup := range []struct { + name string + command []string + args []string + }{ + {name: "image defaults"}, + {name: "image ENTRYPOINT with args", args: []string{"--instance-model-name", "custom-model", "--agent-env-vars=KEEP=1"}}, + {name: "explicit command", command: []string{"/custom-launcher", "wrapper-option"}, args: []string{"argument with spaces", "literal $HOME", ""}}, + } { + t.Run(role.name+"/"+startup.name, func(t *testing.T) { + t.Parallel() -func TestNativeNovaConfigStorage(t *testing.T) { - t.Log("Mount a file at the obsolete expansion path alongside the original configuration") - pod := renderTestPodSpec() - pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: "unrelated", MountPath: "/tmp/datacenter.toml", ReadOnly: true}, - ) - pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "unrelated", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated"}}, - }}) + t.Log("Author startup, health, lifecycle and initialization independently of LPX binary names") + pod := corev1.PodSpec{Containers: []corev1.Container{ + { + Name: "sidecar", Image: "helper-runtime", Command: []string{"/custom-helper"}, Args: []string{"observe"}, + Env: []corev1.EnvVar{{Name: "HELPER_SETTING", Value: "retained"}}, ReadinessProbe: testExecProbe("helper-ready"), + SecurityContext: &corev1.SecurityContext{RunAsUser: ptr.To(int64(1000)), RunAsNonRoot: ptr.To(true)}, + }, + { + Name: "main", Image: "custom-runtime", Command: startup.command, Args: startup.args, + Env: []corev1.EnvVar{{Name: "LPX_ALLOCATION", Value: "forged-allocation"}}, + }, + }} + if startup.args != nil { + pod.Containers[1].SecurityContext = &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(1000)), RunAsGroup: ptr.To(int64(2000)), + RunAsNonRoot: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), + } + } + if startup.command == nil && startup.args != nil { + pod.Volumes = []corev1.Volume{ + {Name: "credentials", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.To(int32(0400)), Sources: []corev1.VolumeProjection{{Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom.ssh"}, + Items: []corev1.KeyToPath{{Key: "private.key", Path: "id"}}, + }}}, + }}}, + {Name: "scratch", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "readonly"}}}, + } + pod.Containers[1].VolumeMounts = []corev1.VolumeMount{ + {Name: "credentials", MountPath: "/custom/ssh", ReadOnly: true}, + {Name: "scratch", MountPath: "/tmp", ReadOnly: true}, + } + } + if startup.command != nil { + pod.Containers[1].Env = append(pod.Containers[1].Env, + corev1.EnvVar{Name: "GROQ_V2_RESET_ON_OPEN", Value: "template-value"}, + corev1.EnvVar{Name: "TEMPLATE_SECRET", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "settings"}, Key: "value", + }}}, + ) + pod.Containers[1].StartupProbe = testExecProbe("custom-started") + pod.Containers[1].ReadinessProbe = testExecProbe("custom-ready") + pod.Containers[1].LivenessProbe = testExecProbe("custom-live") + } + pod.Containers[1].Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"custom-stop"}}}, + } + pod.InitContainers = []corev1.Container{ + {Name: "prepare-ssh-key", Image: "custom-key-image", Command: []string{"/custom-key"}, Args: []string{"literal arg"}}, + {Name: "prepare-lp30", Image: "custom-init-image", Command: []string{"/custom-init"}}, + } + before := pod.DeepCopy() - t.Log("Launch Nova against /configs without trying to overwrite the unrelated mount") - require.NoError(t, configureNodeLocalConductorRuntime(&pod, BuildFamilyXT, "agent", "ssh-secret")) - require.Equal(t, []string{"/bin/nova"}, pod.Containers[0].Command) - requireFlagValue(t, pod.Containers[0].Args, "--datacenter-config-filepath", "/configs/datacenter.toml") + t.Log("Apply runtime bindings for the selected family and role") + configureAgentScheduling(&pod, role.family) + switch { + case role.conductor: + configureNodeLocalConductorRuntime(&pod, role.family, "allocation") + case role.direct: + configureDirectHybridAgentRuntime(&pod, "config") + default: + configureNodeLocalAgentRuntime(&pod, role.family) + } - t.Log("Continue rejecting an overlapping Cyborg expansion destination") - pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: "unrelated", MountPath: "/tmp/lpu_servers", ReadOnly: true}, - ) - require.ErrorContains(t, validateRuntimeConfigStorage(&pod, &pod.Containers[0], "lpu_servers"), "cannot overwrite") + t.Log("Retain the sidecar and bind only the authored main runtime") + require.Len(t, pod.Containers, 2) + require.Equal(t, before.Containers[0], pod.Containers[0]) + container, authored := pod.Containers[1], before.Containers[1] + require.Equal(t, authored.Command, container.Command) + require.Equal(t, authored.Args, container.Args) + if role.conductor { + require.Equal(t, "conductor", container.Name) + require.Equal(t, "allocation", testContainerEnvValue(container.Env, "LPX_ALLOCATION")) + } else { + require.Equal(t, lpuAgentContainerName, container.Name) + } + if role.conductor || role.family == BuildFamilyHX && !role.direct { + require.Equal(t, authored.SecurityContext, container.SecurityContext) + } + + t.Log("Keep static environment values template-owned, including intentional omission") + dynamicEnv := map[string]bool{ + "POD_IP": true, "LPX_ALLOCATION": role.conductor, + "TOPOLOGIES": role.direct, "GAS_DIR": role.direct, + } + container.Env = slices.DeleteFunc(slices.Clone(container.Env), func(variable corev1.EnvVar) bool { return dynamicEnv[variable.Name] }) + if role.conductor { + authored.Env = slices.DeleteFunc(slices.Clone(authored.Env), func(variable corev1.EnvVar) bool { return variable.Name == "LPX_ALLOCATION" }) + } + require.Equal(t, authored.Env, container.Env) + require.Equal(t, authored.StartupProbe, container.StartupProbe) + require.Equal(t, authored.ReadinessProbe, container.ReadinessProbe) + require.Equal(t, authored.LivenessProbe, container.LivenessProbe) + require.Equal(t, authored.Lifecycle, container.Lifecycle) + require.Equal(t, before.InitContainers, pod.InitContainers) + + t.Log("Keep authored storage, including omitted mounts and read-only projected credentials") + require.Equal(t, before.Volumes, pod.Volumes) + require.Equal(t, authored.VolumeMounts, container.VolumeMounts) + }) + } + } } diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go index 265d6a0c95d2..2fd8cd42f594 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_test.go +++ b/deploy/operator/internal/dynamo/lpx/selection_test.go @@ -75,7 +75,7 @@ func TestSelectedModelNames(t *testing.T) { dgd := &dynamov1beta1.DynamoGraphDeployment{Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ Components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{ testLPXComponent("small", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}), - testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor}, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}), + testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}), }, }} dgd.Spec.Components[0].Replicas = &draftReplicas @@ -97,9 +97,8 @@ func TestSelectedModelNames(t *testing.T) { _, err = SelectedModelNames(dgd) require.ErrorIs(t, err, ErrUnsupportedRuntime) } - t.Log("Derive the existing default runtime identity with an implicit conductor") + t.Log("Derive the default runtime identity for a single serving component") dgd.Spec.Components = dgd.Spec.Components[1:] - dgd.Spec.Components[0].Roles = dgd.Spec.Components[0].Roles[1:] names, err = SelectedModelNames(dgd) require.NoError(t, err) require.Equal(t, []string{"default"}, names) @@ -126,32 +125,27 @@ func TestSelectedMinAvailableOwnership(t *testing.T) { t.Parallel() for _, test := range []struct { - name string - draftMin *int32 - targetMin *int32 - singleton bool - implicitConductor bool - wantErr bool + name string + draftMin *int32 + targetMin *int32 + singleton bool + wantErr bool }{ {name: "omitted draft"}, {name: "default draft", draftMin: ptr.To(int32(1))}, {name: "nondefault draft", draftMin: ptr.To(int32(2)), wantErr: true}, {name: "target", targetMin: ptr.To(int32(1))}, {name: "singleton", targetMin: ptr.To(int32(2)), singleton: true}, - {name: "implicit singleton conductor", targetMin: ptr.To(int32(2)), singleton: true, implicitConductor: true}, } { t.Run(test.name, func(t *testing.T) { t.Log("Put a non-LPX component before the target and reverse the authored LPX order") target := testLPXComponent("target", "target-build", - dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor}, + dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}) target.MinAvailable = test.targetMin if test.singleton { target.Replicas = ptr.To(int32(2)) } - if test.implicitConductor { - target.Roles = target.Roles[1:] - } dgd := newSelectedTestDGD(t, "min-available", dynamov1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", MinAvailable: ptr.To(int32(1))}, target) if !test.singleton { diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index 3978b968dc6a..f84fb0ace1dc 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -7,7 +7,6 @@ package lpx import ( "fmt" - "path" "strings" dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -70,7 +69,6 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC return field.ErrorList{field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")} } allErrs := field.ErrorList{} - conductorCount := 0 for index := range dgd.Spec.Components { component := &dgd.Spec.Components[index] if !component.IsLPX() { @@ -78,9 +76,6 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC } componentPath := componentsPath.Index(index) conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) - if conductor != nil { - conductorCount++ - } if componentCount != 2 { continue } @@ -101,9 +96,7 @@ func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentC allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "shared target replicas must be one")) } } - if componentCount == 2 && conductorCount != 1 { - allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one conductor role")) - } + allErrs = append(allErrs, ValidateConductorRoles(&dgd.Spec, componentsPath, componentsPath.Index)...) return allErrs } @@ -131,20 +124,46 @@ func validateSelectedLPXComponent(component *dynamov1beta1.DynamoComponentDeploy allErrs = append(allErrs, field.Required(configPath.Child("buildId"), "LPX component requires a buildId")) } for index, role := range component.Roles { + rolePath := componentPath.Child("roles").Index(index) if role.PodTemplate == nil { continue } - rolePath := componentPath.Child("roles").Index(index) - main, mainPath, errs := validateSelectedRuntimeContainer(role.PodTemplate, "LPX "+role.Name, rolePath) - allErrs = append(allErrs, errs...) - if main != nil && role.Name == dynamov1beta1.ComponentRoleLPXAgent { - allErrs = append(allErrs, validateAllocationInjectionTargetFields(main, mainPath)...) - } + allErrs = append(allErrs, validateSelectedRuntimeContainer(role.PodTemplate, "LPX "+role.Name, rolePath)...) allErrs = append(allErrs, validateLPXRolePlacement(&role.PodTemplate.Spec, rolePath.Child("podTemplate", "spec"), role.Name == dynamov1beta1.ComponentRoleLPXAgent)...) } return allErrs } +// ValidateConductorRoles requires one explicit serving template across LPX components. +// Draft components remain Agent-only. Arguments must be non-nil; spec is not mutated. +// componentsPath and componentPath locate the collection and each component in the source API. +func ValidateConductorRoles(spec *dynamov1beta1.DynamoGraphDeploymentSpec, componentsPath *field.Path, componentPath func(int) *field.Path) field.ErrorList { + // Inspect authored roles so admission and preflight enforce the same snapshot-independent shape. + allErrs := field.ErrorList{} + hasLPX, conductorCount := false, 0 + for componentIndex := range spec.Components { + component := &spec.Components[componentIndex] + if !component.IsLPX() { + continue + } + hasLPX = true + for roleIndex, role := range component.Roles { + if role.Name != dynamov1beta1.ComponentRoleLPXConductor { + continue + } + conductorCount++ + if role.PodTemplate == nil { + rolePath := componentPath(componentIndex).Child("roles").Index(roleIndex) + allErrs = append(allErrs, field.Required(rolePath.Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) + } + } + } + if hasLPX && conductorCount != 1 { + allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one conductor role")) + } + return allErrs +} + // ValidateAgentContainerNames reserves the Agent identity in authored LPX agent // templates. Conductor identities depend on the immutable build's execution mode. // dgd must be non-nil and is not mutated. @@ -172,7 +191,7 @@ func ValidateAgentContainerNames(dgd *dynamov1beta1.DynamoGraphDeployment) field // validateRolePodSpecContainerNames checks both lists that share the Pod's name space. // spec and fldPath must be non-nil. func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path, reservedName string) field.ErrorList { - // Both lists must avoid renamed main and generated init-container names. + // Both lists must avoid the materialized role container name. allErrs := field.ErrorList{} for _, group := range []struct { name string @@ -236,60 +255,19 @@ func validateSelectedRuntimeContainer( template *corev1.PodTemplateSpec, role string, rolePath *field.Path, -) (*corev1.Container, *field.Path, field.ErrorList) { +) field.ErrorList { containersPath := rolePath.Child("podTemplate", "spec", "containers") for index := range template.Spec.Containers { container := &template.Spec.Containers[index] if container.Name != commonconsts.MainContainerName { continue } - containerPath := containersPath.Index(index) - return container, containerPath, nil + return nil } - return nil, nil, field.ErrorList{ + return field.ErrorList{ field.Required( containersPath, fmt.Sprintf("%s component requires a %q runtime container", role, commonconsts.MainContainerName), ), } } - -// validateAllocationInjectionTargetFields requires nonnil container and containerPath. -func validateAllocationInjectionTargetFields(container *corev1.Container, containerPath *field.Path) field.ErrorList { - // Classify Command and Args once before reporting violations in contract order. - var hasAllocation, terminatesArguments, hasShell [2]bool - for vector, values := range [2][]string{container.Command, container.Args} { - for _, value := range values { - hasAllocation[vector] = hasAllocation[vector] || value == "--allocation" || - strings.HasPrefix(value, "--allocation=") - terminatesArguments[vector] = terminatesArguments[vector] || value == "--" - if !hasShell[vector] { - switch path.Base(value) { - case "ash", "bash", "dash", "ksh", "sh", "zsh": - hasShell[vector] = true - } - } - } - } - - allErrs := field.ErrorList{} - if hasAllocation[0] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list")) - } - if hasAllocation[1] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container must not set --allocation; Dynamo renders the immutable Agent clique list")) - } - if terminatesArguments[0] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container must not terminate arguments before Dynamo appends --allocation")) - } - if terminatesArguments[1] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container must not terminate arguments before Dynamo appends --allocation")) - } - if hasShell[0] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("command"), "selected LPX main container cannot use a shell because Dynamo appends --allocation")) - } - if hasShell[1] { - allErrs = append(allErrs, field.Forbidden(containerPath.Child("args"), "selected LPX main container cannot use a shell because Dynamo appends --allocation")) - } - return allErrs -} diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index 440855854fc3..229b5d0ce55c 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -55,7 +55,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1v2-b300n1-nic1 + - gpu-node-0 containers: - args: - --dynamo-model-name @@ -70,11 +70,41 @@ spec: env: - name: GLUE_RDMA_PATH_MTU value: '1024' - image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 + - name: RDMA_PORT + value: '19877' + - name: SERVER_HOSTS_FILE + value: /tmp/lpu_servers + image: example.invalid/gpu-runtime:test name: main resources: limits: nvidia.com/gpu: '1' + command: + - /usr/local/bin/cyborg-entrypoint + - --expand-hosts + - -- + - /usr/local/bin/cyborg + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 volumeMounts: - name: config mountPath: /configs @@ -109,30 +139,30 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1r1-lpu1 - - c1r1-lpu2 - - c1r1-lpu3 - - c1r1-lpu4 - - c1r1-lpu5 - - c1r1-lpu6 - - c1r1-lpu7 - - c1r1-lpu8 - - c1r2-lpu1 - - c1r2-lpu2 - - c1r2-lpu3 - - c1r2-lpu4 - - c1r2-lpu5 - - c1r2-lpu6 - - c1r2-lpu7 - - c1r2-lpu8 - - c1r7-lpu1 - - c1r7-lpu2 - - c1r7-lpu3 - - c1r7-lpu4 - - c1r7-lpu5 - - c1r7-lpu6 - - c1r7-lpu7 - - c1r7-lpu8 + - lpu-node-01 + - lpu-node-02 + - lpu-node-03 + - lpu-node-04 + - lpu-node-05 + - lpu-node-06 + - lpu-node-07 + - lpu-node-08 + - lpu-node-09 + - lpu-node-10 + - lpu-node-11 + - lpu-node-12 + - lpu-node-13 + - lpu-node-14 + - lpu-node-15 + - lpu-node-16 + - lpu-node-17 + - lpu-node-18 + - lpu-node-19 + - lpu-node-20 + - lpu-node-21 + - lpu-node-22 + - lpu-node-23 + - lpu-node-24 containers: - args: - -c @@ -142,8 +172,37 @@ spec: ${i}\"\n break\n fi\ndone\n: \"${part:?no partition row for Grove pod index ${GROVE_PCLQ_POD_INDEX}}\"\nexec\ \ agent-hybrid-v2 \\\n --assemble-dir \"${GAS_DIR}\" \\\n --part \"${part}\" \\\n --port \"${RDMA_PORT}\" \\\ \n --nic \"${NIC_NAME}\" \\\n --metadata-file \"$(mktemp)\" \\\n --readiness-port \"${READINESS_PORT}\"" - image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f + image: example.invalid/lpu-runtime:test name: main + startupProbe: + failureThreshold: 60 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 6 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + env: + - name: GLUE_RDMA_PATH_MTU + value: '1024' + - name: NIC_NAME + value: mlx5_0 + - name: UCX_TLS + value: tcp + - name: RDMA_PORT + value: '19877' + - name: READINESS_PORT + value: '19878' volumeMounts: - mountPath: /nfs name: model-storage @@ -173,7 +232,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1v2-b300n1-nic1 + - gpu-node-0 containers: - args: - --model diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index beebf1ccfcc9..46112c875d26 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -233,30 +233,30 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1r1-lpu1 - - c1r1-lpu2 - - c1r1-lpu3 - - c1r1-lpu4 - - c1r1-lpu5 - - c1r1-lpu6 - - c1r1-lpu7 - - c1r1-lpu8 - - c1r2-lpu1 - - c1r2-lpu2 - - c1r2-lpu3 - - c1r2-lpu4 - - c1r2-lpu5 - - c1r2-lpu6 - - c1r2-lpu7 - - c1r2-lpu8 - - c1r7-lpu1 - - c1r7-lpu2 - - c1r7-lpu3 - - c1r7-lpu4 - - c1r7-lpu5 - - c1r7-lpu6 - - c1r7-lpu7 - - c1r7-lpu8 + - lpu-node-01 + - lpu-node-02 + - lpu-node-03 + - lpu-node-04 + - lpu-node-05 + - lpu-node-06 + - lpu-node-07 + - lpu-node-08 + - lpu-node-09 + - lpu-node-10 + - lpu-node-11 + - lpu-node-12 + - lpu-node-13 + - lpu-node-14 + - lpu-node-15 + - lpu-node-16 + - lpu-node-17 + - lpu-node-18 + - lpu-node-19 + - lpu-node-20 + - lpu-node-21 + - lpu-node-22 + - lpu-node-23 + - lpu-node-24 containers: - args: - -c @@ -333,7 +333,7 @@ spec: name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 - name: UCX_TLS value: tcp - image: us-docker.pkg.dev/nv-hwlpu-20260103060253/lpu-monorepo/agent-hybrid-v2:steved-1node.dfefd5a16f + image: example.invalid/lpu-runtime:test name: agent readinessProbe: failureThreshold: 6 @@ -456,12 +456,9 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1v2-b300n1-nic1 + - gpu-node-0 containers: - args: - - --expand-hosts - - -- - - /usr/local/bin/cyborg - --dynamo-model-name - meta-llama/Llama-3.2-1B-Instruct - --model-card @@ -473,6 +470,9 @@ spec: - "16" command: - /usr/local/bin/cyborg-entrypoint + - --expand-hosts + - -- + - /usr/local/bin/cyborg env: - name: CYBORG_BATCH_SIZE value: "1" @@ -543,7 +543,7 @@ spec: name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a - name: TOTAL_REPLICAS value: "1" - image: us-docker.pkg.dev/nv-hwlpu-20260103060253/cyborg/cyborg:build-862 + image: example.invalid/gpu-runtime:test livenessProbe: failureThreshold: 1 httpGet: @@ -773,7 +773,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - c1v2-b300n1-nic1 + - gpu-node-0 containers: - args: - --model diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index dc7f74d22934..2183b8eddec9 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -29,9 +29,38 @@ spec: - name: main image: example.invalid/gpu-runtime:test command: + - /usr/local/bin/cyborg-entrypoint + - --expand-hosts + - -- - /opt/gpu-runtime args: - serve + env: + - name: RDMA_PORT + value: '19877' + - name: SERVER_HOSTS_FILE + value: /tmp/lpu_servers + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 volumeMounts: - name: config mountPath: /configs @@ -60,6 +89,35 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + startupProbe: + failureThreshold: 60 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 6 + httpGet: + path: /readyz + port: 19878 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + env: + - name: GLUE_RDMA_PATH_MTU + value: '1024' + - name: NIC_NAME + value: mlx5_0 + - name: UCX_TLS + value: tcp + - name: RDMA_PORT + value: '19877' + - name: READINESS_PORT + value: '19878' volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 889c5a52c696..51e889ece842 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -265,12 +265,12 @@ spec: podSpec: containers: - args: - - --expand-hosts - - -- - - /opt/gpu-runtime - serve command: - /usr/local/bin/cyborg-entrypoint + - --expand-hosts + - -- + - /opt/gpu-runtime env: - name: CYBORG_BATCH_SIZE value: "1" diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 77033300c345..5dc27dc9dc3f 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -28,10 +28,87 @@ spec: - /opt/dynamo-lpu args: - serve + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/dynamo-lpu-ssh/private.key + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpi-use-tcp + - --agent-disable-auto-restart + - --agent-server-max-retries + - '60' + - --agent-server-retry-sleep-dur + - 2s + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --store-backend + - kubernetes + - --allocation + - $(LPX_ALLOCATION) + env: + - name: OMPI_ALLOW_RUN_AS_ROOT + value: '1' + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: '1' + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + readinessProbe: + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - {name: model-storage, mountPath: /nfs} - {name: config, mountPath: /configs} - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + initContainers: + - args: + - -m + - '0600' + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true volumes: - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} @@ -49,6 +126,17 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + env: + - name: LPU_RUN_SYSTEM_INIT + value: 'true' volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 38637a1c3f57..014544e1569b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -239,8 +239,11 @@ spec: containers: - args: - serve - - --agent-path - - /bin/agent + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key - --agent-wd - /tmp - --conductor-bind-host @@ -255,21 +258,12 @@ spec: - /configs/model_config.toml - --json-logs - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - - --allocation - - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" - --agent-server-retry-sleep-dur - 2s - - --controlplane-start-port - - "12345" - --graceful-shutdown-timeout - 0s - --response-timeout @@ -278,10 +272,10 @@ spec: - dynamo - --endpoint-id - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - --store-backend - kubernetes + - --allocation + - $(LPX_ALLOCATION) command: - /opt/dynamo-lpu env: @@ -297,6 +291,8 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address + - name: LPX_ALLOCATION + value: agt - name: NATS_SERVER value: nats-address - name: OMPI_ALLOW_RUN_AS_ROOT diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index 54851b187fb2..d19ffb8c9c52 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -19,7 +19,7 @@ spec: buildId: node-local-v2-connected roles: - name: agent - podTemplate: + podTemplate: &agent-template spec: containers: - name: main @@ -27,6 +27,21 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + command: + - /bin/quasar-entrypoint + args: + - --partition-metadata + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + env: + - name: LPU_RUN_SYSTEM_INIT + value: 'true' volumeMounts: - mountPath: /nfs name: model-storage @@ -56,37 +71,93 @@ spec: containers: - name: main image: example.invalid/lpu-runtime:test + command: + - /bin/nova + args: + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/dynamo-lpu-ssh/private.key + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpi-use-tcp + - --agent-disable-auto-restart + - --agent-server-max-retries + - '60' + - --agent-server-retry-sleep-dur + - 2s + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --store-backend + - kubernetes + - --allocation + - $(LPX_ALLOCATION) + env: + - name: OMPI_ALLOW_RUN_AS_ROOT + value: '1' + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: '1' + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + readinessProbe: + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - {name: model-storage, mountPath: /nfs} - {name: config, mountPath: /configs} - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + initContainers: + - args: + - -m + - '0600' + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true volumes: - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - {name: single-v2-ssh-key, emptyDir: {}} - name: agent - podTemplate: - spec: - containers: - - name: main - image: example.invalid/lpu-runtime:test - resources: - limits: - lpu.nvidia.com/devices: '1' - volumeMounts: - - mountPath: /nfs - name: model-storage - - name: config - mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - - {name: hugepages, mountPath: /dev/hugepages} - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + podTemplate: *agent-template diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index e06d1f8a99f4..10ac016bdf09 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -601,8 +601,11 @@ spec: podSpec: containers: - args: - - --agent-path - - /bin/agent + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key - --agent-wd - /tmp - --conductor-bind-host @@ -617,21 +620,12 @@ spec: - /configs/model_config.toml - --json-logs - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - - --allocation - - agt0:agt1:agt2 - --agent-disable-auto-restart - --agent-server-max-retries - "60" - --agent-server-retry-sleep-dur - 2s - - --controlplane-start-port - - "12345" - --graceful-shutdown-timeout - 0s - --response-timeout @@ -640,10 +634,10 @@ spec: - dynamo - --endpoint-id - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - --store-backend - kubernetes + - --allocation + - $(LPX_ALLOCATION) command: - /bin/nova env: @@ -659,6 +653,8 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address + - name: LPX_ALLOCATION + value: agt0:agt1:agt2 - name: NATS_SERVER value: nats-address - name: OMPI_ALLOW_RUN_AS_ROOT diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index b3574c680630..98ebaedff285 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -32,6 +32,27 @@ spec: - /opt/gpu-runtime args: - serve + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 volumeMounts: - {name: model-storage, mountPath: /nfs} resources: @@ -46,11 +67,41 @@ spec: - name: main image: example.invalid/lpu-runtime:test args: - - --instance-model-name - - test/v3-hybrid + - start resources: limits: lpu.nvidia.com/devices: '1' + command: + - /bin/hydra-entrypoint + startupProbe: + exec: + command: + - /bin/hydra-entrypoint + - ready + failureThreshold: 240 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + exec: + command: + - /bin/hydra-entrypoint + - ready + failureThreshold: 6 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + env: + - name: GLUE_RDMA_PATH_MTU + value: '1024' + - name: NIC_NAME + value: mlx5_0 + - name: UCX_TLS + value: tcp + - name: RDMA_PORT + value: '19877' + - name: READINESS_PORT + value: '19878' volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index c470c2eaf7eb..586034a4bb74 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -27,10 +27,127 @@ spec: args: - --instance-model-name - test/llama32-1b-lp30 + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/dynamo-lpu-ssh/private.key + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpi-use-tcp + - --agent-disable-auto-restart + - --agent-server-max-retries + - '60' + - --agent-server-retry-sleep-dur + - 2s + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --store-backend + - kubernetes + - --allocation + - $(LPX_ALLOCATION) + - --agent-env-vars + - BMC_SSH_ENABLED=false + - --agent-env-vars + - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 + - --agent-env-vars + - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false + - --agent-env-vars + - GROQ_HAL_V2_BACKGROUND_METRICS=false + - --agent-env-vars + - GROQ_PG613_PORT_CONFIG=EB3 + - --agent-env-vars + - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false + - --agent-env-vars + - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 + - --agent-env-vars + - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 + - --agent-env-vars + - GROQ_V2_RESET_ON_OPEN=0 + - --agent-env-vars + - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false + - --agent-env-vars + - LPU_FPGA_RECONFIGURE_METHOD=cvp + - --agent-env-vars + - LPU_PARTITION_RUNNER_DATAPLANE=pcie + - --agent-env-vars + - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 + - --agent-env-vars + - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 + - --agent-env-vars + - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true + - --agent-env-vars + - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 + - --agent-env-vars + - REDFISH_BMC_ENABLED=false + command: + - /bin/nova + env: + - name: OMPI_ALLOW_RUN_AS_ROOT + value: '1' + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: '1' + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + readinessProbe: + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - {name: model-storage, mountPath: /nfs} - {name: config, mountPath: /configs} - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + initContainers: + - args: + - -m + - '0600' + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true volumes: - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} @@ -41,12 +158,50 @@ spec: containers: - name: main image: example.invalid/lpu-runtime:test - args: - - --instance-model-name - - test/llama32-1b-lp30 + args: [] resources: limits: lpu.nvidia.com/devices: '1' + command: + - /bin/quasar-entrypoint + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + env: + - name: BMC_SSH_ENABLED + value: 'false' + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: 'false' + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: 'false' + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: '0' + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: '0' + - name: GROQ_V2_RESET_ON_OPEN + value: '0' + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: 'false' + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: REDFISH_BMC_ENABLED + value: 'false' + securityContext: + privileged: true + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - mountPath: /nfs name: model-storage @@ -55,6 +210,26 @@ spec: - {name: host-dev, mountPath: /dev} - {name: host-sys, mountPath: /sys} - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev volumes: - name: model-storage persistentVolumeClaim: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index a2cf685e1376..5648761b2e72 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -265,8 +265,11 @@ spec: - args: - --instance-model-name - test/llama32-1b-lp30 - - --agent-path - - /bin/agent + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key - --agent-wd - /tmp - --conductor-bind-host @@ -281,21 +284,12 @@ spec: - /configs/model_config.toml - --json-logs - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - - --allocation - - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" - --agent-server-retry-sleep-dur - 2s - - --controlplane-start-port - - "12345" - --graceful-shutdown-timeout - 0s - --response-timeout @@ -304,10 +298,10 @@ spec: - dynamo - --endpoint-id - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - --store-backend - kubernetes + - --allocation + - $(LPX_ALLOCATION) - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars @@ -361,6 +355,8 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address + - name: LPX_ALLOCATION + value: agt - name: NATS_SERVER value: nats-address - name: OMPI_ALLOW_RUN_AS_ROOT diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index b0dcd7136e32..32d2f60004ae 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -19,17 +19,55 @@ spec: buildId: node-local-v3-hx-sd-draft roles: - name: agent - podTemplate: + podTemplate: &agent-template spec: containers: - name: main image: example.invalid/lpu-runtime:test - args: - - --instance-model-name - - test/v3-specdecode + args: [] resources: limits: lpu.nvidia.com/devices: '1' + command: + - /bin/quasar-entrypoint + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 + env: + - name: BMC_SSH_ENABLED + value: 'false' + - name: GROQ_HAL_V2_BACKGROUND_METRICS + value: 'false' + - name: GROQ_PG613_PORT_CONFIG + value: EB3 + - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 + value: 'false' + - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE + value: '0' + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 + value: 0000:9a:00.0 + - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 + value: 0000:7e:00.0 + - name: GROQ_V2_HEALTH_CHECK_ON_OPEN + value: '0' + - name: GROQ_V2_RESET_ON_OPEN + value: '0' + - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY + value: 'false' + - name: LPU_FPGA_RECONFIGURE_METHOD + value: cvp + - name: REDFISH_BMC_ENABLED + value: 'false' + securityContext: + privileged: true + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - name: model-storage mountPath: /nfs @@ -38,6 +76,26 @@ spec: - {name: host-dev, mountPath: /dev} - {name: host-sys, mountPath: /sys} - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + initContainers: + - command: + - /bin/lp30-agent-v2-init + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: example.invalid/lpu-runtime:test + name: prepare-lp30 + securityContext: + privileged: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /sys + name: host-sys + - mountPath: /dev + name: host-dev volumes: - name: model-storage persistentVolumeClaim: @@ -60,38 +118,130 @@ spec: args: - --instance-model-name - test/v3-specdecode + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/dynamo-lpu-ssh/private.key + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpi-use-tcp + - --agent-disable-auto-restart + - --agent-server-max-retries + - '60' + - --agent-server-retry-sleep-dur + - 2s + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --store-backend + - kubernetes + - --allocation + - $(LPX_ALLOCATION) + - --agent-env-vars + - BMC_SSH_ENABLED=false + - --agent-env-vars + - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 + - --agent-env-vars + - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false + - --agent-env-vars + - GROQ_HAL_V2_BACKGROUND_METRICS=false + - --agent-env-vars + - GROQ_PG613_PORT_CONFIG=EB3 + - --agent-env-vars + - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false + - --agent-env-vars + - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 + - --agent-env-vars + - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 + - --agent-env-vars + - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 + - --agent-env-vars + - GROQ_V2_RESET_ON_OPEN=0 + - --agent-env-vars + - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false + - --agent-env-vars + - LPU_FPGA_RECONFIGURE_METHOD=cvp + - --agent-env-vars + - LPU_PARTITION_RUNNER_DATAPLANE=pcie + - --agent-env-vars + - LPU_PARTITION_RUNNER_PCIE_SOURCE_PORT=33000 + - --agent-env-vars + - LPU_RUNNER_OUTSTANDING_TASK_DRAIN_TIMEOUT_SECONDS=10 + - --agent-env-vars + - LPU_VSAP_DIB_BARRIER_BEFORE_PUSH_ENABLED=true + - --agent-env-vars + - LPU_VSAP_MAX_TENSORS_PER_CHUNK=4 + - --agent-env-vars + - REDFISH_BMC_ENABLED=false + command: + - /bin/nova + env: + - name: OMPI_ALLOW_RUN_AS_ROOT + value: '1' + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: '1' + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + readinessProbe: + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - {name: model-storage, mountPath: /nfs} - {name: config, mountPath: /configs} - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + initContainers: + - args: + - -m + - '0600' + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true volumes: - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - {name: single-v2-ssh-key, emptyDir: {}} - name: agent - podTemplate: - spec: - containers: - - name: main - image: example.invalid/lpu-runtime:test - args: - - --instance-model-name - - test/v3-specdecode - resources: - limits: - lpu.nvidia.com/devices: '1' - volumeMounts: - - name: model-storage - mountPath: /nfs - - name: config - mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - volumes: - - name: model-storage - persistentVolumeClaim: - claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + podTemplate: *agent-template diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index f60b2710c1df..7250fd4ecfed 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -681,8 +681,11 @@ spec: - args: - --instance-model-name - test/v3-specdecode - - --agent-path - - /bin/agent + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key - --agent-wd - /tmp - --conductor-bind-host @@ -697,21 +700,12 @@ spec: - /configs/model_config.toml - --json-logs - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - - --allocation - - agt0:agt1:agt2 - --agent-disable-auto-restart - --agent-server-max-retries - "60" - --agent-server-retry-sleep-dur - 2s - - --controlplane-start-port - - "12345" - --graceful-shutdown-timeout - 0s - --response-timeout @@ -720,10 +714,10 @@ spec: - dynamo - --endpoint-id - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - --store-backend - kubernetes + - --allocation + - $(LPX_ALLOCATION) - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars @@ -777,6 +771,8 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address + - name: LPX_ALLOCATION + value: agt0:agt1:agt2 - name: NATS_SERVER value: nats-address - name: OMPI_ALLOW_RUN_AS_ROOT diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml index 5fa55fbc25cc..bef87e350737 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml @@ -33,20 +33,98 @@ spec: podTemplate: spec: containers: - - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + - image: example.invalid/lpu-runtime:test name: main args: - --agent-env-vars - EXTRA_RUNTIME_KNOB + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/dynamo-lpu-ssh/private.key + - --agent-wd + - /tmp + - --conductor-bind-host + - 0.0.0.0 + - --conductor-hostname + - $(POD_IP) + - --control-server-addr + - $(POD_IP):51045 + - --datacenter-config-filepath + - /configs/datacenter.toml + - --model-config-filepath + - /configs/model_config.toml + - --json-logs + - --disable-tracing + - --mpi-use-tcp + - --agent-disable-auto-restart + - --agent-server-max-retries + - '60' + - --agent-server-retry-sleep-dur + - 2s + - --graceful-shutdown-timeout + - 0s + - --response-timeout + - 120s + - --frontend-type + - dynamo + - --endpoint-id + - dyn://$(DYN_NAMESPACE).backend.generate + - --store-backend + - kubernetes + - --allocation + - $(LPX_ALLOCATION) env: - name: MPI_BTL_TCP_IF_INCLUDE - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: OMPI_MCA_btl_tcp_if_include - value: 7.241.144.0/24 + value: 192.0.2.0/24 + - name: OMPI_ALLOW_RUN_AS_ROOT + value: '1' + - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM + value: '1' + - name: OMPI_MCA_plm_rsh_args + value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key + command: + - /bin/nova + readinessProbe: + httpGet: + path: /readyz + port: 51045 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false volumeMounts: - {name: model-storage, mountPath: /nfs} - {name: config, mountPath: /configs} - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + initContainers: + - args: + - -m + - '0600' + - /ssh-pk/private.key + - /tmp/dynamo-lpu-ssh/private.key + command: + - /bin/install + image: example.invalid/lpu-runtime:test + name: prepare-ssh-key + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - mountPath: /tmp/dynamo-lpu-ssh + name: single-v2-ssh-key + - mountPath: /ssh-pk + name: ssh-secret + readOnly: true volumes: - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} @@ -55,16 +133,27 @@ spec: podTemplate: spec: containers: - - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + - image: example.invalid/lpu-runtime:test name: main args: - - --agent-env-vars - - EXTRA_RUNTIME_KNOB + - --partition-metadata env: - name: MPI_BTL_TCP_IF_INCLUDE - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: OMPI_MCA_btl_tcp_if_include - value: 7.241.144.0/24 + value: 192.0.2.0/24 + - name: LPU_RUN_SYSTEM_INIT + value: 'true' + command: + - /bin/quasar-entrypoint + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 2222 + timeoutSeconds: 1 volumeMounts: - mountPath: /nfs name: model-storage diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml index 641e793e44c1..b7ff682b85f8 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml @@ -190,11 +190,11 @@ spec: - name: LPU_RUN_SYSTEM_INIT value: "true" - name: MPI_BTL_TCP_IF_INCLUDE - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: NATS_SERVER value: nats-address - name: OMPI_MCA_btl_tcp_if_include - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: POD_IP valueFrom: fieldRef: @@ -212,7 +212,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + image: example.invalid/lpu-runtime:test name: agent readinessProbe: failureThreshold: 3 @@ -320,8 +320,11 @@ spec: - args: - --agent-env-vars - EXTRA_RUNTIME_KNOB - - --agent-path - - /bin/agent + - --mpirun-path + - /bin/mpirun + - --mpi-ssh-cmd + - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -i /tmp/dynamo-lpu-ssh/private.key - --agent-wd - /tmp - --conductor-bind-host @@ -336,21 +339,12 @@ spec: - /configs/model_config.toml - --json-logs - --disable-tracing - - --mpirun-path - - /bin/mpirun - - --mpi-ssh-cmd - - ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - -i /tmp/dynamo-lpu-ssh/private.key - --mpi-use-tcp - - --allocation - - agt - --agent-disable-auto-restart - --agent-server-max-retries - "60" - --agent-server-retry-sleep-dur - 2s - - --controlplane-start-port - - "12345" - --graceful-shutdown-timeout - 0s - --response-timeout @@ -359,10 +353,10 @@ spec: - dynamo - --endpoint-id - dyn://$(DYN_NAMESPACE).backend.generate - - --request-plane - - tcp - --store-backend - kubernetes + - --allocation + - $(LPX_ALLOCATION) command: - /bin/nova env: @@ -378,8 +372,10 @@ spec: value: inference-engine - name: ETCD_ENDPOINTS value: etcd-address + - name: LPX_ALLOCATION + value: agt - name: MPI_BTL_TCP_IF_INCLUDE - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: NATS_SERVER value: nats-address - name: OMPI_ALLOW_RUN_AS_ROOT @@ -387,7 +383,7 @@ spec: - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM value: "1" - name: OMPI_MCA_btl_tcp_if_include - value: 7.241.144.0/24 + value: 192.0.2.0/24 - name: OMPI_MCA_plm_rsh_args value: -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentityFile=/tmp/dynamo-lpu-ssh/private.key @@ -408,7 +404,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + image: example.invalid/lpu-runtime:test name: conductor readinessProbe: failureThreshold: 3 @@ -442,7 +438,7 @@ spec: - /tmp/dynamo-lpu-ssh/private.key command: - /bin/install - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 + image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} securityContext: diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml b/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml deleted file mode 100644 index 7efbfe2940fb..000000000000 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2_manifest_defaults.input.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler - generation: 1 - name: dgd-single-v2-manifest-defaults - namespace: inference-engine - uid: 22222222-2222-4222-8222-222222222222 -spec: - components: - - name: lpx - type: lpx - replicas: 1 - lpx: - buildId: single-v2-manifest-defaults/build_manifest_defaults - roles: - - name: conductor - podTemplate: - spec: - containers: - - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 - name: main - volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} - volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} - - name: agent - podTemplate: - spec: - containers: - - image: nvcr.io/nvidia/ai-dynamo/lpu-runtime:v2 - name: main - volumeMounts: - - mountPath: /nfs - name: model-storage - - name: config - mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - - {name: hugepages, mountPath: /dev/hugepages} - volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 26fb3229e8d7..484f3b398a74 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -393,6 +393,17 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( )...) } + // Conversion preserves alpha service keys in ComponentName; report conductor errors at their source paths. + conductorComponentsPath := componentsPath + componentPath := componentsPath.Index + if v.hasRuntimeVersionSource(runtimeVersionSourceV1Alpha1) { + conductorComponentsPath = fldPath.Child("services") + componentPath = func(index int) *field.Path { + return conductorComponentsPath.Key(spec.Components[index].ComponentName) + } + } + allErrs = append(allErrs, dynamolpx.ValidateConductorRoles(spec, conductorComponentsPath, componentPath)...) + if spec.Restart != nil { allErrs = append(allErrs, v.validateRestart(spec.Restart, fldPath.Child("restart"), components)...) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 6cd824e91cf1..1051ca3f1bbb 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -15,15 +15,18 @@ import ( // lpxDGDAdmissionCases builds fresh LPX scenarios for the single native DGD admission table. func lpxDGDAdmissionCases() []dgdAdmissionTestCase { const longLPXComponentName = "abcdefghijklmnopqrstuvwxyzabcd" + const conductorRoleErr = "spec.components: Forbidden: LPX components must declare exactly one conductor role" + const conductorTemplateErr = "spec.components[0].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" + const alphaConductorRoleErr = "spec.services: Forbidden: LPX components must declare exactly one conductor role" + const alphaConductorTemplateErr = "spec.services[lpx].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" // Keep LPX inputs and oracles together without a separate admission execution path. return []dgdAdmissionTestCase{ { - name: "singleton LPX with implicit conductor preserves omitted replicas", + name: "singleton LPX with explicit conductor preserves omitted replicas", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { component := &dgd.Spec.Components[0] component.Replicas = nil - component.Roles = component.Roles[:1] component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, @@ -144,7 +147,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "LPX shell allocation target validation is deferred to the child controller", + name: "LPX permits template-owned shell commands", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { container := &dgd.Spec.Components[0].Roles[0].PodTemplate.Spec.Containers[0] container.Command = []string{"/usr/bin/env"} @@ -166,13 +169,17 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles[1].Name = "unknown" }), - wantWebhookErrs: []string{`spec.components[0].roles[1].name: Unsupported value: "unknown": supported values: "conductor", "agent"`}, + wantWebhookErrs: []string{ + `spec.components[0].roles[1].name: Unsupported value: "unknown": supported values: "conductor", "agent"`, + conductorRoleErr, + }, }, { - name: "v1alpha1 LPX image errors aggregate at the authored role indices", + name: "v1alpha1 LPX image and conductor template errors aggregate at the authored role indices", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { component := dgd.Spec.Services["lpx"] component.Roles[0], component.Roles[1] = component.Roles[1], component.Roles[0] + component.Roles[0].PodTemplate = nil spec := &component.Roles[1].PodTemplate.Spec spec.Containers = append(spec.Containers, corev1.Container{Name: "sidecar"}) spec.InitContainers = []corev1.Container{{Name: "init"}} @@ -180,6 +187,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{ "spec.components[0].roles[1].podTemplate.spec.containers[1].image: Required value: must specify a non-empty image", "spec.components[0].roles[1].podTemplate.spec.initContainers[0].image: Required value: must specify a non-empty image", + "spec.services[lpx].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate", }, }, { @@ -203,7 +211,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }, }, { - name: "LPX defers implicit conductor name checks until build mode is known", + name: "LPX rejects a missing conductor role on CREATE despite a conductor-named sidecar", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { component := &dgd.Spec.Components[0] component.Roles = component.Roles[:1] @@ -212,9 +220,10 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { corev1.Container{Name: "conductor", Image: "sidecar"}, ) }), + wantWebhookErrs: []string{conductorRoleErr}, }, { - name: "v1alpha1 LPX defers inherited conductor init names until build mode is known", + name: "v1alpha1 LPX allows an Agent init container named conductor with an explicit conductor template", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { component := dgd.Spec.Services["lpx"] component.Roles[0].PodTemplate.Spec.InitContainers = []corev1.Container{{Name: "conductor", Image: "setup"}} @@ -302,6 +311,106 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { dgd.Spec.Services["lpx"].LPX.BuildID = " " }), }, + // Conductor startup is explicit on both source API versions and on every update. + { + name: "LPX rejects a missing conductor template on CREATE", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[1].PodTemplate = nil + }), + wantWebhookErrs: []string{conductorTemplateErr}, + }, + { + name: "v1alpha1 LPX rejects a missing conductor role on CREATE", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + component := dgd.Spec.Services["lpx"] + component.Roles = component.Roles[:1] + }), + wantWebhookErrs: []string{alphaConductorRoleErr}, + }, + { + name: "v1alpha1 LPX rejects a missing conductor template on CREATE", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil + }), + wantWebhookErrs: []string{alphaConductorTemplateErr}, + }, + { + name: "v1alpha1 LPX conductor template error identifies the service key and authored role index", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, k8sptr.To(int32(1))) + target := dgd.Spec.Services["target"] + target.Roles[0], target.Roles[1] = target.Roles[1], target.Roles[0] + target.Roles[0].PodTemplate = nil + }), + wantWebhookErrs: []string{"spec.services[target].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate"}, + }, + { + name: "LPX rejects removing the conductor template on UPDATE", + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[1].PodTemplate = nil + }), + wantWebhookErrs: []string{conductorTemplateErr}, + }, + { + name: "v1alpha1 LPX rejects removing the conductor role on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + component := dgd.Spec.Services["lpx"] + component.Roles = component.Roles[:1] + }), + wantWebhookErrs: []string{alphaConductorRoleErr}, + }, + { + name: "v1alpha1 LPX rejects removing the conductor template on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil + }), + wantWebhookErrs: []string{alphaConductorTemplateErr}, + }, + { + name: "LPX rejects retaining an absent conductor template on an unrelated UPDATE", + seedWithoutWebhook: true, + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[1].PodTemplate = nil + }), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Roles[1].PodTemplate = nil + dgd.Spec.Components[0].Replicas = k8sptr.To(int32(3)) + }), + wantWebhookErrs: []string{conductorTemplateErr}, + }, + { + name: "v1alpha1 LPX rejects retaining an absent conductor template on an unrelated UPDATE", + seedWithoutWebhook: true, + oldDeployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil + }), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil + dgd.Spec.Services["lpx"].Replicas = k8sptr.To(int32(3)) + }), + wantWebhookErrs: []string{alphaConductorTemplateErr}, + }, + { + name: "LPX admits SpecDecode UPDATE with one conductor and Agent-only draft", + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(1))) + }), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(2))) + }), + }, + { + name: "v1alpha1 LPX admits SpecDecode UPDATE with one conductor and Agent-only draft", + oldDeployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, k8sptr.To(int32(1))) + }), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, k8sptr.To(int32(2))) + }), + }, // Selected LPX workload shapes. { name: "LPX roles reject provider overrides", @@ -311,11 +420,12 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{"spec.components[0].roles[0].providerOverride: Forbidden: LPX roles do not support provider overrides"}, }, { - name: "LPX can remove an explicit conductor and retain its implicit fallback", + name: "LPX rejects removing the conductor role on UPDATE", oldDeployment: betaLPXDGDForAdmission(nil), deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles = dgd.Spec.Components[0].Roles[:1] }), + wantWebhookErrs: []string{conductorRoleErr}, }, { name: "LPX provider override support on update is deferred to the child controller", @@ -398,14 +508,20 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Roles = nil }), - wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "agent" role`}, + wantWebhookErrs: []string{ + `spec.components[0].roles: Required value: must contain the "agent" role`, + conductorRoleErr, + }, }, { name: "v1alpha1 lpx requires an agent role", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Roles = nil }), - wantWebhookErrs: []string{`spec.components[0].roles: Required value: must contain the "agent" role`}, + wantWebhookErrs: []string{ + `spec.components[0].roles: Required value: must contain the "agent" role`, + alphaConductorRoleErr, + }, }, { name: "v1alpha1 lpx requires lpx", @@ -423,12 +539,13 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantCELErr: "spec.services[lpx]: Invalid value: lpx may only be set when componentType is lpx", }, { - name: "LPX shared leader composition is deferred to the child controller", + name: "LPX rejects more than one conductor across components", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { second := dgd.Spec.Components[0] second.ComponentName = "lpx-2" dgd.Spec.Components = append(dgd.Spec.Components, second) }), + wantWebhookErrs: []string{conductorRoleErr}, }, { name: "shared LPX draft does not consume the target Grove name budget", @@ -444,12 +561,13 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "implicit LPX conductor name overflow validation is deferred to the child controller", + name: "LPX rejects a missing conductor role independently of component name length", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { component := &dgd.Spec.Components[0] component.ComponentName = longLPXComponentName component.Roles = component.Roles[:1] }), + wantWebhookErrs: []string{conductorRoleErr}, }, // Ordinary components have no LPX child to reject controller-owned scheduler selection. { @@ -496,7 +614,9 @@ func betaLPXDGDForAdmission( {Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, }}}, - {Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, + {Name: nvidiacomv1beta1.ComponentRoleLPXConductor, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "conductor-runtime", Command: []string{"/bin/nova"}}}, + }}}, }, }, } @@ -520,7 +640,9 @@ func alphaLPXDGDForAdmission( {Name: nvidiacomv1alpha1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "lpu-runtime"}}, }}}, - {Name: nvidiacomv1alpha1.ComponentRoleLPXConductor}, + {Name: nvidiacomv1alpha1.ComponentRoleLPXConductor, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "conductor-runtime", Command: []string{"/bin/nova"}}}, + }}}, }, }, } diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index abe8b7b8a3c8..efb1e782ad9c 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -257,7 +257,7 @@ vLLM and SGLang take tensor/pipeline/data parallelism as CLI flags (covered in t You normally do not need to set the top-level `spec.backendFramework` field — the operator infers the backend from the worker command. Set it explicitly (`vllm`, `sglang`, or `trtllm`) only when a feature needs the framework known up front, such as GMS failover or multinode TensorRT-LLM. -For an XT LPU-only deployment, declare separate Agent and Nova conductor templates in a `type: lpx` component: +For LPX deployments, declare independent `agent` and `conductor` role templates. Every LPX graph requires exactly one explicit conductor template, including a graph with a single LPX component. This fragment shows the role structure only; the placeholder images use their own entrypoints: ```yaml spec: @@ -272,72 +272,20 @@ spec: spec: containers: - name: main - image: - volumeMounts: - - name: config - mountPath: /configs - - name: host-dev - mountPath: /dev - - name: host-sys - mountPath: /sys - - name: hugepages - mountPath: /dev/hugepages - - name: ssh-secret - mountPath: /ssh-pk - readOnly: true - - name: model-storage - mountPath: /models - volumes: - - name: host-dev - hostPath: - path: /dev - type: Directory - - name: host-sys - hostPath: - path: /sys - type: Directory - - name: hugepages - emptyDir: - medium: HugePages - - name: ssh-secret - secret: - secretName: mpi-run-ssh-secret - defaultMode: 0644 - - name: model-storage - persistentVolumeClaim: - claimName: model-storage + image: - name: conductor podTemplate: spec: containers: - name: main - image: - volumeMounts: - - name: config - mountPath: /configs - - name: single-v2-ssh-key - mountPath: /tmp/dynamo-lpu-ssh - - name: model-storage - mountPath: /models - volumes: - - name: ssh-secret - secret: - secretName: mpi-run-ssh-secret - defaultMode: 0600 - - name: single-v2-ssh-key - emptyDir: {} - - name: model-storage - persistentVolumeClaim: - claimName: model-storage + image: ``` -Create the `model-storage` PVC in the DGD's namespace with storage accessible to all LPX Pods. Replace `my-model/build` with an XT LPU-only build available through the configured LPX model registry and `` with its compatible runtime image. +For LP20 and LP30, supply compatible builds and images, static arguments and environment settings, init containers, health probes, volumes, and mounts in the role templates. The operator uses the image's entrypoint unless the template supplies `command`; it does not generate runtime launchers, SSH-key preparation, or probes. Hardware policy still replaces the main-container `securityContext` for XT agents and direct-hybrid agents with privileged root settings, discarding other authored container security fields. Direct-hybrid Pods also force root user and group settings. -Declare static volumes and every LPX runtime container mount in its role's `podTemplate`, including the conductor's SSH source and destination volumes. Use the configured MPI SSH Secret name; `mpi-run-ssh-secret` is the chart default. The operator generates the `config` volume with its content-based ConfigMap name and init containers with their own mounts. Keep the `hugepages` volume and mount for XT Agents and HX hybrid Agents; omit both for HX LPU-only and speculative-decoding Agents. For a hybrid engine, configure the hybrid runtime's template, volumes, and mounts on the conductor role. +The operator supplies generated configuration, model and hardware bindings, and the LPU conductor's `LPX_ALLOCATION` value. A Nova conductor must include `--allocation` and `$(LPX_ALLOCATION)` in its authored arguments, along with its other runtime options. Mount the generated `config` volume and model storage where the selected runtime expects them. SSH-based startup must declare its own Secret volume, key preparation, and mounts. -Runtime configuration and scratch files use the container's writable `/tmp`; a separate volume is optional. For a read-only root filesystem, mount a writable `emptyDir` at `/tmp`. The conductor's separate `/tmp/dynamo-lpu-ssh` mount shares its prepared key with the init container and is still required. - -For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only an `agent` role; the target has `conductor` and `agent` roles. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. +For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only an `agent` role; the target has `conductor` and `agent` roles. A hybrid engine places its GPU runtime in the conductor template. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. For per-backend setup and tuning, see [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/overview.md), [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/overview.md), and [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/overview.md). diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index b258d2985a64..0b96df09883e 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -338,7 +338,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -385,7 +385,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -1942,7 +1942,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| @@ -1976,7 +1976,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 1eeb9c07e82e..c4000dc03950 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -61,7 +61,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c DGD-only configuration for a `type: lpx` component; omitted from the standalone DCD schema. Set the required `buildId` and optional build `settings`. See [LPXConfig](full-api-reference.mdx#lpxconfig) for the full field set. - Supported modes are LPU, hybrid, and LPU draft with LPU target. Declare one complete component, or separate speculative draft and target components. The draft has only an `agent` role; the target has `conductor` and `agent` roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. + Supported modes are LPU, hybrid, and LPU draft with LPU target. Every LPX graph requires exactly one explicit `conductor` role and template. A single LPX component declares both `conductor` and `agent` roles. In speculative decoding, the draft has only an `agent` role; the target declares both roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. @@ -73,7 +73,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Pod template for the component's pods. The operator injects its defaults (image, command, env, ports, probes, resources, volume mounts) into the container named `main`, merging your overrides by name. If no `main` container is present, the operator generates one. Every other container is treated as a user-managed sidecar and receives no injected defaults — sidecars must specify their own required fields such as `image`. Replaces the ten separate per-component fields (`resources`, `envs`, `livenessProbe`, and so on) that existed in `v1alpha1`. - For `type: lpx`, omit this field and use `roles[].podTemplate`. Each supplied LPX template requires an explicit `main` container image; see `roles` below for conductor-template fallback. + For `type: lpx`, omit this field and use `roles[].podTemplate`. Every LPX role requires its own template with an explicit `main` container image. core/v1.PodTemplateSpec @@ -91,7 +91,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Named Pod-producing parts of a compound component, keyed by `name`. Multinode components must contain exactly one `leader` and one `worker`; omitted role counts default to `1` and `multinode.nodeCount - 1`, respectively. Omit the field to retain the implicit leader/worker layout. - LPX components use `agent` for Agent Pods and `conductor` for the conductor. With one LPX component, the conductor role is optional. In a speculative pair, the draft has only an agent role and the target declares both roles. Hybrid execution may also create Nova Pods from Agent configuration. + LPX components use `agent` for Agent Pods and `conductor` for the serving runtime. Exactly one LPX component in the graph must declare a conductor role with an explicit template. In a speculative pair, the draft has only an agent role and the target declares both roles. A hybrid engine's GPU runtime uses the conductor template. Standalone DCD roles cannot use LPX templates or DGD-only provider overrides. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). @@ -104,7 +104,9 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Logical cardinality of this role in one complete component instance; minimum `1`. For ordinary multinode components, admission defaults and persists an omitted value as `leader: 1` or `worker: multinode.nodeCount - 1`; an explicit value must match that fixed shape. For LPX, omitted conductor replicas use `1`, subject to runtime-specific limits. Agent replicas are derived from the model build; an explicit count must match the build. - Native Pod template for this role. LPX-only; required for the agent. Each supplied template requires a `main` container with an explicit image. An omitted conductor template reuses the same component's agent template; a single LPX component also uses this fallback when its conductor role is omitted. A speculative pair uses the target's conductor. An explicit conductor template merges with shared defaults independently, without inheriting the agent template. + Native Pod template for this role. LPX-only; required for every declared role. Each template requires a `main` container with an explicit image. Agent and conductor templates are independent. + + For LP20 and LP30, the template owns runtime commands, static arguments and environment settings, init containers, health probes, and startup storage. It may use the image's entrypoint or supply `command`. The operator retains device-access security requirements and supplies generated configuration, model and hardware bindings, and `LPX_ALLOCATION` for the LPU conductor. A Nova template consumes that value through its authored `--allocation` argument. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for security-context overrides, role structure, and runtime examples. core/v1.PodTemplateSpec diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index dea9e5cc8c38..7dd6af346dcd 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -471,7 +471,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. See [ComponentRoleSpec](#componentrolespec). **Validation:** Optional: \{\} @@ -644,7 +644,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} @@ -2744,7 +2744,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. See [ComponentRoleSpec](#v1beta1-componentrolespec). **Validation:** Optional: \{\} @@ -2859,7 +2859,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components require an agent role and may declare a conductor role. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} From fba0382e847491aa3b7ce6766c67ba863ab136ce Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:38:22 +0300 Subject: [PATCH 082/196] fix(operator): relax Cyborg model-storage validation (#520) Signed-off-by: Andrew Paprotsky (cherry picked from commit 16b8913a61c14f5748c6a3dbc2c752bdbc6832ab) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/cyborg_render.go | 3 +-- .../internal/dynamo/lpx/cyborg_render_test.go | 24 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index d294e665a2e6..124aa2cd5dc2 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -13,7 +13,6 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" ) // configureHybridCyborg consumes a fresh hybrid clique from an admitted source @@ -33,7 +32,7 @@ func configureHybridCyborg( if err != nil { return err } - if !apiequality.Semantic.DeepEqual(cyborgStorage.volume, modelStorage.volume) || !apiequality.Semantic.DeepEqual(cyborgStorage.mount, modelStorage.mount) { + if cyborgStorage.mount.MountPath != modelStorage.mount.MountPath { return fmt.Errorf("selected Cyborg podTemplate conflicts with model storage mount %q", modelStorage.mount.MountPath) } cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, cyborg.Spec.Replicas) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 85bfbe4708d2..7045d0b0a5fd 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -112,6 +112,12 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { corev1.EnvVar{Name: "RDMA_PORT", Value: "12345"}, corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, ) + + t.Log("Use independently provisioned Cyborg model storage at the shared runtime path") + decode.Spec.PodSpec.Volumes[0].PersistentVolumeClaim.ClaimName = "cyborg-models" + decode.Spec.PodSpec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + decode.Spec.PodSpec.Containers[0].VolumeMounts[1].SubPath = "cyborg" + decode.Spec.PodSpec.Containers[0].VolumeMounts[1].ReadOnly = true input := RenderInput{ Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, } @@ -138,8 +144,10 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ Name: "RDMA_PORT", Value: "12345", }) - require.Contains(t, cyborg.Spec.PodSpec.Volumes, renderTestPodSpec().Volumes[0]) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, renderTestPodSpec().Containers[0].VolumeMounts[0]) + require.Equal(t, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "cyborg-models", ReadOnly: true}, cyborg.Spec.PodSpec.Volumes[0].PersistentVolumeClaim) + require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "model-storage", MountPath: "/models", SubPath: "cyborg", ReadOnly: true, + }) require.Equal(t, []string{"/custom-cyborg", "--wrapper-option"}, cyborg.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"argument with spaces", "literal $HOME", ""}, cyborg.Spec.PodSpec.Containers[0].Args) @@ -161,23 +169,27 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.Nil(t, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args) - t.Log("Reject incomplete endpoint and fanout coverage of the same split-I/O runtime") + t.Log("Reject invalid Cyborg runtime bindings") for _, test := range []struct { name string replicas int32 + mountPath string wantError string }{ - {"incomplete endpoints", 1, "Cyborg replicas 1 must be divisible by ioFpgaCount 2"}, - {"incomplete fanout", 2, "Cyborg replicas 2 must provide fanoutFactor 2 clients"}, + {"incomplete endpoints", 1, "/models", "Cyborg replicas 1 must be divisible by ioFpgaCount 2"}, + {"incomplete fanout", 2, "/models", "Cyborg replicas 2 must provide fanoutFactor 2 clients"}, + {"different storage path", 4, "/other-models", "conflicts with model storage mount"}, + {"empty storage path", 4, "", "has no mount path"}, } { t.Run(test.name, func(t *testing.T) { - t.Log("Reject incomplete Cyborg coverage during runtime rendering") + t.Log("Reject incompatible runtime bindings during rendering") pcs := renderTestPCS(true) cyborg := namedClique(t, pcs, "cond") cyborg.Spec.Replicas = test.replicas cyborg.Spec.MinAvailable = ptr.To(test.replicas) cyborg.Spec.PodSpec.Containers[0].Command = []string{"/usr/local/bin/dynamo_main"} + cyborg.Spec.PodSpec.Containers[0].VolumeMounts[1].MountPath = test.mountPath _, err := renderSelectedForTest(pcs, []*ModelProjection{projection}, RenderInput{ Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, }) From 43b5eba4178c71f43ad09abd4c68a0454cf6e2a7 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:19:25 -0700 Subject: [PATCH 083/196] refactor(lpx)!: move ownership of LpuPipelineRequest to PodCliqueSet (#436) (cherry picked from commit 36c48236d8c4b80bab075958bb4f7ed80e986123) Signed-off-by: Andrew Paprotsky --- .../components/operator/files/role.yaml | 9 - .../platform/tests/lpx_config_test.yaml | 2 - .../tests/validate_removed_values_test.yaml | 1 - deploy/operator/api/config/v1alpha1/lpx.go | 4 +- deploy/operator/cmd/main.go | 3 - deploy/operator/go.mod | 1 - deploy/operator/go.sum | 2 - .../controller/dgd_lpx_envtest_test.go | 118 -- .../internal/controller/dgd_lpx_handoff.go | 11 +- .../dgd_lpx_handoff_envtest_test.go | 131 +- .../controller/dgd_lpx_handoff_test.go | 3 +- .../controller/lpx/attempt_deadline.go | 132 +- .../controller/lpx/attempt_deadline_test.go | 336 ++-- .../controller/lpx/attempt_retirement.go | 153 ++ .../controller/lpx/attempt_retirement_test.go | 159 ++ .../controller/lpx/component_roles_test.go | 19 - .../internal/controller/lpx/controller.go | 152 +- .../controller/lpx/controller_test.go | 293 ++-- .../internal/controller/lpx/dependencies.go | 154 -- .../operator/internal/controller/lpx/grove.go | 63 - .../internal/controller/lpx/lifecycle.go | 1212 +++---------- .../internal/controller/lpx/lifecycle_test.go | 1498 ++++------------- .../internal/controller/lpx/modeldownload.go | 14 +- .../controller/lpx/modeldownload_test.go | 4 +- .../internal/controller/lpx/preflight.go | 51 +- .../internal/controller/lpx/render.go | 12 +- .../internal/controller/lpx/render_test.go | 30 +- .../operator/internal/controller/lpx/watch.go | 127 +- .../internal/controller/lpx/watch_test.go | 247 --- deploy/operator/internal/dynamo/graph_lpx.go | 8 + .../internal/dynamo/graph_lpx_test.go | 4 +- .../internal/dynamo/lpx/build_snapshot.go | 2 +- .../internal/dynamo/lpx/cyborg_render.go | 2 - .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 6 +- .../internal/dynamo/lpx/model_fixture_test.go | 2 +- .../internal/dynamo/lpx/model_projection.go | 45 +- .../internal/dynamo/lpx/model_registry.go | 22 +- .../dynamo/lpx/model_registry_test.go | 12 +- deploy/operator/internal/dynamo/lpx/render.go | 7 + .../internal/dynamo/lpx/render_test.go | 15 +- .../lpx/resolve_selected_workload_test.go | 6 +- .../dynamo/lpx/scheduler/v1alpha1/types.go | 324 ++-- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 0 .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 8 +- .../node-local-v2-hybrid.input.yaml | 0 .../from_dgd_yaml/node-local-v2-hybrid.yaml | 8 +- .../node-local-v2-lpu-only.input.yaml | 0 .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 6 +- .../node-local-v2-specdecode.input.yaml | 0 .../node-local-v2-specdecode.yaml | 8 +- .../node-local-v3-hx-hybrid.input.yaml | 0 .../node-local-v3-hx-hybrid.yaml | 6 +- .../node-local-v3-hx-lpu-only.input.yaml | 0 .../node-local-v3-hx-lpu-only.yaml | 6 +- .../node-local-v3-hx-specdecode.input.yaml | 0 .../node-local-v3-hx-specdecode.yaml | 8 +- .../from_dgd_yaml/single_v2.input.yaml | 0 .../testdata/from_dgd_yaml/single_v2.yaml | 6 +- .../dynamo/lpx/workload_projection.go | 36 - .../dynamo/lpx/workload_projection_test.go | 79 +- .../dynamo/lpx/workload_projection_v2.go | 19 +- .../dynamo/lpx/workload_projection_v2_test.go | 22 +- .../dynamo/lpx/workload_projection_v3.go | 19 +- .../dynamo/lpx/workload_projection_v3_test.go | 23 +- deploy/operator/internal/features/lpx.go | 17 +- deploy/operator/internal/features/lpx_test.go | 31 +- ...hdeployment_lpx_validation_envtest_test.go | 2 - 67 files changed, 1943 insertions(+), 3757 deletions(-) delete mode 100644 deploy/operator/internal/controller/dgd_lpx_envtest_test.go create mode 100644 deploy/operator/internal/controller/lpx/attempt_retirement.go create mode 100644 deploy/operator/internal/controller/lpx/attempt_retirement_test.go delete mode 100644 deploy/operator/internal/controller/lpx/dependencies.go delete mode 100644 deploy/operator/internal/controller/lpx/grove.go rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/lpx_v2_vllm.yaml (98%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml (97%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml (97%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml (97%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml (97%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml (98%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml (98%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/single_v2.input.yaml (100%) rename deploy/operator/internal/dynamo/{ => lpx}/testdata/from_dgd_yaml/single_v2.yaml (98%) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 9c2eab7c0113..857338198f89 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -357,14 +357,6 @@ rules: - patch - update - watch -- apiGroups: - - scheduler.grove.io - resources: - - podgangs - verbs: - - get - - list - - watch - apiGroups: - scheduling.lpu.nvidia.com resources: @@ -374,7 +366,6 @@ rules: - delete - get - list - - update - watch - apiGroups: - scheduling.run.ai diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index 9018857a2db8..87f2f502f3d6 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -3,7 +3,6 @@ suite: LPX operator configuration and RBAC templates: - - templates/validate-values.yaml - charts/dynamo-operator/templates/operator-config.yaml - charts/dynamo-operator/templates/manager-rbac.yaml - charts/dynamo-operator/templates/validate-values.yaml @@ -65,6 +64,5 @@ tests: - delete - get - list - - update - watch documentIndex: 0 diff --git a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml index 51662fcc3895..ac53bf99917d 100644 --- a/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml +++ b/deploy/helm/charts/platform/tests/validate_removed_values_test.yaml @@ -96,7 +96,6 @@ tests: global.kai-scheduler.install: true asserts: - notFailedTemplate: {} - # --- Default values should render cleanly --- - it: should render successfully with default values diff --git a/deploy/operator/api/config/v1alpha1/lpx.go b/deploy/operator/api/config/v1alpha1/lpx.go index a18920528add..de5040b03724 100644 --- a/deploy/operator/api/config/v1alpha1/lpx.go +++ b/deploy/operator/api/config/v1alpha1/lpx.go @@ -6,8 +6,8 @@ package v1alpha1 // LPXConfiguration holds LPX scheduler integration and model-registry settings. type LPXConfiguration struct { // Enabled opts the operator into LPX request production. When true, startup - // requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and - // scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. + // requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API to + // be installed separately. // +kubebuilder:default=false Enabled bool `json:"enabled"` diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index c54bf58ed9f2..d05658a4185e 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -76,7 +76,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" istioclientsetscheme "istio.io/client-go/pkg/clientset/versioned/scheme" gaiev1 "sigs.k8s.io/gateway-api-inference-extension/api/v1" @@ -128,8 +127,6 @@ func initCRDSchemes() { // operator only consumes them (creates/reads), it does not reconcile them. utilruntime.Must(snapshotv1alpha1.AddToScheme(crdScheme)) - utilruntime.Must(groveschedulerv1alpha1.AddToScheme(crdScheme)) - utilruntime.Must(lpxv1alpha1.AddToScheme(crdScheme)) utilruntime.Must(apiextensionsv1.AddToScheme(crdScheme)) diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 4a0db3fa943b..05be1fb384b6 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -7,7 +7,6 @@ require ( emperror.dev/errors v0.8.1 github.com/Masterminds/semver/v3 v3.4.0 github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 - github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1 github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f github.com/ai-dynamo/snapshot/api v0.1.0 github.com/bsm/gomega v1.27.10 diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 83d26aae74f7..2baeb82cfa45 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -20,8 +20,6 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13 h1:eVDVuyh5vDgWG6I2Mffs68g+25qW94VAG3EfDTxbhc0= github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.13/go.mod h1:csL4kgDc1MoD7Erjyu8QyVrbL/dv8fIZ3sYkJcc2nrQ= -github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1 h1:9WJ/OFtVNQMozYVUl6OfXDz/jF4hFvdj7hL7SHauIOU= -github.com/ai-dynamo/grove/scheduler/api v0.1.0-alpha.12-rc1/go.mod h1:72lQ5kc40jXxyOrfuAICc8qxV0/Mwy8k8woM8LUoo9E= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f h1:PBmn3itAlMFHc0HgDOcr0IwV0uiUDQehcg7/aumIobU= github.com/ai-dynamo/modelexpress/modelexpress_client/go v0.0.0-20260428070018-96d409e6f37f/go.mod h1:i7tIkusSEcMQqN4IDd3cXZ6EL0Kfke6F00Nn5WbuH3Y= github.com/ai-dynamo/snapshot/api v0.1.0 h1:S8HG6K+YhloUc/usu03wiJ8fTo3Mhju/oWQVFDYVw/o= diff --git a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_envtest_test.go deleted file mode 100644 index 967118cb0d57..000000000000 --- a/deploy/operator/internal/controller/dgd_lpx_envtest_test.go +++ /dev/null @@ -1,118 +0,0 @@ -//go:build !clustertest - -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "testing" - - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" - "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { - t.Log("Install the real CRDs in an isolated API server; no runtime controllers or cluster workloads") - env := operatorenv.New(operatorenv.Options{SetupWebhooks: setupProductionWebhooks}).RunT(t) - require.True(t, env.Client().Scheme().Recognizes(v1alpha1.LPXGraphDeploymentGVK)) - require.False(t, env.Client().Scheme().Recognizes(v1beta1.GroupVersion.WithKind("LPXGraphDeployment"))) - source := newLPXHandoffSource(t, "node-local-v2-lpu-only") - source.Namespace, source.UID, source.Generation = env.Namespace(), "", 0 - require.NoError(t, env.Client().Create(t.Context(), source)) - - t.Log("Persist the pending component projection using the real DGD status schema") - result := &ReconcileResult{} - projectLPXChildStatus(source, nil, result, &source.Status) - source.Status.Components = result.ComponentStatus - source.Status.State = result.State - require.NoError(t, env.Client().Status().Update(t.Context(), source)) - require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) - require.False(t, source.Status.Components["lpx"].Ready) - - t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") - handoff := &dgdLPXHandoff{client: env.Client()} - child, err := handoff.Reconcile(t.Context(), source) - require.NoError(t, err) - require.Equal(t, int64(1), child.Generation) - require.NotEmpty(t, child.UID) - require.NotEqual(t, source.UID, child.UID) - require.Equal(t, metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK), metav1.GetControllerOf(child)) - require.NoError(t, dynamo.ValidateLPXSource(child, source)) - - t.Log("Project the observed download payload and its actionable failure") - child.Status.ObservedGeneration = child.Generation - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} - child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, - LastTransitionTime: metav1.Now(), Reason: "PublicationDenied", Message: "Check the namespace quota"}} - require.NoError(t, env.Client().Status().Update(t.Context(), child)) - projected := v1beta1.DynamoGraphDeploymentStatus{} - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, child, result, &projected) - require.Equal(t, v1beta1.DGDStateFailed, result.State) - require.Equal(t, Reason(child.Status.Conditions[0].Reason), result.Reason) - require.Equal(t, Message(child.Status.Conditions[0].Message), result.Message) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) - - t.Log("Ordinary source metadata does not advance the child generation or frozen source identity") - source.Labels = map[string]string{"unrelated": "metadata"} - require.NoError(t, env.Client().Update(t.Context(), source)) - unchanged, err := handoff.Reconcile(t.Context(), source) - require.NoError(t, err) - require.Equal(t, child.ResourceVersion, unchanged.ResourceVersion) - require.Equal(t, child.Annotations, unchanged.Annotations) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, unchanged, result, &projected) - require.Equal(t, v1beta1.DGDStateFailed, result.State) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) - - t.Log("An LPX template edit advances the real child generation and invalidates the old observation") - lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" - require.NoError(t, env.Client().Update(t.Context(), source)) - updated, err := handoff.Reconcile(t.Context(), source) - require.NoError(t, err) - require.Equal(t, child.Generation+1, updated.Generation) - require.NotEqual(t, child.Spec.InputRevision, updated.Spec.InputRevision) - require.NoError(t, dynamo.ValidateLPXSource(updated, source)) - require.Equal(t, child.Status.ObservedGeneration, updated.Status.ObservedGeneration) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, updated, result, &projected) - require.Equal(t, v1beta1.DGDStatePending, result.State) - require.Equal(t, Reason("LPXChildPending"), result.Reason) - require.Nil(t, projected.LPX) - - t.Log("Reject malformed revision hashes in the API server") - invalid := updated.DeepCopy() - invalid.Spec.InputRevision = "unversioned" - require.True(t, apierrors.IsInvalid(env.Client().Update(t.Context(), invalid))) - - t.Log("The status subresource retains child observations without rewriting source DGD status") - updated.Status.ObservedGeneration = updated.Generation - updated.Status.Conditions[0].ObservedGeneration = updated.Generation - require.NoError(t, env.Client().Status().Update(t.Context(), updated)) - stored := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(updated), stored)) - require.Equal(t, updated.Status, stored.Status) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, stored, result, &projected) - require.Equal(t, v1beta1.DGDStateFailed, result.State) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) - storedSource := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) - require.Equal(t, source.Status, storedSource.Status) - - t.Log("An empty scheduling batch must persist as an array, not null") - stored.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: stored.Generation}} - require.True(t, apierrors.IsInvalid(env.Client().Status().Update(t.Context(), stored))) - stored.Status.Placement.LPXAttempt.Requests = []v1beta1.LPXAttemptRequestStatus{} - require.NoError(t, env.Client().Status().Update(t.Context(), stored)) - require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(stored), stored)) - require.Equal(t, []v1beta1.LPXAttemptRequestStatus{}, stored.Status.Placement.LPXAttempt.Requests) -} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index 767cabd8b1ad..f803e9f65c9f 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -31,7 +31,7 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra return nil, err } exists := err == nil - if exists && !exactLPXSourceOwner(child, source) { + if exists && !metav1.IsControlledBy(child, source) { return nil, fmt.Errorf("LPXGraphDeployment %q belongs to a different source; adoption is not supported", child.Name) } if !source.HasLPXComponent() { @@ -72,13 +72,6 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra return child, err } -func exactLPXSourceOwner(child *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) bool { - owner := metav1.GetControllerOf(child) - return child.Name == source.Name && child.Namespace == source.Namespace && - owner != nil && owner.Kind == "DynamoGraphDeployment" && owner.APIVersion == v1beta1.GroupVersion.String() && - owner.Name == source.Name && owner.UID == source.UID -} - func (r *dgdLPXHandoff) deleteChild(ctx context.Context, child *v1alpha1.LPXGraphDeployment) error { if !child.DeletionTimestamp.IsZero() { return nil @@ -92,7 +85,7 @@ func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGrap if err := r.client.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil { return client.IgnoreNotFound(err) } - if !exactLPXSourceOwner(child, source) { + if !metav1.IsControlledBy(child, source) { // It is not our dependent; do not delete it and do not wait for it. return nil } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 4b5474d41eee..423ccd67e30f 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -26,11 +26,11 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -116,30 +116,21 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { if err := lpxv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { return err } - if err := groveschedulerv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { - return err - } return setupProductionWebhooks(mgr, opts) }, }).RunT(t) - dependencies := make([]*apiextensionsv1.CustomResourceDefinition, 0, 2) - for _, dependency := range []struct{ group, kind, plural string }{ - {lpxv1alpha1.APIGroup, "LpuPipelineRequest", "lpupipelinerequests"}, - {groveschedulerv1alpha1.SchemeGroupVersion.Group, "PodGang", "podgangs"}, - } { - dependencies = append(dependencies, &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{Name: dependency.plural + "." + dependency.group}, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: dependency.group, Scope: apiextensionsv1.NamespaceScoped, - Names: apiextensionsv1.CustomResourceDefinitionNames{Kind: dependency.kind, ListKind: dependency.kind + "List", Plural: dependency.plural}, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ - Name: "v1alpha1", Served: true, Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{Type: "object", XPreserveUnknownFields: ptr.To(true)}}, - }}, - }, - }) + lprCRD := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "lpupipelinerequests." + lpxv1alpha1.APIGroup}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: lpxv1alpha1.APIGroup, Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{Kind: "LpuPipelineRequest", ListKind: "LpuPipelineRequestList", Plural: "lpupipelinerequests"}, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1alpha1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{Type: "object", XPreserveUnknownFields: ptr.To(true)}}, + }}, + }, } - _, err = envtest.InstallCRDs(env.RESTConfig(), envtest.CRDInstallOptions{CRDs: dependencies}) + _, err = envtest.InstallCRDs(env.RESTConfig(), envtest.CRDInstallOptions{CRDs: []*apiextensionsv1.CustomResourceDefinition{lprCRD}}) require.NoError(t, err) t.Log("Deny Grove publication with real namespace quota admission, not a mocked client") @@ -308,3 +299,101 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { assert.Equal(c, ptr.To(true), configMap.Immutable) }, 20*time.Second, 50*time.Millisecond) } + +func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { + t.Log("Install the real CRDs in an isolated API server; no runtime controllers or cluster workloads") + env := operatorenv.New(operatorenv.Options{SetupWebhooks: setupProductionWebhooks}).RunT(t) + require.True(t, env.Client().Scheme().Recognizes(v1alpha1.LPXGraphDeploymentGVK)) + require.False(t, env.Client().Scheme().Recognizes(v1beta1.GroupVersion.WithKind("LPXGraphDeployment"))) + source := newLPXHandoffSource(t, "node-local-v2-lpu-only") + source.Namespace, source.UID, source.Generation = env.Namespace(), "", 0 + require.NoError(t, env.Client().Create(t.Context(), source)) + + t.Log("Persist the pending component projection using the real DGD status schema") + result := &ReconcileResult{} + projectLPXChildStatus(source, nil, result, &source.Status) + source.Status.Components = result.ComponentStatus + source.Status.State = result.State + require.NoError(t, env.Client().Status().Update(t.Context(), source)) + require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) + require.False(t, source.Status.Components["lpx"].Ready) + + t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") + handoff := &dgdLPXHandoff{client: env.Client()} + child, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, int64(1), child.Generation) + require.NotEmpty(t, child.UID) + require.NotEqual(t, source.UID, child.UID) + require.Equal(t, metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK), metav1.GetControllerOf(child)) + require.NoError(t, dynamo.ValidateLPXSource(child, source)) + + t.Log("Project the observed download payload and its actionable failure") + child.Status.ObservedGeneration = child.Generation + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} + child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, + LastTransitionTime: metav1.Now(), Reason: "PublicationDenied", Message: "Check the namespace quota"}} + require.NoError(t, env.Client().Status().Update(t.Context(), child)) + projected := v1beta1.DynamoGraphDeploymentStatus{} + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, child, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, Reason(child.Status.Conditions[0].Reason), result.Reason) + require.Equal(t, Message(child.Status.Conditions[0].Message), result.Message) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + + t.Log("Ordinary source metadata does not advance the child generation or frozen source identity") + source.Labels = map[string]string{"unrelated": "metadata"} + require.NoError(t, env.Client().Update(t.Context(), source)) + unchanged, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, child.ResourceVersion, unchanged.ResourceVersion) + require.Equal(t, child.Annotations, unchanged.Annotations) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, unchanged, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + + t.Log("An LPX template edit advances the real child generation and invalidates the old observation") + dynamolpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" + require.NoError(t, env.Client().Update(t.Context(), source)) + updated, err := handoff.Reconcile(t.Context(), source) + require.NoError(t, err) + require.Equal(t, child.Generation+1, updated.Generation) + require.NotEqual(t, child.Spec.InputRevision, updated.Spec.InputRevision) + require.NoError(t, dynamo.ValidateLPXSource(updated, source)) + require.Equal(t, child.Status.ObservedGeneration, updated.Status.ObservedGeneration) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, updated, result, &projected) + require.Equal(t, v1beta1.DGDStatePending, result.State) + require.Equal(t, Reason("LPXChildPending"), result.Reason) + require.Nil(t, projected.LPX) + + t.Log("Reject malformed revision hashes in the API server") + invalid := updated.DeepCopy() + invalid.Spec.InputRevision = "unversioned" + require.True(t, apierrors.IsInvalid(env.Client().Update(t.Context(), invalid))) + + t.Log("The status subresource retains child observations without rewriting source DGD status") + updated.Status.ObservedGeneration = updated.Generation + updated.Status.Conditions[0].ObservedGeneration = updated.Generation + require.NoError(t, env.Client().Status().Update(t.Context(), updated)) + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(updated), stored)) + require.Equal(t, updated.Status, stored.Status) + result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} + projectLPXChildStatus(source, stored, result, &projected) + require.Equal(t, v1beta1.DGDStateFailed, result.State) + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) + storedSource := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) + require.Equal(t, source.Status, storedSource.Status) + + t.Log("An empty scheduling batch must persist as an array, not null") + stored.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: stored.Generation}} + require.True(t, apierrors.IsInvalid(env.Client().Status().Update(t.Context(), stored))) + stored.Status.Placement.LPXAttempt.Requests = []v1beta1.LPXAttemptRequestStatus{} + require.NoError(t, env.Client().Status().Update(t.Context(), stored)) + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(stored), stored)) + require.Equal(t, []v1beta1.LPXAttemptRequestStatus{}, stored.Status.Placement.LPXAttempt.Requests) +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 2fee66eaa25f..07d125b6c4ea 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -42,7 +42,7 @@ import ( // newLPXHandoffSource loads a fresh authored workload without compiler or child-controller fixtures. func newLPXHandoffSource(t *testing.T, fixture string) *v1beta1.DynamoGraphDeployment { t.Helper() - data, err := os.ReadFile(filepath.Join("../dynamo/testdata/from_dgd_yaml", fixture+".input.yaml")) + data, err := os.ReadFile(filepath.Join("../dynamo/lpx/testdata/from_dgd_yaml", fixture+".input.yaml")) require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(data, source)) @@ -179,7 +179,6 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { handoff := &dgdLPXHandoff{client: kube} child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) - require.True(t, exactLPXSourceOwner(child, source)) require.NotEmpty(t, child.Spec.InputRevision) require.NoError(t, dynamo.ValidateLPXSource(child, source)) require.NotContains(t, child.Annotations, "lpx.nvidia.com/podcliqueset-name") diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index 44793bacf084..751f9018c983 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -13,14 +13,10 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -33,7 +29,6 @@ const ( lpxSchedulingDispositionObservedReason string = "LPXSchedulingDispositionObserved" lpxSchedulingDeadlineExceededReason string = "LPXSchedulingDeadlineExceeded" lpxAttemptAuthorityLostReason string = "LPXAttemptAuthorityLost" - lpxAttemptRecordingFinalizer = "scheduling.lpu.nvidia.com/dynamo-attempt-recording" ) type lpxDeadlineTransition struct { @@ -115,7 +110,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( } if attempt.ExceededAt != nil { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, observed, lpxSchedulingDeadlineExceededReason, + ctx, deployment, attempt, requests, lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; no retry is authorized", ) return state, time.Time{}, nil, err @@ -144,7 +139,7 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( } if missing { state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, observed, lpxAttemptAuthorityLostReason, + ctx, deployment, attempt, requests, lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring its scheduling batch without replacement", ) return state, time.Time{}, nil, err @@ -162,71 +157,34 @@ func (r *graphReconciler) reconcileLPXAttemptDeadline( return nil, deadlineAt, requests, nil } -// retireLPXEngineRequest records intentional removal before deleting the exact request. +// retireLPXEngineRequest records intentional removal before deleting the +// request's owning PCS. func (r *graphReconciler) retireLPXEngineRequest(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest, reason string, intentional bool) (lpxClassification, error) { // Persist batch membership before deletion can be mistaken for lost authority. if current := currentLPXAttemptStatus(deployment); current != nil && intentional && current.ExceededAt == nil { for i, row := range current.Requests { if row.Name == request.Name && row.AttemptDigest != "" && (row.UID == request.UID || row.UID == "" && row.AttemptDigest == request.Annotations[lpxAttemptDigestAnnotation]) { next := current.DeepCopy() - // Retain exact cleanup authority until a replacement digest can be prepared. + // Deleting the shared PCS replaces every request it owns. next.Requests[i].UID = request.UID - next.Requests[i].AttemptDigest = "" + for j := range next.Requests { + next.Requests[j].AttemptDigest = "" + } return newLPXDeadlineTransition(next, nvidiacomv1beta1.DGDStatePending, lpxRetiringReason, reason, time.Nanosecond), nil } } } - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return nil, err - } - if request.DeletionTimestamp.IsZero() { - uid, rv := request.UID, request.ResourceVersion - if err := r.Delete(ctx, request, client.Preconditions{UID: &uid, ResourceVersion: &rv}); err != nil && !apierrors.IsNotFound(err) { - return nil, err - } - } - // Invalid live intent needs exact Pod cleanup; native replacement already owns its Pods. - if !intentional { - if err := r.retireLPXAttemptPods(ctx, request); err != nil { - return nil, err - } + owner := metav1.GetControllerOf(request) + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != "PodCliqueSet" || owner.UID == "" { + return nil, fmt.Errorf("LPX request %q does not have an exact PodCliqueSet controller owner", request.Name) } - if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { + if _, err := r.deleteLPXPodCliqueSet(ctx, deployment, owner.Name, owner.UID); err != nil { return nil, err } return &lpxRetiring{retirementReason: reason}, nil } -// retireLPXAttemptPods releases only Pods committed to this exact scheduling request. -func (r *graphReconciler) retireLPXAttemptPods(ctx context.Context, request *lpxv1alpha1.LPUPipelineRequest) error { - if request.Status == nil || request.Status.Committed == nil || request.Status.Committed.Execution.NodeLocal == nil { - return nil - } - for _, partition := range request.Status.Committed.Execution.NodeLocal.PartitionSelections { - for _, row := range partition.SelectedRows { - refs := make([]lpxv1alpha1.ObjectReference, 0, 2) - if row.Current != nil { - refs = append(refs, row.Current.PodRef) - } - if row.Target != nil { - refs = append(refs, row.Target.PodRef) - } - for _, ref := range refs { - if ref.Namespace != request.Namespace || ref.Name == "" || ref.UID == "" { - return fmt.Errorf("LPX request %q contains an invalid committed Pod identity", request.Name) - } - uid := types.UID(ref.UID) - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, Name: ref.Name}} - if err := r.Delete(ctx, pod, client.Preconditions{UID: &uid}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { - return err - } - } - } - } - return nil -} - func (r *graphReconciler) reconcileLPXAttemptPreparation( deployment *nvidiacomv1alpha1.LPXGraphDeployment, deadlineSeconds *int64, @@ -252,11 +210,17 @@ func (r *graphReconciler) reconcileLPXAttemptPreparation( if active { next = current.DeepCopy() } + for _, row := range current.Requests { if active || row.UID != "" && row.AttemptDigest == "" { previous[row.Name] = row } } + // Move the preserved attempt clock to its replacement PCS once every old request is gone. + if active && lpxAttemptRequestsRetired(current.Requests) { + next.ObservedGeneration = want.ObservedGeneration + next.PodCliqueSetUID = want.PodCliqueSetUID + } } next.Requests = make([]nvidiacomv1beta1.LPXAttemptRequestStatus, 0, len(want.Requests)) // Desired rows are already sorted; only newly joining engines need disposition checks. @@ -302,6 +266,18 @@ func (r *graphReconciler) reconcileLPXAttemptPreparation( ) } +func lpxAttemptRequestsRetired(requests []nvidiacomv1beta1.LPXAttemptRequestStatus) bool { + if len(requests) == 0 { + return false + } + for _, request := range requests { + if request.UID != "" || request.AttemptDigest != "" { + return false + } + } + return true +} + // revalidateLPXAttemptPublication fences live authority after preparation has // established the current nonnil attempt and an unpublished desired request. func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, pcsName string) error { @@ -411,21 +387,6 @@ func lpxRequestsByName(requests []lpxv1alpha1.LPUPipelineRequest) map[string]*lp return byName } -func (r *graphReconciler) releaseLPXAttemptRecordingFinalizer(ctx context.Context, request *lpxv1alpha1.LPUPipelineRequest) error { - // The scheduler finishes cleanup and removes scheduling.lpu.nvidia.com/plan-protection - // independently, never waiting for lpxAttemptRecordingFinalizer. Dynamo releases - // its finalizer last to retain the request identity and terminal status during cleanup. - // Unknown finalizers block release; their owners must resolve them, not Dynamo. - if request.DeletionTimestamp.IsZero() || len(request.Finalizers) != 1 || request.Finalizers[0] != lpxAttemptRecordingFinalizer { - return nil - } - controllerutil.RemoveFinalizer(request, lpxAttemptRecordingFinalizer) - if err := r.Update(ctx, request); err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("release LPX attempt recording finalizer on %q: %w", request.Name, err) - } - return nil -} - // exactLPXAttemptState reports aggregate request state; attempt must be nonnil. func exactLPXAttemptState( attempt *nvidiacomv1beta1.LPXAttemptStatus, @@ -468,41 +429,14 @@ func (r *graphReconciler) continueTerminalLPXAttempt( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, attempt *nvidiacomv1beta1.LPXAttemptStatus, - observed map[string]*lpxv1alpha1.LPUPipelineRequest, + requests []lpxv1alpha1.LPUPipelineRequest, reason string, message string, ) (lpxClassification, error) { - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + pending, err := r.retireLPXAttempt(ctx, deployment, attempt, requests) + if err != nil { return nil, err } - // Retire recorded identities in their canonical request order. - pending := false - for _, row := range attempt.Requests { - request := observed[row.Name] - if row.UID == "" || request == nil || request.UID != row.UID { - continue - } - pending = true - if !request.DeletionTimestamp.IsZero() { - if err := r.retireLPXAttemptPods(ctx, request); err != nil { - return nil, err - } - if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { - return nil, err - } - continue - } - uid, resourceVersion := request.UID, request.ResourceVersion - if err := r.Delete(ctx, request, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ - UID: &uid, ResourceVersion: &resourceVersion, - }}); err != nil && !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("delete terminal LPX request %q: %w", request.Name, err) - } - if err := r.retireLPXAttemptPods(ctx, request); err != nil { - return nil, err - } - } - // A failed scheduling batch never scales down or deletes the shared PCS. if !pending && attempt.ExceededAt != nil && deployment.Generation > attempt.ObservedGeneration { return newLPXDeadlineTransition(nil, nvidiacomv1beta1.DGDStatePending, lpxAttemptPreparedReason, "The failed batch is gone; a later edit permits scheduling", time.Nanosecond), nil diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 1a4bb03efab5..1c97e05ef19d 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -8,6 +8,7 @@ package lpx import ( "context" "errors" + "fmt" "testing" "time" @@ -15,7 +16,6 @@ import ( grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -38,33 +38,28 @@ func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) require.Nil(t, rejected) createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) require.Equal(t, lpxAttemptPreparedReason, lpxResult(classification).Reason) prepared := applyLPXAttemptTransition(t, dgd, classification) require.NotEmpty(t, prepared.PodCliqueSetUID) require.Empty(t, ownedLPXRequests(t, ctx, reconciler, dgd)) - _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + _, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) requests := ownedLPXRequests(t, ctx, reconciler, dgd) require.Len(t, requests, 1) - require.Contains(t, requests[0].Finalizers, lpxAttemptRecordingFinalizer) + require.Empty(t, requests[0].Finalizers) dgd.Status.Placement = nil - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) recovered := applyLPXAttemptTransition(t, dgd, classification) require.Equal(t, requests[0].UID, recovered.Requests[0].UID, "publication reconciliation recovers the existing request") - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: prepared} + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: recovered} require.NoError(t, reconciler.Delete(ctx, &requests[0])) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) require.NoError(t, err) - armed := applyLPXAttemptTransition(t, dgd, classification) - require.Equal(t, requests[0].UID, armed.Requests[0].UID) - require.True(t, armed.DeadlineAt.Time.Equal(requests[0].CreationTimestamp.Add(30*time.Second))) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) - require.NoError(t, err) require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) for _, test := range []struct { @@ -88,7 +83,10 @@ func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) attempt.DisarmedAt = ptr.To(metav1.Now()) } objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) - objects = append(objects, deadlineTestRequest(deployment, "removed-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound)) + pcs := findLPXTestPodCliqueSet(t, objects) + removed := deadlineTestRequest(deployment, pcs, "removed-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) + objects = append(objects, removed) + attempt.Requests = append(attempt.Requests, deadlineTestAttempt(deployment, removed, time.Now()).Requests...) if test.terminating { request := requests[0].DeepCopy() request.DeletionTimestamp = ptr.To(metav1.Now()) @@ -96,31 +94,28 @@ func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) objects = append(objects, request) } r := newLPXTestReconciler(t, registry, deployment, source, objects...) - key := client.ObjectKeyFromObject(deployment) - - t.Log("Full reconciliation repeatedly reports terminal authority loss without republishing or deleting the PCS") - for range 3 { - result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + t.Log("Terminal authority loss deletes the PCS before its requests") + var transition *lpxDeadlineTransition + for range 2 { + classification, _, _, err := r.reconcileLPXAttemptDeadline(ctx, deployment, source) require.NoError(t, err) - require.NoError(t, r.Get(ctx, key, deployment)) - failed := meta.FindStatusCondition(deployment.Status.Conditions, "Failed") - require.NotNil(t, failed) - require.Equal(t, metav1.ConditionTrue, failed.Status) - require.Equal(t, lpxAttemptAuthorityLostReason, failed.Reason) - live := ownedLPXRequests(t, ctx, r, deployment) - if test.terminating { - require.Len(t, live, 1) - require.Equal(t, requests[0].UID, live[0].UID) - require.False(t, live[0].DeletionTimestamp.IsZero()) - } else { - require.Empty(t, live) - require.Zero(t, result.RequeueAfter) - } - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) - require.Equal(t, armed.PodCliqueSetUID, pcs.UID) - require.Equal(t, int32(1), pcs.Spec.Replicas) - require.True(t, pcs.DeletionTimestamp.IsZero()) + require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) + transition = classification.(*lpxDeadlineTransition) + deployment.Status.Placement.LPXAttempt = transition.attempt + } + pcs = &grovev1alpha1.PodCliqueSet{} + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs))) + removed = getLPXRequest(t, ctx, r.Client, deployment.Namespace, "removed-engine") + require.NoError(t, r.Delete(ctx, removed), "emulate garbage collection in the fake client") + live := ownedLPXRequests(t, ctx, r, deployment) + if test.terminating { + require.Len(t, live, 1) + require.Equal(t, requests[0].UID, live[0].UID) + require.False(t, live[0].DeletionTimestamp.IsZero()) + require.Equal(t, lpxRetirementRequeueAfter, transition.requeueAfter) + } else { + require.Empty(t, live) + require.Equal(t, lpxRetirementRequeueAfter, transition.requeueAfter) } }) } @@ -179,7 +174,7 @@ func TestLPXAttemptPartialPublicationLossFencesEverySibling(t *testing.T) { createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) for range 3 { - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) applyLPXAttemptTransition(t, dgd, classification) } @@ -200,7 +195,7 @@ func TestLPXAttemptPartialPublicationLossFencesEverySibling(t *testing.T) { classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) require.NoError(t, err) require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) - _, _ = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + _, _ = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.LessOrEqual(t, len(ownedLPXRequests(t, ctx, reconciler, dgd)), 1) } @@ -209,14 +204,24 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { source := newLPXTestSource(lpx.PipelineSingle, "build-v2") dgd := newLPXTestDeployment(t, source) source.Spec.Scheduling = deadlineTestScheduling() - request := deadlineTestRequest(dgd, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) - running := deadlineTestRequest(dgd, "older-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) - request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} pcs := deadlineTestPCS(dgd, "pcs-uid") + request := deadlineTestRequest(dgd, pcs, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) + running := deadlineTestRequest(dgd, pcs, "older-engine", time.Now().UTC().Truncate(time.Second).Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "engine-group", Namespace: dgd.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 2}, + } + request.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: 1} + running.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: 0} + request.Finalizers = []string{"scheduling.lpu.nvidia.com/lpx-cleanup"} attempt := deadlineTestAttempt(dgd, request, time.Now().Add(time.Minute)) attempt.DeadlineAt, attempt.PodCliqueSetUID = nil, "" dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - reconciler := newLPXTestReconciler(t, nil, dgd, source, request, running, pcs) + reconciler := newLPXTestReconciler(t, nil, dgd, source, request, running, pcs, group) + running = getLPXRequest(t, t.Context(), reconciler.Client, running.Namespace, running.Name) t.Log("Recover the native deadline and record the Bound scheduling disposition") classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) @@ -237,7 +242,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { t.Log("Resumed scheduling expires under the original elapsed deadline") request.Status.Phase = lpxv1alpha1.RequestPhasePending - reconciler = newLPXTestReconciler(t, nil, dgd, source, request, running, pcs) + reconciler = newLPXTestReconciler(t, nil, dgd, source, request, running, pcs, group) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) result := lpxResult(classification) @@ -250,29 +255,33 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { applyLPXAttemptTransition(t, dgd, classification) require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) - t.Log("Delete only the expired request, preserving the PCS and previously running engine") + t.Log("Scale down the failed engine before deleting its request, preserving the older engine") _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) - deleting := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name) - require.False(t, deleting.DeletionTimestamp.IsZero()) storedPCS := &grovev1alpha1.PodCliqueSet{} pcsKey := client.ObjectKeyFromObject(pcs) require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) - require.EqualValues(t, 1, storedPCS.Spec.Replicas) - require.Equal(t, pcs.UID, storedPCS.UID) - require.Equal(t, running.UID, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).UID) + require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) + require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).DeletionTimestamp.IsZero()) - t.Log("Release the recording finalizer only after scheduler cleanup finishes") - deleting.Finalizers = []string{lpxAttemptRecordingFinalizer} - require.NoError(t, reconciler.Update(t.Context(), deleting)) + t.Log("Retire only the failed request after Grove has accepted the lower scale") + _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + require.NoError(t, err) + deleting := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name) + require.False(t, deleting.DeletionTimestamp.IsZero()) + require.Equal(t, running, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name)) _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) + + t.Log("The request disappears when scheduler cleanup releases its own finalizer") + deleting.Finalizers = nil + require.NoError(t, reconciler.Update(t.Context(), deleting)) _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) require.NoError(t, err) - require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) - require.EqualValues(t, 1, storedPCS.Spec.Replicas) require.True(t, apierrors.IsNotFound(reconciler.Get(t.Context(), client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) require.Zero(t, classification.(*lpxDeadlineTransition).requeueAfter) @@ -291,7 +300,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(pcs), storedPCS)) require.Equal(t, int32(1), storedPCS.Spec.Replicas) - t.Log("A stale cached attempt cannot clean up siblings after a newer status records replacement") + t.Log("A stale cached attempt has no exact objects to clean up after replacement") key := client.ObjectKeyFromObject(dgd) require.NoError(t, reconciler.Get(t.Context(), key, dgd)) cached := dgd.DeepCopy() @@ -304,7 +313,7 @@ func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { cached.Status.Placement.LPXAttempt.Requests = append(cached.Status.Placement.LPXAttempt.Requests, deadlineTestAttempt(cached, running, time.Now()).Requests[0]) _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), cached, source) - require.ErrorContains(t, err, "authority changed") + require.NoError(t, err) require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).DeletionTimestamp.IsZero()) require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, replacement.Name).DeletionTimestamp.IsZero()) @@ -341,8 +350,9 @@ func TestLPXAttemptBatchEditsPreserveUnchangedEngines(t *testing.T) { t.Log("Keep the existing engine and its deadline across a new source generation") source := newLPXTestSource(lpx.PipelineSingle, "build-v2") deployment := newLPXTestDeployment(t, source) + pcs := deadlineTestPCS(deployment, "pcs-uid") now := time.Now().UTC().Truncate(time.Second) - old := deadlineTestRequest(deployment, "old", now.Add(-time.Hour), tc.phase) + old := deadlineTestRequest(deployment, pcs, "old", now.Add(-time.Hour), tc.phase) deadline := now.Add(time.Minute) if tc.phase != lpxv1alpha1.RequestPhasePending { deadline = now.Add(-time.Minute) @@ -360,7 +370,7 @@ func TestLPXAttemptBatchEditsPreserveUnchangedEngines(t *testing.T) { want.Requests[0].AttemptDigest = "" } if tc.add { - added := deadlineTestRequest(deployment, "new", now, lpxv1alpha1.RequestPhasePending) + added := deadlineTestRequest(deployment, pcs, "new", now, lpxv1alpha1.RequestPhasePending) if tc.deleting { added.DeletionTimestamp = ptr.To(metav1.Now()) } @@ -397,23 +407,63 @@ func TestLPXAttemptBatchEditsPreserveUnchangedEngines(t *testing.T) { } } -func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { +func TestLPXAttemptReplacementAdoptsNewPodCliqueSetIdentity(t *testing.T) { + for _, timed := range []bool{false, true} { + t.Run(fmt.Sprintf("timed=%t", timed), func(t *testing.T) { + t.Log("Record an active attempt after every old request has finished garbage collection") + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + deployment := newLPXTestDeployment(t, source) + deployment.Generation = 2 + current := &nvidiacomv1beta1.LPXAttemptStatus{ + ObservedGeneration: 1, + PodCliqueSetUID: "previous-pcs-uid", + Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ + Name: "engine", + }}, + } + var deadlineSeconds *int64 + if timed { + current.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Minute))) + deadlineSeconds = ptr.To[int64](30) + } + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: current} + replacement := deadlineTestPCS(deployment, "replacement-pcs-uid") + reconciler := newLPXTestReconciler(t, nil, deployment, source, replacement) + want := &nvidiacomv1beta1.LPXAttemptStatus{ + ObservedGeneration: deployment.Generation, + PodCliqueSetUID: replacement.UID, + Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ + Name: "engine", AttemptDigest: "sha256:replacement", + }}, + } + + t.Log("Move the preserved attempt to the replacement PCS before publishing its request") + transition := reconciler.reconcileLPXAttemptPreparation( + deployment, + deadlineSeconds, + want, + map[string]*lpxv1alpha1.LPUPipelineRequest{}, + ) + require.NotNil(t, transition) + next := applyLPXAttemptTransition(t, deployment, transition) + require.Equal(t, deployment.Generation, next.ObservedGeneration) + require.Equal(t, replacement.UID, next.PodCliqueSetUID) + require.Equal(t, current.DeadlineAt, next.DeadlineAt) + require.Equal(t, want.Requests, next.Requests) + require.NoError(t, reconciler.revalidateLPXAttemptPublication(t.Context(), deployment, replacement.Name)) + }) + } +} + +func TestLPXAttemptScaleInCancellationPrecedesDeadline(t *testing.T) { for _, tc := range []struct { - name string - phase lpxv1alpha1.RequestPhase - replacing, keepClique bool - settings bool + name string + phase lpxv1alpha1.RequestPhase }{ - {name: "scale-in", phase: lpxv1alpha1.RequestPhasePending}, - {name: "completed-scale-in", phase: lpxv1alpha1.RequestPhaseBound}, - {name: "native-replacement", phase: lpxv1alpha1.RequestPhasePending, replacing: true}, - {name: "native-child-still-deleting", phase: lpxv1alpha1.RequestPhasePending, replacing: true, keepClique: true}, - {name: "completed-native-replacement", phase: lpxv1alpha1.RequestPhaseBound, replacing: true}, - {name: "settings-edit", phase: lpxv1alpha1.RequestPhasePending, settings: true}, - {name: "completed-settings-edit", phase: lpxv1alpha1.RequestPhaseBound, settings: true}, + {name: "pending", phase: lpxv1alpha1.RequestPhasePending}, + {name: "completed", phase: lpxv1alpha1.RequestPhaseBound}, } { t.Run(tc.name, func(t *testing.T) { - replacing := tc.replacing || tc.settings t.Log("Publish two engines, with only the second engine recorded in the scheduling batch") ctx := t.Context() deployment, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -421,12 +471,11 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { r, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) objects := lpxMaterializedObjects(t, r, deployment, source, desired) createLPXTestObjects(t, ctx, r.Client, objects...) - _, err := r.reconcileSelectedLPX(ctx, deployment, desired) - require.NoError(t, err) + publishSelectedLPXForTest(t, ctx, r, deployment, desired) running := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[0].requestName) running.Status = newLPXNodeLocalBoundStatus(running.Generation, 1, "sha256:bound") pending := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[1].requestName) - pending.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduler.example/cleanup"} + pending.Finalizers = []string{"scheduler.example/cleanup"} pending.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(pending.Generation), } @@ -436,17 +485,7 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { deadline := time.Now().UTC().Truncate(time.Second).Add(-time.Minute) deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: deadlineTestAttempt(deployment, pending, deadline)} source.Spec.Scheduling = deadlineTestScheduling() - if tc.settings { - source.Spec.Components[0].LPX.Settings.Raw = []byte(`{"prop_sync":false}`) - deployment.Generation++ - running.Finalizers = []string{"scheduler.example/cleanup"} - } else if replacing { - clique := findLPXTestClique(t, objects, desired.plan.ForReplica(1).Agents[0].CliqueName) - clique.Finalizers = []string{"grove.example/cleanup"} - clique.DeletionTimestamp = ptr.To(metav1.Now()) - } else { - source.Spec.Components[0].Replicas = ptr.To[int32](1) - } + source.Spec.Components[0].Replicas = ptr.To[int32](1) objects = append(objects, running, pending) r = newLPXTestReconciler(t, registry, deployment, source, objects...) r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ @@ -456,95 +495,52 @@ func TestLPXAttemptEmptyBatchRetirementPrecedesDeadline(t *testing.T) { }) key := client.ObjectKeyFromObject(deployment) - t.Log("Persist cancellation, while pending native replacement retains its deadline") - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + t.Log("Persist cancellation before evaluating the expired deadline") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.NoError(t, err) require.NoError(t, r.Get(ctx, key, deployment)) attempt := currentLPXAttemptStatus(deployment) - require.Equal(t, replacing && tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) - if replacing { - require.Len(t, attempt.Requests, 1) - require.Equal(t, pending.UID, attempt.Requests[0].UID) - require.Empty(t, attempt.Requests[0].AttemptDigest) - if tc.phase == lpxv1alpha1.RequestPhaseBound { - require.NotNil(t, attempt.DisarmedAt) - } - } else { - require.Equal(t, []nvidiacomv1beta1.LPXAttemptRequestStatus{}, attempt.Requests) - require.NotNil(t, attempt.DisarmedAt) - } + require.Equal(t, []nvidiacomv1beta1.LPXAttemptRequestStatus{}, attempt.Requests) + require.Nil(t, attempt.ExceededAt) + require.NotNil(t, attempt.DisarmedAt) require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) - if tc.replacing && !tc.keepClique { - t.Log("Grove can finish deleting the old clique before scheduler retirement resumes") - clique := &grovev1alpha1.PodClique{} - cliqueKey := client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.ForReplica(1).Agents[0].CliqueName} - require.NoError(t, r.Get(ctx, cliqueKey, clique)) - clique.Finalizers = nil - require.NoError(t, r.Update(ctx, clique)) - } - if !replacing { - t.Log("A source read failure on the next reconcile cannot expire the canceled batch") - readError := errors.New("temporary source observation failure") - original := r.Client - r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, source := object.(*nvidiacomv1beta1.DynamoGraphDeployment); source { - return readError - } - return delegated.Get(ctx, key, object, opts...) - }, - }) - _, readErr := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - r.Client = original - require.NoError(t, r.Get(ctx, key, deployment)) - require.Nil(t, currentLPXAttemptStatus(deployment).ExceededAt) - require.ErrorIs(t, readErr, readError) - } - t.Log("Cancellation completes, while native replacement still expires under the original clock") - for range 3 { + t.Log("A source read failure on the next reconcile cannot expire the canceled batch") + readError := errors.New("temporary source observation failure") + original := r.Client + r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, source := object.(*nvidiacomv1beta1.DynamoGraphDeployment); source { + return readError + } + return delegated.Get(ctx, key, object, opts...) + }, + }) + _, readErr := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + r.Client = original + require.NoError(t, r.Get(ctx, key, deployment)) + require.Nil(t, currentLPXAttemptStatus(deployment).ExceededAt) + require.ErrorIs(t, readErr, readError) + + t.Log("Cancellation remains disarmed while only the removed request retires") + for range 4 { _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - if tc.keepClique && err != nil { - t.Log("The next observation retries an optimistic concurrency conflict:", err) - require.True(t, apierrors.IsConflict(err)) - } else { - require.NoError(t, err) - } + require.NoError(t, err) require.NoError(t, r.Get(ctx, key, deployment)) attempt = currentLPXAttemptStatus(deployment) - require.Equal(t, replacing && tc.phase == lpxv1alpha1.RequestPhasePending, attempt.ExceededAt != nil) - if tc.replacing && tc.phase == lpxv1alpha1.RequestPhaseBound { - require.Nil(t, attempt.DeadlineAt) - } else { - require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) - } + require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) } - if replacing { - if tc.replacing { - require.Nil(t, attempt.DisarmedAt) - } - retiring := getLPXRequest(t, ctx, r.Client, deployment.Namespace, pending.Name) - require.False(t, retiring.DeletionTimestamp.IsZero()) - retiring.Finalizers = []string{lpxAttemptRecordingFinalizer} - require.NoError(t, r.Update(ctx, retiring)) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pending), &lpxv1alpha1.LPUPipelineRequest{}))) - } else { - require.NotNil(t, attempt.DisarmedAt) - } - stored := getLPXRequest(t, ctx, r.Client, deployment.Namespace, running.Name) - require.Equal(t, running.UID, stored.UID) - require.Equal(t, tc.settings, !stored.DeletionTimestamp.IsZero()) + require.Nil(t, attempt.ExceededAt) + require.NotNil(t, attempt.DisarmedAt) pcs := &grovev1alpha1.PodCliqueSet{} require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) require.Equal(t, types.UID("pcs-uid"), pcs.UID) - require.Equal(t, int32(1), pcs.Spec.Replicas) - require.True(t, pcs.DeletionTimestamp.IsZero()) + require.Equal(t, running, getLPXRequest(t, ctx, r.Client, deployment.Namespace, running.Name)) + stored := getLPXRequest(t, ctx, r.Client, deployment.Namespace, pending.Name) + require.False(t, stored.DeletionTimestamp.IsZero()) }) } } - func TestLPXAttemptRetryRequiresLaterEditAndCompletedCleanup(t *testing.T) { for _, tc := range []struct { name string @@ -554,7 +550,8 @@ func TestLPXAttemptRetryRequiresLaterEditAndCompletedCleanup(t *testing.T) { t.Log("Keep the failed batch fenced until both a later edit and exact-request cleanup") source := newLPXTestSource(lpx.PipelineSingle, "build-v2") deployment := newLPXTestDeployment(t, source) - old := deadlineTestRequest(deployment, "old", time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending) + pcs := deadlineTestPCS(deployment, "pcs-uid") + old := deadlineTestRequest(deployment, pcs, "old", time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending) old.Finalizers = []string{"scheduler.example/cleanup"} attempt := deadlineTestAttempt(deployment, old, time.Now().Add(-time.Second)) attempt.ExceededAt = ptr.To(metav1.Now()) @@ -567,8 +564,11 @@ func TestLPXAttemptRetryRequiresLaterEditAndCompletedCleanup(t *testing.T) { objects = append(objects, old) } r := newLPXTestReconciler(t, nil, deployment, source, objects...) - currents := lpxRequestsByName(ownedLPXRequests(t, t.Context(), r, deployment)) - classification, err := r.continueTerminalLPXAttempt(t.Context(), deployment, attempt, currents, lpxSchedulingDeadlineExceededReason, "expired") + var requests []lpxv1alpha1.LPUPipelineRequest + if !tc.oldGone { + requests = []lpxv1alpha1.LPUPipelineRequest{*old} + } + classification, err := r.continueTerminalLPXAttempt(t.Context(), deployment, attempt, requests, lpxSchedulingDeadlineExceededReason, "expired") require.NoError(t, err) transition := classification.(*lpxDeadlineTransition) if tc.laterEdit && tc.oldGone { @@ -586,11 +586,12 @@ func TestLPXAttemptRecoversUIDWithoutTimeoutOrSource(t *testing.T) { t.Log("Recover an already-published UID even when its timeout option or source is unavailable", sourceAvailable) source := newLPXTestSource(lpx.PipelineSingle, "build-v2") deployment := newLPXTestDeployment(t, source) - request := deadlineTestRequest(deployment, "unrecorded", time.Now(), lpxv1alpha1.RequestPhasePending) + pcs := deadlineTestPCS(deployment, "pcs-uid") + request := deadlineTestRequest(deployment, pcs, "unrecorded", time.Now(), lpxv1alpha1.RequestPhasePending) attempt := deadlineTestAttempt(deployment, request, time.Now()) attempt.DeadlineAt, attempt.Requests[0].UID = nil, "" deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - r := newLPXTestReconciler(t, nil, deployment, source, request) + r := newLPXTestReconciler(t, nil, deployment, source, request, pcs) if !sourceAvailable { source = nil } @@ -622,17 +623,18 @@ func deadlineTestAttempt(dgd *nvidiacomv1alpha1.LPXGraphDeployment, request *lpx } } -func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { +func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { generation := int64(1) request := &lpxv1alpha1.LPUPipelineRequest{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: dgd.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", Generation: generation, CreationTimestamp: metav1.NewTime(created), + Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, Annotations: map[string]string{ lpxAttemptDigestAnnotation: "sha256:attempt", lpxDeploymentUIDAnnotation: string(dgd.UID), - lpxPCSUIDAnnotation: "pcs-uid", + lpxPCSUIDAnnotation: string(pcs.UID), }, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, Status: &lpxv1alpha1.LPUPipelineRequestStatus{Phase: phase, ObservedGeneration: &generation}, diff --git a/deploy/operator/internal/controller/lpx/attempt_retirement.go b/deploy/operator/internal/controller/lpx/attempt_retirement.go new file mode 100644 index 000000000000..6c2e4e21ccaf --- /dev/null +++ b/deploy/operator/internal/controller/lpx/attempt_retirement.go @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "fmt" + "math" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// retireLPXAttempt retires only the recorded batch, preserving other engines in +// its shared PCS. deployment and attempt must be nonnil; requests are owned by deployment. +func (r *graphReconciler) retireLPXAttempt( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + attempt *nvidiacomv1beta1.LPXAttemptStatus, + requests []lpxv1alpha1.LPUPipelineRequest, +) (bool, error) { + // Partition the shared workload using exact request identities, including replacements. + identities := make(map[string]types.UID, len(attempt.Requests)) + for _, row := range attempt.Requests { + identities[row.Name] = row.UID + } + pcsName := dynamo.PCSNameForLPX(deployment) + var batch, retained []*lpxv1alpha1.LPUPipelineRequest + for index := range requests { + request := &requests[index] + owner := metav1.GetControllerOf(request) + if owner == nil || owner.UID != attempt.PodCliqueSetUID { + continue + } + pcsName = owner.Name + if uid := identities[request.Name]; uid != "" && uid == request.UID { + batch = append(batch, request) + } else { + retained = append(retained, request) + } + } + + // Preserve the shared PCS when a later scheduling batch fails during scale-out. + if len(retained) > 0 { + return r.retireLPXScaleOut(ctx, deployment, pcsName, attempt.PodCliqueSetUID, retained, batch) + } + pending, err := r.deleteLPXPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID) + return pending || len(batch) > 0, err +} + +// retireLPXScaleOut removes trailing engines through Grove before deleting their +// requests, whose scheduler finalizers may wait for pods to disappear. deployment +// must be nonnil and retained must contain at least one request outside the batch. +func (r *graphReconciler) retireLPXScaleOut( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcsName string, + pcsUID types.UID, + retained, batch []*lpxv1alpha1.LPUPipelineRequest, +) (bool, error) { + // Derive the surviving prefix from immutable request targets, independent of source edits. + groupName, replicas, err := lpxRetainedEngineScale(retained, batch) + if err != nil { + return false, err + } + + // An authoritative owner read fences a replaced PCS before changing its group's scale. + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if pcs.UID != pcsUID { + return true, nil + } + if !metav1.IsControlledBy(pcs, deployment) { + return false, fmt.Errorf("refusing to scale PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + } + if !pcs.DeletionTimestamp.IsZero() { + return true, nil + } + + // Wait for Grove to create the group, then lower its scale without ever growing it. + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: groupName}, group); err != nil { + return true, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(group, pcs) { + return false, fmt.Errorf("LPX scaling group lacks the current PCS owner") + } + if !group.DeletionTimestamp.IsZero() { + return true, nil + } + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return false, err + } + if group.Spec.Replicas > replicas { + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, + } + if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { + return false, err + } + return true, nil + } + + // Keep request cleanup fenced by exact UIDs until the scheduler releases each finalizer. + if err := r.deleteStaleLPXRequests(ctx, deployment, batch); err != nil { + return false, err + } + return len(batch) > 0, nil +} + +// lpxRetainedEngineScale requires nonnil requests and a nonempty retained slice. +// Grove removes a suffix when scaling down; reject targets that would retire any +// engine outside the failed batch, including a model sharing the same replica. +func lpxRetainedEngineScale(retained, batch []*lpxv1alpha1.LPUPipelineRequest) (string, int32, error) { + // Retain every replica with scheduler intent outside the failed batch. + groupName := "" + var replicas int64 + for _, request := range retained { + scope := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef + if scope == nil || scope.Name == "" || scope.ReplicaIndex < 0 || scope.ReplicaIndex >= math.MaxInt32 { + return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) + } + if groupName != "" && scope.Name != groupName { + return "", 0, fmt.Errorf("LPX requests target different scaling groups") + } + groupName = scope.Name + replicas = max(replicas, scope.ReplicaIndex+1) + } + + // A failed scale-out must lie entirely beyond the surviving engines. + for _, request := range batch { + scope := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef + if scope == nil || scope.Name != groupName || scope.ReplicaIndex < replicas { + return "", 0, fmt.Errorf("cannot retire LPX request %q without disturbing an engine outside its scheduling batch", request.Name) + } + } + return groupName, int32(replicas), nil +} diff --git a/deploy/operator/internal/controller/lpx/attempt_retirement_test.go b/deploy/operator/internal/controller/lpx/attempt_retirement_test.go new file mode 100644 index 000000000000..64fb1d9ff25b --- /dev/null +++ b/deploy/operator/internal/controller/lpx/attempt_retirement_test.go @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestLPXScaleOutRetirementProtectsUnrelatedAuthority(t *testing.T) { + const missingGroup = "missing group" + for _, scenario := range []string{"foreign group", "updated child", "scale conflict", missingGroup, "replacement request", "shared replica"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Publish two engines and record only the second engine in the failed batch") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + failed := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) + attempt := deadlineTestAttempt(child, failed, time.Now().Add(-time.Second)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + + t.Log("Change the observed authority before terminal cleanup attempts a destructive write") + wantError := true + switch scenario { + case "foreign group": + group.OwnerReferences[0].UID = "foreign-pcs" + require.NoError(t, r.Update(ctx, group)) + case "updated child": + newer := child.DeepCopy() + newer.Generation++ + require.NoError(t, r.Update(ctx, newer)) + case "scale conflict": + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { + return apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), group.Name, errors.New("stale scale")) + }, + }) + case missingGroup: + require.NoError(t, r.Delete(ctx, group)) + wantError = false + case "replacement request": + require.NoError(t, r.Delete(ctx, failed)) + failed.UID, failed.ResourceVersion = "replacement-uid", "" + require.NoError(t, r.Create(ctx, failed)) + wantError = false + case "shared replica": + failed.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex = 0 + require.NoError(t, r.Update(ctx, failed)) + } + + t.Log("Preserve the PCS and request identities when scale-down is unsafe or not yet possible") + requests := ownedLPXRequests(t, ctx, r, child) + pending, err := r.retireLPXAttempt(ctx, child, attempt, requests) + if wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, scenario == missingGroup, pending) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) + for _, before := range requests { + require.Equal(t, &before, getLPXRequest(t, ctx, r.Client, before.Namespace, before.Name)) + } + if scenario != missingGroup { + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + } + }) + } +} + +func TestLPXFailedSpecDecodeScaleOutRetiresAllModels(t *testing.T) { + t.Log("Publish one target and two draft requests for each of two engines") + ctx := t.Context() + child, source, registry := newLPXSpecDecodeTestDGD(t) + lpx.ServingComponent(source).Replicas = nil + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + group.Spec.Replicas = 2 + createLPXTestObjects(t, ctx, r.Client, objects...) + desired, rejected := requirePreparedLPX(t, r, ctx, child, source) + require.Nil(t, rejected) + publishSelectedLPXForTest(t, ctx, r, child, desired) + requests := ownedLPXRequests(t, ctx, r, child) + require.Len(t, requests, 6) + attempt := currentLPXAttemptStatus(child).DeepCopy() + attempt.Requests = nil + for _, request := range requests { + if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 1 { + attempt.Requests = append(attempt.Requests, deadlineTestAttempt(child, &request, time.Now()).Requests...) + } + } + attempt.ExceededAt = ptr.To(metav1.Now()) + require.Len(t, attempt.Requests, 3) + + t.Log("Retire all requests in the failed engine while preserving the serving engine's models") + for range 2 { + _, err := r.retireLPXAttempt(ctx, child, attempt, requests) + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + for _, request := range requests { + if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 0 { + require.Equal(t, &request, getLPXRequest(t, ctx, r.Client, request.Namespace, request.Name)) + } else { + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(&request), &lpxv1alpha1.LPUPipelineRequest{}))) + } + } +} + +func TestLPXScaleOutRetirementWaitsForOwnerGarbageCollection(t *testing.T) { + t.Log("Keep old requests observable after their shared PCS is already absent") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + failed := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) + attempt := deadlineTestAttempt(child, failed, time.Now().Add(-time.Second)) + pcs := findLPXTestPodCliqueSet(t, objects) + requests := ownedLPXRequests(t, ctx, r, child) + require.Len(t, requests, 2) + require.NoError(t, r.Delete(ctx, pcs)) + + t.Log("Keep terminal cleanup pending until owner GC removes every old request") + pending, err := r.retireLPXAttempt(ctx, child, attempt, requests) + require.NoError(t, err) + require.True(t, pending) + for index := range requests { + require.NoError(t, r.Delete(ctx, &requests[index])) + } + pending, err = r.retireLPXAttempt(ctx, child, attempt, nil) + require.NoError(t, err) + require.False(t, pending) +} diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go index 99319e2c38ac..07cfecd0d9c8 100644 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ b/deploy/operator/internal/controller/lpx/component_roles_test.go @@ -11,31 +11,12 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestLPXRoleDependenciesIncludeDraftAndConductor(t *testing.T) { - t.Log("Give both components consumed Agent claims and the conductor its own dependency") - source := newLPXSpecDecodeTestSource() - for _, component := range lpx.Components(source) { - pod := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec - pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(component.ComponentName + "-gpu")}} - pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} - } - conductor := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor) - conductor.PodTemplate.Spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("conductor-gpu")}} - conductor.PodTemplate.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "gpu"}} - before := source.DeepCopy() - - t.Log("Index each authored role's consumed claim without modifying the source") - require.ElementsMatch(t, []string{"draft-gpu", "lpx-gpu", "conductor-gpu"}, lpxDRAClaimReferences(true)(source)) - require.Equal(t, before, source) -} - func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { t.Log("Materialize two draft instances and one target in their single shared scaling group") deployment, source, registry := newLPXSpecDecodeTestDGD(t) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 248c0c03fd9f..5456ab76b631 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "maps" - "slices" "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -27,7 +26,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -35,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -const lpxGraphDeploymentFinalizer = "nvidia.com/lpx-graph-deployment" +const legacyLPXGraphDeploymentFinalizer = "nvidia.com/lpx-graph-deployment" func lpxSourceKey(deployment *v1alpha1.LPXGraphDeployment) (client.ObjectKey, error) { owner := metav1.GetControllerOf(deployment) @@ -70,7 +68,7 @@ type graphReconciler struct { recorder events.EventRecorder apiReader client.Reader runtimeConfig *commoncontroller.RuntimeConfig - modelRegistry lpxModelRegistry + modelRegistry lpx.ModelRegistry Config *configv1alpha1.OperatorConfiguration DockerSecretRetriever dynamo.SecretsRetriever } @@ -81,7 +79,7 @@ func (r *graphReconciler) GetRecorder() events.EventRecorder { // Setup registers the LPX controller and its dependencies. func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runtimeConfig *commoncontroller.RuntimeConfig, secrets dynamo.SecretsRetriever) error { - // Disabled integration still serves child status and explicit deletion. + // Disabled integration still serves child status; deletion uses owner garbage collection. r := &graphReconciler{ Client: mgr.GetClient(), recorder: mgr.GetEventRecorder("lpxgraphdeployment"), @@ -113,7 +111,7 @@ func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runti } // unavailableReason owns the LPX activation policy; an empty reason means enabled. -// Status and explicit deletion remain available regardless of this policy. +// Status remains available regardless of this policy. func (r *graphReconciler) unavailableReason() string { if !r.runtimeConfig.Gate.Enabled(features.Grove) { return "Grove is disabled" @@ -127,8 +125,7 @@ func (r *graphReconciler) unavailableReason() string { // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update -// +kubebuilder:rbac:groups=scheduler.grove.io,resources=podgangs,verbs=get;list;watch -// +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) { deployment := &v1alpha1.LPXGraphDeployment{} @@ -136,7 +133,10 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, client.IgnoreNotFound(err) } if !deployment.DeletionTimestamp.IsZero() { - return ctrl.Result{}, r.finalize(ctx, deployment) + if controllerutil.RemoveFinalizer(deployment, legacyLPXGraphDeploymentFinalizer) { + return ctrl.Result{}, r.Update(ctx, deployment) + } + return ctrl.Result{}, nil } unavailableReason := r.unavailableReason() @@ -204,11 +204,6 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu state = lpxResult(rejected) return r.retireInvalidWorkload(ctx, deployment, rejected.reason) } - if !controllerutil.ContainsFinalizer(deployment, lpxGraphDeploymentFinalizer) { - controllerutil.AddFinalizer(deployment, lpxGraphDeploymentFinalizer) - return ctrl.Result{RequeueAfter: time.Nanosecond}, r.Update(ctx, deployment) - } - selected, classification, err := r.reconcileLPXSafetyPreflight(ctx, deployment, source, nil) if err != nil { return ctrl.Result{}, err @@ -245,9 +240,8 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment, selected *lpxMaterializing) (reconcileOutcome, ctrl.Result, error) { state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} - // Pair retirement with the authoritative PCS, not a cached roster predating current requests. pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { if !apierrors.IsNotFound(err) { return state, ctrl.Result{}, err } @@ -260,52 +254,52 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { return state, ctrl.Result{}, err } - // Retire obsolete scheduler intent before changing its runtime or engine count. - rosterChanged := pcs != nil && metav1.IsControlledBy(pcs, deployment) && !slices.EqualFunc( - pcs.Spec.Template.PodCliqueScalingGroupConfigs, desired.Spec.Template.PodCliqueScalingGroupConfigs, - func(a, b grovev1alpha1.PodCliqueScalingGroupConfig) bool { - return a.Name == b.Name && slices.Equal(a.CliqueNames, b.CliqueNames) - }) - transition, err := r.retireChangedLPXRequests(ctx, deployment, selected, rosterChanged) + // Scale the live Grove group before stale request retirement. LPX scheduler + // finalizers may wait for Grove to remove the corresponding pods, so the + // publication fence must not block this write. + if err := r.reconcileLPXScalingGroupScale(ctx, source, pcs, selected.plan.LPXScalingGroup, true); err != nil { + return state, ctrl.Result{}, err + } + currents, classification, err := r.reconcileLPXPublicationFence(ctx, deployment, selected, pcs == nil) if err != nil { return state, ctrl.Result{}, err } - if transition != nil { - return lpxResult(transition), projectLPXLifecycleStatus(deployment, transition), nil + if classification != nil { + state = lpxResult(classification) + return state, projectLPXLifecycleStatus(deployment, classification), nil } for _, resource := range resources { if err := r.syncLPXResource(ctx, deployment, resource); err != nil { return state, ctrl.Result{}, err } } - synced, changed, retiring, err := r.reconcileGrovePodCliqueSetForLPX(ctx, deployment, lpx.ServingComponent(source).Replicas, pcs, desired) + // Grove seeds native scale only at creation; preserve that seed when replicas are omitted. + if pcs != nil && lpx.ServingComponent(source).Replicas == nil { + group := &desired.Spec.Template.PodCliqueScalingGroupConfigs[0] + for _, existing := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { + if existing.Name == group.Name { + group.Replicas = existing.Replicas + break + } + } + } + + changed, synced, err := commoncontroller.SyncObservedResource( + ctx, + r, + deployment, + pcs, + desired, + commoncontroller.WithPreservedListOrder(), + ) if err != nil { return state, ctrl.Result{}, err } - if retiring != nil { - return lpxResult(retiring), projectLPXLifecycleStatus(deployment, retiring), nil - } if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { return state, ctrl.Result{}, err } - // Grove seeds scaling groups once; only explicit component replicas override native scale. - if replicas := lpx.ServingComponent(source).Replicas; replicas != nil { - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.LPXScalingGroup}, group); err != nil { - if !apierrors.IsNotFound(err) { - return state, ctrl.Result{}, err - } - } else if !metav1.IsControlledBy(group, synced) || !group.DeletionTimestamp.IsZero() { - return state, ctrl.Result{}, fmt.Errorf("LPX scaling group lacks the current PCS owner") - } else if group.Spec.Replicas != *replicas { - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, - } - if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { - return state, ctrl.Result{}, err - } - } + if err := r.reconcileLPXScalingGroupScale(ctx, source, synced, selected.plan.LPXScalingGroup, false); err != nil { + return state, ctrl.Result{}, err } readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, source, deployment, synced) if err != nil { @@ -328,7 +322,7 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a } state.State = v1beta1.DGDStateSuccessful } - classification, err := r.reconcileSelectedLPX(ctx, deployment, selected) + classification, err = r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, selected, currents) if err != nil { return state, ctrl.Result{}, err } @@ -336,6 +330,40 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a return state, projectLPXLifecycleStatus(deployment, classification), nil } +// reconcileLPXScalingGroupScale applies explicit component replicas to the +// Grove group. Grove seeds this group once, so later replica changes use its +// scale subresource without rewriting the PCS template. +func (r *graphReconciler) reconcileLPXScalingGroupScale( + ctx context.Context, + source *v1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + groupName string, + scaleDownOnly bool, +) error { + replicas := lpx.ServingComponent(source).Replicas + if replicas == nil || pcs == nil { + return nil + } + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: groupName}, group); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if !metav1.IsControlledBy(group, pcs) || !group.DeletionTimestamp.IsZero() { + return fmt.Errorf("LPX scaling group lacks the current PCS owner") + } + if group.Spec.Replicas == *replicas || scaleDownOnly && group.Spec.Replicas < *replicas { + return nil + } + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, + } + return r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)) +} + // projectLPXLifecycleStatus transfers reconcile-owned results to the same child before returning. func projectLPXLifecycleStatus(deployment *v1alpha1.LPXGraphDeployment, classification lpxClassification) ctrl.Result { if transition, ok := classification.(*lpxDeadlineTransition); ok { @@ -362,17 +390,6 @@ func (r *graphReconciler) retireInvalidWorkload(ctx context.Context, deployment return ctrl.Result{}, nil } -func (r *graphReconciler) finalize(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { - if !controllerutil.ContainsFinalizer(deployment, lpxGraphDeploymentFinalizer) { - return nil - } - if err := r.finalizeLPXRequests(ctx, deployment); err != nil { - return err - } - controllerutil.RemoveFinalizer(deployment, lpxGraphDeploymentFinalizer) - return r.Update(ctx, deployment) -} - // Publication is an identity-sensitive write boundary: an uncached read fences // a source/child replacement or input edit, not an attempt to repair informer lag. func (r *graphReconciler) validateLPXPublicationSource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { @@ -451,16 +468,20 @@ func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployme // Discover obsolete ConfigMaps rooted in this exact LPX child. stale := make([]client.Object, 0) - if err := visitLifecycleObjectPages(ctx, r.apiReader, &corev1.ConfigMapList{}, func(object k8sruntime.Object) error { - configMap := object.(client.Object) + configMaps := &corev1.ConfigMapList{} + if err := r.apiReader.List(ctx, configMaps, + client.InNamespace(deployment.Namespace), + client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)}, + ); err != nil { + return err + } + for index := range configMaps.Items { + configMap := &configMaps.Items[index] if metav1.IsControlledBy(configMap, deployment) { if _, desired := desiredNames[configMap.GetName()]; !desired { stale = append(stale, configMap) } } - return nil - }, client.InNamespace(deployment.Namespace)); err != nil { - return err } if len(stale) == 0 { return nil @@ -515,7 +536,8 @@ func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1a if err != nil { return err } - // Keep the endpoint on this materialization even when another child shares the source. + // Keep cleanup and pod selection scoped to this materialization. + service.Labels[lpxOwnerUIDLabel] = string(deployment.UID) service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue service.Spec.Selector[grovecommon.LabelPartOfKey] = pcsName return r.syncLPXResource(ctx, deployment, service) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 0271affd27dd..d7172c47e424 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -113,118 +113,159 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { } } -func TestLPXIntentChangesRetireRequestsBeforeGroveWrites(t *testing.T) { +func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + + stale := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) + stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, stale)) + + liveSource := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), liveSource)) + lpx.ServingComponent(liveSource).Replicas = ptr.To(int32(1)) + liveSource.Generation++ + require.NoError(t, r.Update(ctx, liveSource)) + liveChild := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) + liveChild.Generation++ + var err error + liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveSource, "") + require.NoError(t, err) + require.NoError(t, r.Update(ctx, liveChild)) + + t.Log("Persist the lower Grove scale before request retirement can block reconciliation") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Equal(t, int32(1), group.Spec.Replicas) + stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.True(t, stale.DeletionTimestamp.IsZero()) + + t.Log("Delete the stale request while its scheduler finalizer remains pending") + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.False(t, stale.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), &grovev1alpha1.PodCliqueSet{})) + + t.Log("Reverse the scale-down while the old request still protects its pods") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), liveSource)) + lpx.ServingComponent(liveSource).Replicas = ptr.To(int32(2)) + liveSource.Generation++ + require.NoError(t, r.Update(ctx, liveSource)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) + liveChild.Generation++ + liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveSource, "") + require.NoError(t, err) + require.NoError(t, r.Update(ctx, liveChild)) + for range 2 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas, "Grove must not recreate pods protected by the retiring request") + } + + t.Log("Allow scale-up and a fresh request once the scheduler finishes cleanup") + stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + retiredUID := stale.UID + stale.Finalizers = nil + require.NoError(t, r.Update(ctx, stale)) + for range 4 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + replacement := getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.NotEqual(t, retiredUID, replacement.UID) + require.True(t, replacement.DeletionTimestamp.IsZero()) +} + +func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { for _, tc := range []struct { - name string - replicas int32 - drafts int32 - retire []int - wantWrites []string - }{ - {name: "settings", replicas: 2, retire: []int{0, 1}, wantWrites: []string{"pcs"}}, - {name: "scale in", replicas: 1, retire: []int{1}, wantWrites: []string{"pcs", "scale"}}, - {name: "scale out", replicas: 3, wantWrites: []string{"pcs", "scale"}}, - {name: "metadata", replicas: 2}, - {name: "stale cached PCS", replicas: 2}, - {name: "image", replicas: 2, wantWrites: []string{"pcs"}}, - {name: "add draft", replicas: 1, drafts: 3, retire: []int{0, 1, 2}, wantWrites: []string{"pcs"}}, - {name: "remove draft", replicas: 1, drafts: 1, retire: []int{0, 1, 2}, wantWrites: []string{"pcs"}}, - } { + name string + missing bool + }{{name: "deadline"}, {name: "missing request", missing: true}} { t.Run(tc.name, func(t *testing.T) { - t.Log("Publish the engines and hold their scheduler cleanup finalizers") - var child *v1alpha1.LPXGraphDeployment - var source *v1beta1.DynamoGraphDeployment - var registry *lpx.ModelRegistry - if tc.drafts != 0 { - child, source, registry = newLPXSpecDecodeTestDGD(t) + t.Log("Observe a completed engine and a newer scheduling batch in their shared PCS") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + serving := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(ctx, serving)) + pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) + pending.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, pending)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + attempt := deadlineTestAttempt(child, pending, time.Now().Add(-time.Second)) + if tc.missing { + attempt.DeadlineAt = nil + pending.Finalizers = nil + require.NoError(t, r.Update(ctx, pending)) + require.NoError(t, r.Delete(ctx, pending)) } else { - child, source, registry = newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + attempt.ExceededAt = ptr.To(metav1.Now()) } - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, t.Context(), r.Client, objects...) - _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) - require.NoError(t, err) - requests := make([]*lpxv1alpha1.LPUPipelineRequest, len(selected.requests)) - for i, desired := range selected.requests { - requests[i] = getLPXRequest(t, t.Context(), r.Client, child.Namespace, desired.requestName) - requests[i].Finalizers = []string{"test.example/scheduler"} - require.NoError(t, r.Update(t.Context(), requests[i])) + child.Status.Placement.LPXAttempt = attempt + require.NoError(t, r.Status().Update(ctx, child)) + + t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) + for range 3 { + _, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) + require.Equal(t, serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) } - var wantRetired []string - for _, i := range tc.retire { - wantRetired = append(wantRetired, requests[i].Name) + if tc.missing { + requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + return } - t.Log("Select the edited source while keeping the original Grove children") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - component := lpx.ServingComponent(source) - component.Replicas = ptr.To(tc.replicas) - switch tc.name { - case "settings": - component.LPX.Settings.Raw = []byte(`{"prop_sync":false}`) - case "metadata": - source.Annotations["test.example/unrelated"] = "updated" - case "image": - component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "runtime:next" - case "add draft", "remove draft": - source.GetComponentByName("draft").Replicas = ptr.To(tc.drafts) - } - source.Generation++ - require.NoError(t, r.Update(t.Context(), source)) - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") - require.NoError(t, err) + t.Log("A later edit still waits for the failed request's scheduler finalizer") + pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.False(t, pending.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) child.Generation++ - require.NoError(t, r.Update(t.Context(), child)) - selected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) - require.Nil(t, rejected) - - t.Log("Affected requests must be deleting before either Grove update, without awaiting finalizers") - var retired, writes []string - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - err := delegated.Get(ctx, key, object, opts...) - if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok && err == nil && tc.name == "stale cached PCS" { - pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames = []string{"stale-agent"} - } - return err - }, - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { - retired = append(retired, object.GetName()) - } - return delegated.Delete(ctx, object, opts...) - }, - Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - require.ElementsMatch(t, wantRetired, retired, "retirement must precede PCS updates") - writes = append(writes, "pcs") - } - return delegated.Update(ctx, object, opts...) - }, - SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - require.ElementsMatch(t, wantRetired, retired, "retirement must precede scale updates") - writes = append(writes, subresource) - return delegated.SubResource(subresource).Update(ctx, object, opts...) - }, - }) - _, _, err = r.reconcileWorkload(t.Context(), child, source, selected) + require.NoError(t, r.Update(ctx, child)) + _, err := r.Reconcile(ctx, request) require.NoError(t, err) - require.ElementsMatch(t, wantRetired, retired) - require.Equal(t, tc.wantWrites, writes) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) - t.Log("The PCS and every unchanged scheduler request retain their identity") - pcs := findLPXTestPodCliqueSet(t, objects) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) - require.EqualValues(t, "pcs-uid", pcs.UID) - require.Equal(t, int32(1), pcs.Spec.Replicas) - for _, before := range requests { - after := getLPXRequest(t, t.Context(), r.Client, child.Namespace, before.Name) - require.Equal(t, before.UID, after.UID) - require.Equal(t, before.Finalizers, after.Finalizers) - require.Equal(t, slices.Contains(wantRetired, before.Name), !after.DeletionTimestamp.IsZero()) + t.Log("After cleanup the later edit can retry without replacing the serving engine") + retiredUID := pending.UID + pending.Finalizers = nil + require.NoError(t, r.Update(ctx, pending)) + for range 5 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) } + replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NotEqual(t, retiredUID, replacement.UID) + require.True(t, replacement.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + require.Equal(t, serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)) }) } } @@ -241,8 +282,7 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { }) r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) - _, err := r.reconcileSelectedLPX(t.Context(), child, selected) - require.NoError(t, err) + publishSelectedLPXForTest(t, t.Context(), r, child, selected) beforeRequests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(t.Context(), beforeRequests)) require.NotEmpty(t, beforeRequests.Items) @@ -268,7 +308,7 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, afterSelected) require.Equal(t, beforePCS, afterPCS) require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) - _, err = r.reconcileSelectedLPX(t.Context(), child, afterSelected) + _, err := reconcileSelectedLPXForTest(t.Context(), r, child, afterSelected) require.NoError(t, err) afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(t.Context(), afterRequests)) @@ -315,7 +355,7 @@ func TestLPXGPUCapacityReportsTheCompleteEngine(t *testing.T) { t.Log("Reconcile the actual LPX workload and publish its per-engine GPU shape") var child *v1alpha1.LPXGraphDeployment var source *v1beta1.DynamoGraphDeployment - var registry *lpx.ModelRegistry + var registry lpx.ModelRegistry var want int64 switch mode { case "lpu-only": @@ -356,7 +396,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { {name: "inconsistent snapshot", reason: "LPXReconciliationFailed", message: "immutable LPX build snapshot"}, } { t.Run(scenario.name, func(t *testing.T) { - t.Log("Stage a real PCS and discovery endpoint before Grove can supply publication identities") + t.Log("Stage a PCS and discovery endpoint before the API supplies a PCS UID") child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) r := newLPXTestReconciler(t, registry, child, source) @@ -377,14 +417,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { require.NoError(t, r.List(t.Context(), requests)) require.Empty(t, requests.Items) - t.Log("Healthy repeated reconciliation must preserve unpublished workload identities") - _, err = r.Reconcile(t.Context(), request) - require.NoError(t, err) beforePCS, beforeEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() - require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - require.Equal(t, beforePCS, pcs) - require.Equal(t, beforeEndpoint, endpoint) t.Log("Introduce a current terminal input failure, or a transient snapshot outage") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) @@ -448,14 +481,15 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T source := newLPXTestSource(lpx.PipelineSingle, "build-v2") child := newLPXTestDeployment(t, source) owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} + labels := map[string]string{lpxOwnerUIDLabel: string(child.UID)} pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-pcs", Namespace: child.Namespace, UID: "old-pcs", OwnerReferences: owner, + Name: "obsolete-pcs", Namespace: child.Namespace, UID: "old-pcs", Labels: labels, OwnerReferences: owner, }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}} endpoint := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-endpoint", Namespace: child.Namespace, UID: "old-endpoint", OwnerReferences: owner, + Name: "obsolete-endpoint", Namespace: child.Namespace, UID: "old-endpoint", Labels: labels, OwnerReferences: owner, }} runtimeConfig := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-runtime", Namespace: child.Namespace, UID: "old-runtime", OwnerReferences: owner, + Name: "obsolete-runtime", Namespace: child.Namespace, UID: "old-runtime", Labels: labels, OwnerReferences: owner, }} ordinary := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ Name: "ordinary-model", Namespace: child.Namespace, UID: "ordinary-service", @@ -626,6 +660,13 @@ func TestLPXEndpointLifecycle(t *testing.T) { delete(source.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) + + t.Log("Retire the recreated discovery endpoint when the workload becomes invalid") + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + _, err := r.retireInvalidLPXWorkload(t.Context(), child, "invalid source") + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) require.NoError(t, r.Get(t.Context(), modelKey, modelService)) require.Equal(t, beforeModelService, modelService, "LPX cleanup must leave the DGD-owned model Service unchanged") } @@ -640,6 +681,7 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) objects := lpxMaterializedObjects(t, r, child, source, selected) createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, selected) t.Log("Route Grove objects to the materialization while retaining the source provenance") pcs := objects[0].(*grovev1alpha1.PodCliqueSet) @@ -647,15 +689,9 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { for _, object := range objects[1:] { require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(child)}}, mapLPXChildToRequests(t.Context(), object)) } - gangs, err := r.listLPXPublicationPodGangs(t.Context(), child, pcs) - require.NoError(t, err) - require.NotEmpty(t, gangs) - for _, gang := range gangs { - require.Equal(t, source.Name, gang.Labels[consts.KubeLabelDynamoGraphDeploymentName]) - } t.Log("Publish the runtime configuration and endpoint under the exact LGD owner") - _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) root := dynamo.PCSNameForLPX(child) @@ -847,13 +883,16 @@ func TestLPXDeletesOnlyStaleOwnedRuntimeConfigMaps(t *testing.T) { child := newLPXTestDeployment(t, source) owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} current := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "runtime-current", Namespace: child.Namespace, UID: "current", OwnerReferences: owner, + Name: "runtime-current", Namespace: child.Namespace, UID: "current", + Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, OwnerReferences: owner, }} stale := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "runtime-stale", Namespace: child.Namespace, UID: "stale", OwnerReferences: owner, + Name: "runtime-stale", Namespace: child.Namespace, UID: "stale", + Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, OwnerReferences: owner, }} foreign := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ Name: "runtime-foreign", Namespace: child.Namespace, UID: "foreign", + Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, }} r := newLPXTestReconciler(t, nil, child, source, current, stale, foreign) @@ -1002,7 +1041,6 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { t.Log("Prepare an immutable render from the current source and child") child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - pcs := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, selected) require.NoError(t, r.validateLPXPublicationSource(t.Context(), child)) t.Log("A source edit racing publication prevents the old render from creating a PCS") @@ -1010,9 +1048,8 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, retiring, err := r.reconcileGrovePodCliqueSetForLPX(t.Context(), child, lpx.ServingComponent(source).Replicas, nil, pcs) + _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) require.ErrorContains(t, err, "input revision") - require.Nil(t, retiring) allPCS := &grovev1alpha1.PodCliqueSetList{} require.NoError(t, r.List(t.Context(), allPCS)) require.Empty(t, allPCS.Items) diff --git a/deploy/operator/internal/controller/lpx/dependencies.go b/deploy/operator/internal/controller/lpx/dependencies.go deleted file mode 100644 index ea8e02605a7a..000000000000 --- a/deploy/operator/internal/controller/lpx/dependencies.go +++ /dev/null @@ -1,154 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dra" - resourcev1 "k8s.io/api/resource/v1" - "k8s.io/apimachinery/pkg/util/sets" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - lpxTopologyBindingRefIndex = "lpx.topologyBinding" - lpxResourceClaimRefIndex = "lpx.resourceClaim" - lpxResourceClaimTemplateRefIndex = "lpx.resourceClaimTemplate" - lpxDeviceClassRefIndex = "lpx.deviceClass" -) - -// lpxTopologyBindingReferences indexes non-nil DGD objects by their LPX topology policy. -func lpxTopologyBindingReferences(obj client.Object) []string { - source := obj.(*nvidiacomv1beta1.DynamoGraphDeployment) - if !source.HasLPXComponent() || source.Spec.Experimental == nil || source.Spec.Experimental.KvTransferPolicy == nil { - return nil - } - name := source.Spec.Experimental.KvTransferPolicy.ClusterTopologyName - if name == "" { - return nil - } - return []string{name} -} - -// lpxDRAClaimReferences indexes non-nil DGD objects by consumed GPU-role claims. -// Extraction uses only source intent, including references not yet created. -func lpxDRAClaimReferences(template bool) client.IndexerFunc { - return func(obj client.Object) []string { - // Index dependencies from every independently authored role. - source := obj.(*nvidiacomv1beta1.DynamoGraphDeployment) - references := sets.New[string]() - for index := range source.Spec.Components { - component := &source.Spec.Components[index] - if !component.IsLPX() { - continue - } - for _, role := range component.Roles { - if role.PodTemplate == nil { - continue - } - pod := &role.PodTemplate.Spec - consumed := sets.New[string]() - for _, container := range dra.AllContainers(pod) { - for _, claim := range container.Resources.Claims { - consumed.Insert(claim.Name) - } - } - for _, claim := range pod.ResourceClaims { - if !consumed.Has(claim.Name) { - continue - } - name := claim.ResourceClaimName - if template { - name = claim.ResourceClaimTemplateName - } - if name != nil && *name != "" { - references.Insert(*name) - } - } - } - } - return references.UnsortedList() - } -} - -// lpxDeviceClassReferences indexes claims and templates, never fetched DGD state. -// A claim-only edit therefore updates this index without requiring a DGD edit. -func lpxDeviceClassReferences(obj client.Object) []string { - var spec *resourcev1.ResourceClaimSpec - switch resource := obj.(type) { - case *resourcev1.ResourceClaim: - spec = &resource.Spec - case *resourcev1.ResourceClaimTemplate: - spec = &resource.Spec.Spec - default: - return nil - } - - // Every alternative can become the selected device request. - classes := sets.New[string]() - for _, request := range spec.Devices.Requests { - if request.Exactly != nil && request.Exactly.DeviceClassName != "" { - classes.Insert(request.Exactly.DeviceClassName) - } - for _, alternative := range request.FirstAvailable { - if alternative.DeviceClassName != "" { - classes.Insert(alternative.DeviceClassName) - } - } - } - return classes.UnsortedList() -} - -func (r *graphReconciler) indexedLPXDependencyRequests(ctx context.Context, dependency client.Object, index string) []ctrl.Request { - // Cached field queries load only the LPX sources referencing this dependency. - sources := &nvidiacomv1beta1.DynamoGraphDeploymentList{} - if err := r.List(ctx, sources, client.InNamespace(dependency.GetNamespace()), client.MatchingFields{index: dependency.GetName()}); err != nil { - ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX sources for dependency", "index", index, "dependency", client.ObjectKeyFromObject(dependency)) - return nil - } - - // Respect each source's namespace filter and enqueue its exact owned materializations. - requests := make([]ctrl.Request, 0, len(sources.Items)) - for i := range sources.Items { - source := &sources.Items[i] - if commoncontroller.NamespaceAllowed(r.Config, r.runtimeConfig, source, source.Namespace) { - requests = append(requests, r.mapLPXSourceToRequests(ctx, source)...) - } - } - return requests -} - -func (r *graphReconciler) mapLPXDeviceClassToRequests(ctx context.Context, class *resourcev1.DeviceClass) []ctrl.Request { - // DeviceClass names live on claims/templates, so resolve both indexed hops. - claims := &resourcev1.ResourceClaimList{} - if err := r.List(ctx, claims, client.MatchingFields{lpxDeviceClassRefIndex: class.Name}); err != nil { - ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX ResourceClaim dependencies", "deviceClass", class.Name) - return nil - } - templates := &resourcev1.ResourceClaimTemplateList{} - if err := r.List(ctx, templates, client.MatchingFields{lpxDeviceClassRefIndex: class.Name}); err != nil { - ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX ResourceClaimTemplate dependencies", "deviceClass", class.Name) - return nil - } - - // Multiple claims and engines can refer to the same class in one graph. - consumers := sets.New[ctrl.Request]() - for i := range claims.Items { - for _, request := range r.indexedLPXDependencyRequests(ctx, &claims.Items[i], lpxResourceClaimRefIndex) { - consumers.Insert(request) - } - } - for i := range templates.Items { - for _, request := range r.indexedLPXDependencyRequests(ctx, &templates.Items[i], lpxResourceClaimTemplateRefIndex) { - consumers.Insert(request) - } - } - return consumers.UnsortedList() -} diff --git a/deploy/operator/internal/controller/lpx/grove.go b/deploy/operator/internal/controller/lpx/grove.go deleted file mode 100644 index 0befaf38ab5a..000000000000 --- a/deploy/operator/internal/controller/lpx/grove.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package lpx - -import ( - "context" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" -) - -// reconcileGrovePodCliqueSetForLPX converges the already-rendered PodCliqueSet -// through native Grove rollout. existingPodCliqueSet may be nil when absent; -// nil replicas leaves scale to Grove. deployment and desired are non-nil. -func (r *graphReconciler) reconcileGrovePodCliqueSetForLPX( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - replicas *int32, - existingPodCliqueSet *grovev1alpha1.PodCliqueSet, - desired *grovev1alpha1.PodCliqueSet, -) (*grovev1alpha1.PodCliqueSet, bool, *lpxRetiring, error) { - // A missing observation permits only creation after the publication fence. - if existingPodCliqueSet == nil { - if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { - return nil, false, nil, err - } - retiring, err := r.retireFirstOwnedLPXRequest(ctx, deployment, desired.Name, "LPX publication was retired before synchronizing a Grove PodCliqueSet spec change") - if err != nil || retiring != nil { - return nil, false, retiring, err - } - } - - // Grove seeds native scale only at creation; preserve that seed when replicas are omitted. - if existingPodCliqueSet != nil && replicas == nil { - group := &desired.Spec.Template.PodCliqueScalingGroupConfigs[0] - for _, existing := range existingPodCliqueSet.Spec.Template.PodCliqueScalingGroupConfigs { - if existing.Name == group.Name { - group.Replicas = existing.Replicas - break - } - } - } - - // Match normal PCS synchronization; Grove owns rollout and immutable-field validation. - modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, existingPodCliqueSet, desired, commoncontroller.WithPreservedListOrder()) - return synced, modified, nil, err -} diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index eb23009cf2c7..2d6fffb147b5 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -20,15 +20,11 @@ import ( "unicode/utf8" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -43,11 +39,10 @@ import ( const ( lpxAttemptDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-attempt-digest" lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation + lpxOwnerUIDLabel = lpxDeploymentUIDAnnotation lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" lpxPCSUIDAnnotation = "scheduling.lpu.nvidia.com/podcliqueset-uid" - lpxPodGangUIDAnnotation = "scheduling.lpu.nvidia.com/podgang-uid" lpxRetirementRequeueAfter = 5 * time.Second - lpxLifecycleListPageSize = 64 maxDGDConditionMessageSize = 32768 ) @@ -56,8 +51,6 @@ const ( lpxSchedulerStatusInvalidReason string = "LPXSchedulerStatusInvalid" ) -var errLPXEngineReplacing = errors.New("Grove is replacing the engine's PodCliques") - // lpxClassification is deliberately private and closed: reconciliation can // report only one legal protocol state, never a freely assembled collection // of correlated booleans. @@ -90,23 +83,13 @@ type lpxModelMaterializing struct { attemptDigest string modelProjection *lpx.ModelProjection replicaIndex int32 -} - -type lpxGroveSnapshot struct { - podCliqueSet *grovev1alpha1.PodCliqueSet - scalingGroup *grovev1alpha1.PodCliqueScalingGroup - generationHash string - incomplete string - podGangs []groveschedulerv1alpha1.PodGang + agentCliqueName string } type lpxGroveIdentity struct { - pcsUID types.UID - podGangName string - podGangUID types.UID - cyborgClique *lpxv1alpha1.CyborgPodCliqueReference - agentUIDs []types.UID - complete bool + pcsUID types.UID + pcs *grovev1alpha1.PodCliqueSet + plan *lpx.MaterializationPlan } type lpxClosed struct { @@ -165,72 +148,44 @@ func lpxResult(classification lpxClassification) reconcileOutcome { // lpxSchedulerResult classifies the receipt without formatting diagnostics before selection. func lpxSchedulerResult(state *lpxSchedulerObserved) reconcileOutcome { result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} + var summary string switch state.Phase { case lpxv1alpha1.RequestPhasePending: result.Reason = "LPXSchedulerPending" - result.Message = "LPX scheduler is waiting to plan the current request" + summary = "LPX scheduler is waiting to plan the current request" case lpxv1alpha1.RequestPhaseNoFit: result.Reason = "LPXNoFit" - result.Message = "LPX scheduler found no placement for the current request" + summary = "LPX scheduler found no placement for the current request" case lpxv1alpha1.RequestPhaseUnsupported: result.State = nvidiacomv1beta1.DGDStateFailed result.Reason = "LPXUnsupported" - result.Message = "LPX scheduler cannot support the current request" + summary = "LPX scheduler cannot support the current request" case lpxv1alpha1.RequestPhasePlanned: result.Reason = "LPXPlanned" - result.Message = "LPX scheduler committed a placement plan for the current request" + summary = "LPX scheduler committed a placement plan for the current request" case lpxv1alpha1.RequestPhaseReserving: result.Reason = "LPXReserving" - result.Message = "LPX scheduler is reserving the planned LPU allocation" + summary = "LPX scheduler is reserving the planned LPU allocation" case lpxv1alpha1.RequestPhaseBinding: result.Reason = "LPXBinding" - result.Message = "LPX scheduler is binding the planned workload Pods" + summary = "LPX scheduler is binding the planned workload Pods" case lpxv1alpha1.RequestPhaseDegraded: - result.State = nvidiacomv1beta1.DGDStateFailed result.Reason = "LPXDegraded" - result.Message = "LPX scheduler detected that the committed plan is no longer valid" - case lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: - result = lpxSchedulerReleaseResult(state) - case lpxv1alpha1.RequestPhaseBound: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = lpxSchedulerStatusInvalidReason - result.Message = "LPX scheduler reported Bound without the exact current-generation plan proof required by Dynamo" - default: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = lpxSchedulerStatusInvalidReason - result.Message = fmt.Sprintf("LPX scheduler reported unknown phase %q", state.Phase) - } - return result -} - -func lpxSchedulerReleaseResult(state *lpxSchedulerObserved) reconcileOutcome { - result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} - phase := "releasing" - result.Reason = "LPXReleasing" - result.Message = "LPX scheduler is releasing the current plan" - if state.Phase == lpxv1alpha1.RequestPhaseReleased { - phase = "released" + summary = "LPX scheduler is repairing or releasing a degraded placement" + case lpxv1alpha1.RequestPhaseReleasing: + result.Reason = "LPXReleasing" + summary = "LPX scheduler is releasing the current plan" + case lpxv1alpha1.RequestPhaseReleased: result.Reason = "LPXReleased" - result.Message = "LPX scheduler released the current plan and is finalizing its journal" - } - if state.Committed == nil || state.Committed.Execution.Release == nil { - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = lpxSchedulerStatusInvalidReason - result.Message = fmt.Sprintf("LPX scheduler reported %s without the required release journal", phase) - return result - } - releaseReason := state.Committed.Execution.Release.Reason - switch releaseReason { - case lpxv1alpha1.ReleaseReasonDependencyChanged, lpxv1alpha1.ReleaseReasonBindingFailed: - result.State = nvidiacomv1beta1.DGDStateFailed - case lpxv1alpha1.ReleaseReasonRequestChanged, lpxv1alpha1.ReleaseReasonRequestDeleted: + summary = "LPX scheduler released the current plan" + case lpxv1alpha1.RequestPhaseBound: + return lpxResult(&lpxBound{}) default: result.State = nvidiacomv1beta1.DGDStateFailed result.Reason = lpxSchedulerStatusInvalidReason - result.Message = fmt.Sprintf("LPX scheduler reported %s with unknown release reason %q", phase, releaseReason) - return result + summary = fmt.Sprintf("LPX scheduler reported unknown phase %q", state.Phase) } - result.Message = fmt.Sprintf("%s after %s", result.Message, releaseReason) + result.Message = lpxMessageWithDiagnostics(summary, state.Diagnostics) return result } @@ -310,7 +265,7 @@ func (r *graphReconciler) reconcileLPXKnownIntentFence( if selected { reason = "The DGD incarnation, generation, model, or selected input changed" } - retiring, retireErr := r.retireLPXRequest(ctx, deployment, "", request, reason) + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, reason, false) return retiring, nil, retireErr } return nil, requests, nil @@ -357,7 +312,7 @@ func (r *graphReconciler) prepareLPXMaterializing( workloadDigest := workload.Digest().String() requests := make([]lpxModelMaterializing, 0, len(projections)*int(plan.Replicas)) for replicaIndex := int32(0); replicaIndex < plan.Replicas; replicaIndex++ { - for _, projection := range projections { + for projectionIndex, projection := range projections { requestDigest := digestAttemptKey( deployment.Namespace, deployment.Name, @@ -369,6 +324,7 @@ func (r *graphReconciler) prepareLPXMaterializing( requestName: lpxRequestName(deployment.Name, requestDigest), modelProjection: projection, replicaIndex: replicaIndex, + agentCliqueName: plan.ForReplica(replicaIndex).Agents[projectionIndex].CliqueName, }) } } @@ -388,47 +344,66 @@ func (r *graphReconciler) retireInvalidLPXWorkload( deployment *nvidiacomv1alpha1.LPXGraphDeployment, reason string, ) (*lpxRetiring, error) { - // Preserve request-first retirement before removing unpublished staging. - requests, err := r.listOwnedLPXRequests(ctx, deployment) - if err != nil { + selector := client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)} + owned := make([]client.Object, 0) + podCliqueSets := &grovev1alpha1.PodCliqueSetList{} + if err := r.apiReader.List(ctx, podCliqueSets, client.InNamespace(deployment.Namespace), selector); err != nil { return nil, err } - if len(requests) > 0 { - return r.retireLPXRequest(ctx, deployment, "", &requests[0], reason) + for index := range podCliqueSets.Items { + if pcs := &podCliqueSets.Items[index]; metav1.IsControlledBy(pcs, deployment) { + owned = append(owned, pcs) + } } - // Source edits can rename resources; discover only this exact child's old objects. - owned := make([]client.Object, 0) - for _, list := range []client.ObjectList{&corev1.ServiceList{}, &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}} { - if err := visitLifecycleObjectPages(ctx, r.apiReader, list, func(object k8sruntime.Object) error { - resource := object.(client.Object) - if metav1.IsControlledBy(resource, deployment) { - owned = append(owned, resource) - } - return nil - }, client.InNamespace(deployment.Namespace)); err != nil { - return nil, err - } + services := &corev1.ServiceList{} + if err := r.apiReader.List(ctx, services, client.InNamespace(deployment.Namespace), selector); err != nil { + return nil, err } - if len(owned) == 0 { - return nil, nil + for index := range services.Items { + if service := &services.Items[index]; metav1.IsControlledBy(service, deployment) { + owned = append(owned, service) + } } - // A newer child seen during listing must not lose its staging to this stale failure. - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + configMaps := &corev1.ConfigMapList{} + if err := r.apiReader.List(ctx, configMaps, client.InNamespace(deployment.Namespace), selector); err != nil { return nil, err } - for _, resource := range owned { - if !resource.GetDeletionTimestamp().IsZero() { - continue + for index := range configMaps.Items { + if configMap := &configMaps.Items[index]; metav1.IsControlledBy(configMap, deployment) { + owned = append(owned, configMap) } - uid, resourceVersion := resource.GetUID(), resource.GetResourceVersion() - if err := r.Delete(ctx, resource, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ - UID: &uid, ResourceVersion: &resourceVersion, - }}); err != nil && !apierrors.IsNotFound(err) { + } + + requests, err := r.listOwnedLPXRequests(ctx, deployment) + if err != nil { + return nil, err + } + if len(owned) == 0 && len(requests) == 0 { + return nil, nil + } + if len(owned) > 0 { + // A newer child seen during listing must not lose its staging to this stale failure. + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { return nil, err } + + propagation := metav1.DeletePropagationBackground + for _, resource := range owned { + if !resource.GetDeletionTimestamp().IsZero() { + continue + } + uid, resourceVersion := resource.GetUID(), resource.GetResourceVersion() + if err := r.Delete(ctx, resource, &client.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}, + PropagationPolicy: &propagation, + }); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + } } + return &lpxRetiring{retirementReason: reason}, nil } @@ -482,11 +457,13 @@ func lpxRequestName(dgdName, digest string) string { return fmt.Sprintf("lpx-%s-%s", prefix, hexDigest[:32]) } -// reconcileLPXAttemptFence requires a nonnil desired whose requests remain immutable during the call. -func (r *graphReconciler) reconcileLPXAttemptFence( +// reconcileLPXPublicationFence requires a nonnil desired whose requests remain immutable during the call. +// A missing PCS cannot be recreated until garbage collection removes every request from its previous incarnation. +func (r *graphReconciler) reconcileLPXPublicationFence( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, + podCliqueSetMissing bool, ) (map[string]*lpxv1alpha1.LPUPipelineRequest, lpxClassification, error) { // Retain each desired position for digest lookup and collision precedence. desiredByName := make(map[string]int, len(desired.requests)) @@ -495,44 +472,43 @@ func (r *graphReconciler) reconcileLPXAttemptFence( } currents := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) foreignCollision := len(desired.requests) - var stale *lpxv1alpha1.LPUPipelineRequest - if err := visitLifecycleObjectPages( - ctx, - r.apiReader, - &lpxv1alpha1.LPUPipelineRequestList{}, - func(object k8sruntime.Object) error { - request := object.(*lpxv1alpha1.LPUPipelineRequest) - desiredIndex, desiredName := desiredByName[request.Name] - if !metav1.IsControlledBy(request, deployment) { - // Preserve desired-order error precedence without retaining collision state. - if desiredName { - foreignCollision = min(foreignCollision, desiredIndex) - } - return nil - } + stale := make([]*lpxv1alpha1.LPUPipelineRequest, 0) + requests, err := r.listLPXRequestCandidates(ctx, deployment) + if err != nil { + return nil, nil, err + } + for index := range requests.Items { + request := &requests.Items[index] + desiredIndex, desiredName := desiredByName[request.Name] + ownedByChild, err := r.ownsLPXRequest(ctx, deployment, request) + if err != nil { + return nil, nil, err + } + if !ownedByChild { + // Preserve desired-order error precedence without retaining collision state. if desiredName { - currents[request.Name] = request - return nil + foreignCollision = min(foreignCollision, desiredIndex) } - if stale == nil || request.Name < stale.Name { - stale = request - } - return nil - }, - client.InNamespace(deployment.Namespace), - ); err != nil { - return nil, nil, err + continue + } + if desiredName { + currents[request.Name] = request + continue + } + stale = append(stale, request) } - if stale != nil { - // Persist cancellation or a replacement scheduling batch before retiring removed engines. + if len(stale) > 0 { + // Persist cancellation before deleting scheduler intent for removed engines. if attempt := currentLPXAttemptStatus(deployment); attempt != nil && attempt.ExceededAt == nil { want := desiredLPXAttemptStatus(desired, deployment.Generation, attempt.PodCliqueSetUID) if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, want, currents); transition != nil { return currents, transition, nil } } - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, stale, "The engine is no longer requested", true) - return currents, retiring, retireErr + if err := r.deleteStaleLPXRequests(ctx, deployment, stale); err != nil { + return nil, nil, err + } + return currents, &lpxRetiring{retirementReason: "One or more engines are no longer requested"}, nil } // Keep exact cleanup authority until the replaced request is authoritatively absent. if attempt := currentLPXAttemptStatus(deployment); attempt != nil { @@ -554,224 +530,153 @@ func (r *graphReconciler) reconcileLPXAttemptFence( if foreignCollision < len(desired.requests) { return nil, nil, fmt.Errorf("LPX request name %q is occupied by a foreign owner", desired.requests[foreignCollision].requestName) } - return currents, nil, nil -} + if podCliqueSetMissing && len(currents) > 0 { + return currents, &lpxRetiring{retirementReason: "Waiting for garbage collection to remove the previous LPX requests"}, nil + } -func visitLifecycleObjectPages( - ctx context.Context, - reader client.Reader, - list client.ObjectList, - visit func(k8sruntime.Object) error, - opts ...client.ListOption, -) error { - continueToken := "" - pageOptions := make([]client.ListOption, 0, len(opts)+2) - for { - page := list.DeepCopyObject().(client.ObjectList) - pageOptions = pageOptions[:0] - pageOptions = append(pageOptions, opts...) - pageOptions = append(pageOptions, client.Limit(lpxLifecycleListPageSize)) - if continueToken != "" { - pageOptions = append(pageOptions, client.Continue(continueToken)) - } - if err := reader.List(ctx, page, pageOptions...); err != nil { - return err - } - if err := meta.EachListItem(page, visit); err != nil { - return err - } - nextToken := page.GetContinue() - if nextToken == "" { - break - } - if nextToken == continueToken { - return fmt.Errorf("lifecycle list %T repeated continuation token %q", list, nextToken) + // Keep scale-up fenced until terminal cleanup can retire the failed engines. + if attempt := currentLPXAttemptStatus(deployment); attempt != nil { + _, _, missing := exactLPXAttemptState(attempt, currents) + if attempt.ExceededAt != nil || missing { + return currents, &lpxRetiring{retirementReason: "Waiting for the terminal scheduling batch to finish cleanup"}, nil } - continueToken = nextToken } - return nil -} -func (r *graphReconciler) listOwnedLPXRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) ([]lpxv1alpha1.LPUPipelineRequest, error) { - requests, available, err := r.listOwnedLPXRequestsIfAvailable(ctx, deployment) - if err == nil && !available { - return nil, fmt.Errorf("LPX request API is unavailable after startup discovery") + // Grove must not recreate pods while a retiring request still protects their names. + for _, request := range desired.requests { + if live := currents[request.requestName]; live != nil && !live.DeletionTimestamp.IsZero() { + return currents, &lpxRetiring{retirementReason: "Waiting for the current LPX request to finish cleanup"}, nil + } } - return requests, err + return currents, nil, nil } -func (r *graphReconciler) listOwnedLPXRequestsIfAvailable( +// listLPXRequestCandidates uses the propagated child identity only +// to narrow the authoritative read. Owner references and recorded UIDs remain +// the authority for adopting or deleting any returned request. +func (r *graphReconciler) listLPXRequestCandidates( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) ([]lpxv1alpha1.LPUPipelineRequest, bool, error) { +) (*lpxv1alpha1.LPUPipelineRequestList, error) { list := &lpxv1alpha1.LPUPipelineRequestList{} - owned := make([]lpxv1alpha1.LPUPipelineRequest, 0) - visit := func(object k8sruntime.Object) error { - request := object.(*lpxv1alpha1.LPUPipelineRequest) - if metav1.IsControlledBy(request, deployment) { - owned = append(owned, *request) - } - return nil - } - err := visitLifecycleObjectPages( - ctx, - r.apiReader, - list, - visit, + err := r.apiReader.List(ctx, list, client.InNamespace(deployment.Namespace), + client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)}, ) - if err != nil { - if meta.IsNoMatchError(err) || k8sruntime.IsNotRegisteredError(err) || apierrors.IsNotFound(err) { - return nil, false, nil - } - return nil, false, err + return list, err +} + +// ownsLPXRequest follows the request's exact controller chain, including a PCS +// whose name predates the current source input. +func (r *graphReconciler) ownsLPXRequest(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest) (bool, error) { + owner := metav1.GetControllerOf(request) + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != "PodCliqueSet" { + return false, nil } - sort.Slice(owned, func(i, j int) bool { return owned[i].Name < owned[j].Name }) - return owned, true, nil + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: owner.Name}, pcs); err == nil { + return metav1.IsControlledBy(pcs, deployment) && metav1.IsControlledBy(request, pcs), nil + } else if !apierrors.IsNotFound(err) { + return false, err + } + + // Every labeled request for the recorded PCS remains retireable after garbage + // collection removes that owner, including requests that had already left the + // active deadline batch. + attempt := currentLPXAttemptStatus(deployment) + return attempt != nil && owner.UID == attempt.PodCliqueSetUID, nil } -// retireFirstOwnedLPXRequest uses the sorted authoritative first request as the retirement witness. -func (r *graphReconciler) retireFirstOwnedLPXRequest( +func (r *graphReconciler) listOwnedLPXRequests( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, - reason string, -) (*lpxRetiring, error) { - requests, err := r.listOwnedLPXRequests(ctx, deployment) - if err != nil || len(requests) == 0 { +) ([]lpxv1alpha1.LPUPipelineRequest, error) { + requests, err := r.listLPXRequestCandidates(ctx, deployment) + if err != nil { return nil, err } - return r.retireLPXRequest(ctx, deployment, pcsName, &requests[0], reason) + owned := make([]lpxv1alpha1.LPUPipelineRequest, 0) + for index := range requests.Items { + request := &requests.Items[index] + ownedByChild, err := r.ownsLPXRequest(ctx, deployment, request) + if err != nil { + return nil, err + } + if ownedByChild { + owned = append(owned, *request) + } + } + sort.Slice(owned, func(i, j int) bool { return owned[i].Name < owned[j].Name }) + return owned, nil } -func (r *graphReconciler) retireLPXRequest( +// deleteLPXPodCliqueSet starts background cascading deletion of an exact PCS. +// A same-name replacement proves that the old UID is already absent and is +// never adopted or deleted. +func (r *graphReconciler) deleteLPXPodCliqueSet( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, pcsName string, - request *lpxv1alpha1.LPUPipelineRequest, - reason string, -) (*lpxRetiring, error) { - if request == nil || !metav1.IsControlledBy(request, deployment) { - return nil, fmt.Errorf("refusing to retire an LPX request without the exact LPXGraphDeployment controller owner") + pcsUID types.UID, +) (bool, error) { + pcs := &grovev1alpha1.PodCliqueSet{} + if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err } - if err := r.releaseLPXAttemptRecordingFinalizer(ctx, request); err != nil { - return nil, err + if pcs.UID != pcsUID { + return false, nil } - - // Read the complete publication set directly from the API before changing shared Grove state. - type requestDeletePrecondition struct { - name string - uid types.UID - resourceVersion string - } - owned := make([]requestDeletePrecondition, 0) - if err := visitLifecycleObjectPages( - ctx, - r.apiReader, - &lpxv1alpha1.LPUPipelineRequestList{}, - func(object k8sruntime.Object) error { - live := object.(*lpxv1alpha1.LPUPipelineRequest) - if metav1.IsControlledBy(live, deployment) && live.DeletionTimestamp.IsZero() { - owned = append(owned, requestDeletePrecondition{ - name: live.Name, - uid: live.UID, - resourceVersion: live.ResourceVersion, - }) - } - return nil - }, - client.InNamespace(deployment.Namespace), - ); err != nil { - return nil, fmt.Errorf("list the exact LPXGraphDeployment-owned LPX requests for attempt-wide retirement: %w", err) + if !metav1.IsControlledBy(pcs, deployment) { + return false, fmt.Errorf("refusing to delete PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) } - - // Delete every live publication with exact-object preconditions before scaling the shared PCS. - for _, current := range owned { - live := &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{ - Namespace: deployment.Namespace, - Name: current.name, - UID: current.uid, - ResourceVersion: current.resourceVersion, - }} - if err := r.Delete(ctx, live, &client.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: ¤t.uid, ResourceVersion: ¤t.resourceVersion}, - }); err != nil && !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("delete exact LPXGraphDeployment-owned LPX request %q: %w", current.name, err) + if !pcs.DeletionTimestamp.IsZero() { + return true, nil + } + if deployment.DeletionTimestamp.IsZero() { + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return false, err } } - - // Hold the shared Grove attempt at zero only after every publication is retiring. - if err := r.scaleDownLPXPodCliqueSet(ctx, deployment, pcsName, ""); err != nil { - return nil, fmt.Errorf("scale down the LPX PodCliqueSet after retiring all attempt requests: %w", err) + propagation := metav1.DeletePropagationBackground + if err := r.Delete(ctx, pcs, &client.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, + PropagationPolicy: &propagation, + }); err != nil && !apierrors.IsNotFound(err) { + return false, err } - return &lpxRetiring{ - retirementReason: reason, - }, nil + return true, nil } -// scaleDownLPXPodCliqueSet holds the complete Grove attempt at zero while LPX -// releases its deleted requests. LPX requires every exact old workload Pod to be -// absent and treats a same-name replacement UID as a cleanup blocker. -// Grove supports mutable PodCliqueSet scale-to-zero and deletes every child -// replica without changing the immutable clique composition. -func (r *graphReconciler) scaleDownLPXPodCliqueSet( +// deleteStaleLPXRequests removes scaled-in scheduler intent without disturbing +// the shared PCS. Grove and the LPX scheduler converge their pods independently. +func (r *graphReconciler) deleteStaleLPXRequests( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, - pcsUID types.UID, + requests []*lpxv1alpha1.LPUPipelineRequest, ) error { - scale := func(pcs *grovev1alpha1.PodCliqueSet) error { - if pcsUID != "" && pcs.UID != pcsUID { - return nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return fmt.Errorf("refusing to scale down PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) - } - if !pcs.DeletionTimestamp.IsZero() || pcs.Spec.Replicas == 0 { - return nil + if deployment.DeletionTimestamp.IsZero() { + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return err } - pcs.Spec.Replicas = 0 - return r.Update(ctx, pcs) } - if pcsName != "" { - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { - return client.IgnoreNotFound(err) + for _, request := range requests { + if !request.DeletionTimestamp.IsZero() { + continue } - return scale(pcs) - } - return visitLifecycleObjectPages(ctx, r.apiReader, &grovev1alpha1.PodCliqueSetList{}, func(object k8sruntime.Object) error { - pcs := object.(*grovev1alpha1.PodCliqueSet) - if !metav1.IsControlledBy(pcs, deployment) { - return nil + if request.UID == "" { + return fmt.Errorf("refusing to delete LPX request %q without an immutable UID", request.Name) } - return scale(pcs) - }, client.InNamespace(deployment.Namespace)) -} - -func (r *graphReconciler) reconcileSelectedLPX( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, -) (lpxClassification, error) { - // Retiring a removed engine must not block publication for surviving engines. - currents, retiring, err := r.reconcileLPXAttemptFence(ctx, deployment, desired) - if err != nil { - return nil, err - } - if _, recording := retiring.(*lpxDeadlineTransition); recording { - return retiring, nil - } - classification, err := r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents, true) - if retiring != nil && err == nil { - if _, recording := classification.(*lpxDeadlineTransition); !recording { - return retiring, nil + uid, resourceVersion := request.UID, request.ResourceVersion + if err := r.Delete(ctx, request, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ + UID: &uid, ResourceVersion: &resourceVersion, + }}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete stale LPX request %q: %w", request.Name, err) } } - return classification, err + return nil } func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( @@ -779,13 +684,12 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, currents map[string]*lpxv1alpha1.LPUPipelineRequest, - allowPublication bool, ) (lpxClassification, error) { if currents == nil { currents = make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) } - groveIdentities, pending, classification, err := r.reconcileSelectedLPXGroveIdentityPrefix( + pcs, classification, err := r.reconcileSelectedLPXPodCliqueSet( ctx, deployment, desired, @@ -795,47 +699,29 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return classification, err } - // Validate staged identities without crossing the scheduler-publication boundary. - if !allowPublication { - if closed := classifyLPXPublicationReadiness(desired.requests, currents, groveIdentities, pending); closed != nil { - return closed, nil - } - } - - // Prepare the batch from any ready engine, independently of replica zero's rollout. - var attemptProjection *nvidiacomv1beta1.LPXAttemptStatus - trackAttempt := desired.deadlineSeconds != nil || currentLPXAttemptStatus(deployment) != nil - for _, baseGroveIdentity := range groveIdentities { - if baseGroveIdentity == nil || !baseGroveIdentity.complete { - continue - } - if trackAttempt { - attemptProjection = desiredLPXAttemptStatus(desired, deployment.Generation, baseGroveIdentity.pcsUID) - } - transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) - if transition != nil { - return transition, nil - } - break + // Record the immutable PCS and request identities before publication. The + // attempt status is the durable ownership fence used during GC-driven cleanup; + // a scheduling deadline only adds a clock to that record. + attemptProjection := desiredLPXAttemptStatus(desired, deployment.Generation, pcs.UID) + if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents); transition != nil { + return transition, nil } for index := range desired.requests { request := &desired.requests[index] - groveIdentity := groveIdentities[request.replicaIndex] - if currents[request.requestName] != nil || groveIdentity == nil || !groveIdentity.complete { + if currents[request.requestName] != nil { continue } - if attemptProjection != nil { - // Classify known authority loss before attempting a replacement publication. - attempt := currentLPXAttemptStatus(deployment) - if _, _, missing := exactLPXAttemptState(attempt, currents); missing { - return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStateFailed, - lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring its scheduling batch without replacement", 0), nil - } - if err := r.revalidateLPXAttemptPublication(ctx, deployment, desired.plan.PodCliqueSetName); err != nil { - return nil, err - } + // Classify known authority loss before attempting a replacement publication. + attempt := currentLPXAttemptStatus(deployment) + if _, _, missing := exactLPXAttemptState(attempt, currents); missing { + return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStateFailed, + lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring its scheduling batch without replacement", 0), nil } - live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity, trackAttempt) + if err := r.revalidateLPXAttemptPublication(ctx, deployment, desired.plan.PodCliqueSetName); err != nil { + return nil, err + } + groveIdentity := &lpxGroveIdentity{pcsUID: pcs.UID, pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} + live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity) if createErr != nil { return nil, createErr } @@ -845,32 +731,14 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return transition, nil } } - return classifyLPXCurrentRequests(desired.requests, currents, pending), nil -} -// classifyLPXPublicationReadiness requires identities to cover every request's replica index. -// Identity entries, current requests and pending state may be nil. -func classifyLPXPublicationReadiness( - requests []lpxModelMaterializing, - currents map[string]*lpxv1alpha1.LPUPipelineRequest, - identities []*lpxGroveIdentity, - pending *lpxClosed, -) *lpxClosed { - // Complete identities with no request must wait for the Grove synchronization boundary. - for _, request := range requests { - identity := identities[request.replicaIndex] - if identity != nil && identity.complete && currents[request.requestName] == nil { - return &lpxClosed{incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization"} - } - } - return pending + return classifyLPXCurrentRequests(desired.requests, currents), nil } -// classifyLPXCurrentRequests accepts absent requests and a nil pending state. +// classifyLPXCurrentRequests accepts absent requests. func classifyLPXCurrentRequests( requests []lpxModelMaterializing, currents map[string]*lpxv1alpha1.LPUPipelineRequest, - pending *lpxClosed, ) lpxClassification { // Stream terminal, side-effect-free classifications in request order. var firstNonBound lpxClassification @@ -886,166 +754,66 @@ func classifyLPXCurrentRequests( } } - // Keep pending behind every published non-bound request. if firstNonBound != nil { return firstNonBound } - if pending != nil { - return pending - } return &lpxBound{} } -func (r *graphReconciler) reconcileSelectedLPXGroveIdentityPrefix( +func (r *graphReconciler) reconcileSelectedLPXPodCliqueSet( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) ([]*lpxGroveIdentity, *lpxClosed, lpxClassification, error) { - snapshot, incomplete, err := r.observeLPXGroveSnapshot(ctx, deployment, desired, len(currents) == 0) +) (*grovev1alpha1.PodCliqueSet, lpxClassification, error) { + pcs, incomplete, err := r.observeLPXPodCliqueSet(ctx, deployment, desired, len(currents) == 0) if err != nil { - return nil, nil, nil, err - } - if snapshot == nil { - return nil, nil, &lpxClosed{incomplete: incomplete}, nil + return nil, nil, err } - groveIdentities := make([]*lpxGroveIdentity, desired.plan.Replicas) - replicaPlans := make([]*lpx.MaterializationPlan, desired.plan.Replicas) - allAgentGroups := make(map[string]struct{}, int(desired.plan.Replicas)*len(desired.plan.Agents)) - for replicaIndex := int32(0); replicaIndex < desired.plan.Replicas; replicaIndex++ { - plan := desired.plan.ForReplica(replicaIndex) - replicaPlans[replicaIndex] = plan - for _, agent := range plan.Agents { - allAgentGroups[agent.CliqueName] = struct{}{} + if pcs == nil || incomplete != "" { + if len(currents) == 0 { + return nil, &lpxClosed{incomplete: incomplete}, nil } - } - var pending *lpxClosed - retire := make(map[int32]bool) - intentional := make(map[int32]bool) - for replicaIndex, plan := range replicaPlans { - identity, incomplete, drift, err := r.observeLPXGroveIdentity(ctx, deployment, desired, snapshot, plan, allAgentGroups) - if err != nil { - if errors.Is(err, errLPXEngineReplacing) { - intentional[int32(replicaIndex)] = true - incomplete = err.Error() - } else { - return nil, nil, nil, err + for _, request := range desired.requests { + if live := currents[request.requestName]; live != nil { + retiring, err := r.retireLPXEngineRequest(ctx, deployment, live, fmt.Sprintf("The published PCS identity became unavailable: %q", incomplete), false) + return nil, retiring, err } - retire[int32(replicaIndex)] = true - } - if drift != nil { - incomplete = drift.retirementReason - retire[int32(replicaIndex)] = true - } - if incomplete != "" { - if pending == nil { - pending = &lpxClosed{incomplete: incomplete} - } - } - if identity != nil { - identity.complete = incomplete == "" - groveIdentities[replicaIndex] = identity } + return nil, &lpxClosed{incomplete: incomplete}, nil } - // Observe retirement per engine; an incomplete sibling never fences this engine. + // The PCS owns each request; LPX resolves the replica-local clique references itself. for index := range desired.requests { request := &desired.requests[index] - live := currents[request.requestName] - identity := groveIdentities[request.replicaIndex] - digest, intentChanged, err := resolveLPXCurrentRequestIntent(deployment, *request, identity, live) + identity := &lpxGroveIdentity{pcsUID: pcs.UID, pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} + spec, err := json.Marshal(request.modelProjection.RequestSpec(identity.plan, request.agentCliqueName)) if err != nil { - return nil, nil, nil, err - } - request.attemptDigest = digest - if live == nil { - continue - } - if intentChanged { - intentional[request.replicaIndex] = true - } - if intentional[request.replicaIndex] || !live.DeletionTimestamp.IsZero() || lpx.ValidateRequestSize(live) != nil { - retire[request.replicaIndex] = true - } - } - - // All model requests of a replacing engine must finish before any is republished. - for index := range desired.requests { - request := &desired.requests[index] - if !retire[request.replicaIndex] { - continue + return nil, nil, err } - request.attemptDigest = "" - groveIdentities[request.replicaIndex] = nil - pending = &lpxClosed{incomplete: "Waiting for the replaced engine's scheduler requests to finish cleanup"} + request.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) if live := currents[request.requestName]; live != nil { - classification, err := r.retireLPXEngineRequest(ctx, deployment, live, pending.incomplete, intentional[request.replicaIndex]) - if transition, recording := classification.(*lpxDeadlineTransition); recording { - if _, completed, _ := exactLPXAttemptState(currentLPXAttemptStatus(deployment), currents); completed { - disarmedAt := metav1.Now() - transition.attempt.DisarmedAt = &disarmedAt - } - return nil, nil, classification, err + if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, live, err.Error(), true) + return nil, retiring, retireErr } - if err != nil { - return nil, nil, classification, err - } - delete(currents, request.requestName) - } - } - return groveIdentities, pending, nil, nil -} - -// resolveLPXCurrentRequestIntent does not mutate its inputs. deployment must be non-nil; -// identity and live may be nil while their observations are pending. -func resolveLPXCurrentRequestIntent( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired lpxModelMaterializing, - identity *lpxGroveIdentity, - live *lpxv1alpha1.LPUPipelineRequest, -) (string, bool, error) { - // Preserve the observed digest until Grove supplies a complete replacement identity. - if identity == nil || !identity.complete { - if live != nil { - desired.attemptDigest = live.Annotations[lpxAttemptDigestAnnotation] } - return desired.attemptDigest, false, nil } - - // Compare immutable intent using the digest derived from the complete Grove identity. - spec, err := json.Marshal(struct { - Spec lpxv1alpha1.LPUPipelineRequestSpec - AgentUIDs []types.UID - }{desired.modelProjection.RequestSpec(deployment.Namespace, identity.podGangName, identity.cyborgClique), identity.agentUIDs}) - if err != nil { - return "", false, err - } - desired.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) - return desired.attemptDigest, live != nil && validateCurrentLPXRequest(deployment, &desired, identity, live) != nil, nil + return pcs, nil, nil } // classifyPublishedLPX classifies a non-nil published request from its current scheduler receipt. func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassification { - // Expose only scheduler status observed for the current request generation. - status := current.Status - if current.Generation <= 0 || status == nil || status.ObservedGeneration == nil || *status.ObservedGeneration != current.Generation { + if current == nil || current.Status == nil || current.Status.ObservedGeneration == nil || + *current.Status.ObservedGeneration != current.Generation { return &lpxOpen{} } - - // Bound requires one complete committed receipt for this exact request and plan. - committed := status.Committed - if status.Phase == lpxv1alpha1.RequestPhaseBound && committed != nil && - committed.Execution.AcceptedGeneration != nil && *committed.Execution.AcceptedGeneration == current.Generation && - committed.Plan.PlannedFromGeneration == current.Generation && - committed.Plan.Revision > 0 && committed.Plan.Revision == status.LastPlanRevision && - committed.Plan.PlanDigest != "" && committed.Plan.Placement.ExecutionBackend == current.Spec.ExecutionBackend && - current.Spec.ExecutionBackend == lpxv1alpha1.ExecutionBackendNodeLocal && - committed.Plan.Placement.NodeLocal != nil && committed.Plan.Placement.RemoteDra == nil && - committed.Execution.NodeLocal != nil && committed.Execution.RemoteDra == nil { + if current.Status.Phase == lpxv1alpha1.RequestPhaseBound { return &lpxBound{} } - return (*lpxSchedulerObserved)(status) + observed := lpxSchedulerObserved(*current.Status.DeepCopy()) + return &observed } func (r *graphReconciler) createPublishedLPXRequest( @@ -1053,12 +821,11 @@ func (r *graphReconciler) createPublishedLPXRequest( deployment *nvidiacomv1alpha1.LPXGraphDeployment, materializing *lpxModelMaterializing, grove *lpxGroveIdentity, - recordAttempt bool, ) (*lpxv1alpha1.LPUPipelineRequest, error) { if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { return nil, err } - spec := materializing.modelProjection.RequestSpec(deployment.Namespace, grove.podGangName, grove.cyborgClique) + spec := materializing.modelProjection.RequestSpec(grove.plan, materializing.agentCliqueName) request := &lpxv1alpha1.LPUPipelineRequest{ TypeMeta: metav1.TypeMeta{ APIVersion: lpxv1alpha1.GroupVersion.String(), @@ -1069,20 +836,15 @@ func (r *graphReconciler) createPublishedLPXRequest( Namespace: deployment.Namespace, Labels: map[string]string{ consts.KubeLabelDynamoGraphDeploymentName: metav1.GetControllerOf(deployment).Name, + lpxOwnerUIDLabel: string(deployment.UID), }, Annotations: lpxRequestAnnotations(deployment, materializing, grove), }, Spec: spec, } - if recordAttempt { - controllerutil.AddFinalizer(request, lpxAttemptRecordingFinalizer) - } - if err := controllerutil.SetControllerReference(deployment, request, r.Client.Scheme()); err != nil { + if err := controllerutil.SetControllerReference(grove.pcs, request, r.Client.Scheme()); err != nil { return nil, err } - if err := lpx.ValidateRequestSize(request); err != nil { - return nil, fmt.Errorf("constructed LPX request exceeds the scheduler budget: %w", err) - } if err := r.Create(ctx, request); err != nil { return nil, fmt.Errorf( "create LPX request %q for model %q: %w", @@ -1102,13 +864,14 @@ func lpxRequestAnnotations( // Callers validate the exact source controller owner before constructing requests. sourceOwner := metav1.GetControllerOf(deployment) return map[string]string{ - lpxAttemptDigestAnnotation: materializing.attemptDigest, - lpx.DGDUIDAnnotation: string(sourceOwner.UID), - lpxDeploymentUIDAnnotation: string(deployment.UID), - lpxModelAnnotation: materializing.modelProjection.Model(), - lpxPCSUIDAnnotation: string(grove.pcsUID), - lpxPodGangUIDAnnotation: string(grove.podGangUID), - lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), + lpxAttemptDigestAnnotation: materializing.attemptDigest, + lpx.DGDUIDAnnotation: string(sourceOwner.UID), + lpx.DeploymentNameAnnotation: deployment.Name, + lpxDeploymentUIDAnnotation: string(deployment.UID), + lpxModelAnnotation: materializing.modelProjection.Model(), + lpxPCSUIDAnnotation: string(grove.pcsUID), + lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), + lpxv1alpha1.CompilerSnapshotDigestAnnotation: materializing.modelProjection.CompilerSnapshotDigest(), } } @@ -1118,6 +881,9 @@ func validateCurrentLPXRequest( grove *lpxGroveIdentity, request *lpxv1alpha1.LPUPipelineRequest, ) error { + if !metav1.IsControlledBy(request, grove.pcs) { + return fmt.Errorf("LPX request %q does not have the exact PCS owner", request.Name) + } if request.UID == "" { return fmt.Errorf("LPX request %q has no immutable UID", request.Name) } @@ -1127,170 +893,13 @@ func validateCurrentLPXRequest( return fmt.Errorf("LPX request %q immutable annotation %q no longer matches the current attempt", request.Name, key) } } - wantSpec := desired.modelProjection.RequestSpec(deployment.Namespace, grove.podGangName, grove.cyborgClique) + wantSpec := desired.modelProjection.RequestSpec(grove.plan, desired.agentCliqueName) if !apiequality.Semantic.DeepEqual(request.Spec, wantSpec) { return fmt.Errorf("LPX request %q immutable intent no longer matches the current attempt", request.Name) } return nil } -func (r *graphReconciler) observeLPXGroveSnapshot( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - noCurrentRequests bool, -) (*lpxGroveSnapshot, string, error) { - pcs, incomplete, err := r.observeLPXPodCliqueSet(ctx, deployment, desired, noCurrentRequests) - if err != nil || pcs == nil { - return nil, incomplete, err - } - generationHash, wait := observedLPXPCSGenerationHash(pcs) - if wait != "" { - if incomplete == "" { - incomplete = wait - } - } - podGangs, err := r.listLPXPublicationPodGangs(ctx, deployment, pcs) - if err != nil { - return nil, "", err - } - - // Every resolved runtime has one serving clique in this scaling group. - scalingGroupName := desired.plan.LPXScalingGroup - scalingGroup := &grovev1alpha1.PodCliqueScalingGroup{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: scalingGroupName} - if err := r.apiReader.Get(ctx, key, scalingGroup); err != nil { - if apierrors.IsNotFound(err) { - return nil, fmt.Sprintf("Waiting for current LPU PodCliqueScalingGroup %q", scalingGroupName), nil - } - return nil, "", err - } - if scalingGroup.UID == "" || !scalingGroup.DeletionTimestamp.IsZero() || - !metav1.IsControlledBy(scalingGroup, pcs) { - return nil, fmt.Sprintf( - "LPU PodCliqueScalingGroup %q lacks the current PodCliqueSet owner identity", - scalingGroupName, - ), nil - } - return &lpxGroveSnapshot{ - podCliqueSet: pcs, - scalingGroup: scalingGroup, - generationHash: generationHash, - incomplete: incomplete, - podGangs: podGangs, - }, "", nil -} - -func (r *graphReconciler) observeLPXGroveIdentity( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - snapshot *lpxGroveSnapshot, - plan *lpx.MaterializationPlan, - allAgentGroups map[string]struct{}, -) (*lpxGroveIdentity, string, *lpxRetiring, error) { - incomplete := snapshot.incomplete - // Observe native retirement before gang availability or successor revision readiness. - agentUIDs := make([]types.UID, 0, len(plan.Agents)) - projections := desired.workload.ModelProjections() - for index, agent := range plan.Agents { - clique := &grovev1alpha1.PodClique{} - if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: agent.CliqueName}, clique); err != nil { - if apierrors.IsNotFound(err) { - incomplete = fmt.Sprintf("Waiting for Agent PodClique %q", agent.CliqueName) - continue - } - return nil, "", nil, err - } - if !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup) || clique.UID == "" { - incomplete = fmt.Sprintf("Waiting for owned Agent PodClique %q", agent.CliqueName) - continue - } - if !clique.DeletionTimestamp.IsZero() { - return nil, "", nil, errLPXEngineReplacing - } - agentUIDs = append(agentUIDs, clique.UID) - if !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || - clique.Annotations[lpx.WorkloadDigestAnnotation] != projections[index].Digest().String() || - clique.Status.CurrentPodCliqueSetGenerationHash == nil || *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash { - incomplete = fmt.Sprintf("Waiting for current Agent PodClique %q", agent.CliqueName) - } - } - lpxPodGang, ordinaryPodGang, wait := selectLPXPublicationPodGangs( - deployment.Namespace, - snapshot.podGangs, - plan, - allAgentGroups, - ) - if wait != "" { - return nil, wait, nil, nil - } - - identity := &lpxGroveIdentity{ - pcsUID: snapshot.podCliqueSet.UID, - podGangName: lpxPodGang.Name, - podGangUID: lpxPodGang.UID, - agentUIDs: agentUIDs, - } - if !desired.hasCurrentLPXAttemptAnnotations(lpxPodGang, deployment) || - !desired.hasCurrentLPXAttemptAnnotations(ordinaryPodGang, deployment) { - if incomplete == "" { - incomplete = "Waiting for Grove to publish current-generation PodGangs" - } - } - - // A conductor needs complete identity; Cyborg can retain its reference while hashes converge. - cyborg, wait, retiring, err := r.observeLPXServingClique( - ctx, - deployment, - snapshot, - ordinaryPodGang.Name, - desired, - plan, - ) - if err != nil || retiring != nil || (wait != "" && cyborg == nil) { - return nil, wait, retiring, err - } - identity.cyborgClique = cyborg - if wait != "" { - if incomplete == "" { - incomplete = wait - } - } - return identity, incomplete, nil, nil -} - -func (r *graphReconciler) listLPXPublicationPodGangs( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, -) ([]groveschedulerv1alpha1.PodGang, error) { - podGangs := make([]groveschedulerv1alpha1.PodGang, 0, 2) - if err := visitLifecycleObjectPages( - ctx, - r.apiReader, - &groveschedulerv1alpha1.PodGangList{}, - func(object k8sruntime.Object) error { - podGang := object.(*groveschedulerv1alpha1.PodGang) - schedulerName := podGang.Labels[grovecommon.LabelSchedulerName] - if (schedulerName != lpx.SchedulerName && schedulerName != corev1.DefaultSchedulerName) || - podGang.UID == "" || !podGang.DeletionTimestamp.IsZero() || - !metav1.IsControlledBy(podGang, pcs) { - return nil - } - podGangs = append(podGangs, *podGang) - return nil - }, - client.InNamespace(deployment.Namespace), - client.MatchingLabels{ - grovecommon.LabelPartOfKey: pcs.Name, - }, - ); err != nil { - return nil, err - } - return podGangs, nil -} - func (r *graphReconciler) observeLPXPodCliqueSet( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -1308,9 +917,6 @@ func (r *graphReconciler) observeLPXPodCliqueSet( if noCurrentRequests && !metav1.IsControlledBy(pcs, deployment) { return nil, "", fmt.Errorf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) } - if noCurrentRequests && pcs.Spec.Replicas == 0 { - return nil, "The retired Grove attempt is held at zero; successor publication waits for post-retirement Grove synchronization", nil - } if pcs.UID == "" || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) { return nil, "The selected Grove PodCliqueSet lacks the exact LPXGraphDeployment owner identity", nil } @@ -1326,279 +932,3 @@ func (r *graphReconciler) observeLPXPodCliqueSet( } return pcs, incomplete, nil } - -func observedLPXPCSGenerationHash(pcs *grovev1alpha1.PodCliqueSet) (string, string) { - if pcs.Status.CurrentGenerationHash == nil || *pcs.Status.CurrentGenerationHash == "" { - return "", fmt.Sprintf( - "Waiting for Grove to publish PodCliqueSet %q generation identity", - pcs.Name, - ) - } - return *pcs.Status.CurrentGenerationHash, "" -} - -func hasCurrentLPXScalingGroupOwner( - clique *grovev1alpha1.PodClique, - scalingGroup *grovev1alpha1.PodCliqueScalingGroup, -) bool { - owner := metav1.GetControllerOf(clique) - return owner != nil && owner.APIVersion == grovev1alpha1.SchemeGroupVersion.String() && - owner.Kind == "PodCliqueScalingGroup" && owner.Name == scalingGroup.Name && - owner.UID == scalingGroup.UID -} - -func (r *graphReconciler) observeLPXServingClique( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - snapshot *lpxGroveSnapshot, - ordinaryPodGangName string, - desired *lpxMaterializing, - plan *lpx.MaterializationPlan, -) (*lpxv1alpha1.CyborgPodCliqueReference, string, *lpxRetiring, error) { - // Every supported plan has exactly one ordinary-scheduler serving clique. - isCyborg := plan.CyborgClique != "" - name, role := plan.ConductorClique, lpxv1alpha1.PodRoleConductor - roleName, waitingRoleName := "Conductor", "conductor" - if isCyborg { - name, role = plan.CyborgClique, lpxv1alpha1.PodRoleCyborgWorker - roleName, waitingRoleName = "Cyborg", "Cyborg" - } - - // Read the single serving witness and validate its shared scheduler identity. - clique := &grovev1alpha1.PodClique{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: name} - if err := r.apiReader.Get(ctx, key, clique); err != nil { - if apierrors.IsNotFound(err) { - return nil, fmt.Sprintf("Waiting for %s PodClique %q", waitingRoleName, name), nil, nil - } - return nil, "", nil, err - } - if clique.UID == "" || - clique.Labels[grovecommon.LabelPartOfKey] != snapshot.podCliqueSet.Name || - clique.Labels[grovecommon.LabelPodCliqueScalingGroup] != plan.LPXScalingGroup || - clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] != strconv.FormatInt(int64(plan.ReplicaIndex), 10) || - clique.Labels[grovecommon.LabelPodGang] != ordinaryPodGangName || - clique.Spec.PodSpec.SchedulerName != corev1.DefaultSchedulerName || - clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != role || - !desired.hasCurrentLPXAttemptAnnotations(clique, deployment) || - (!isCyborg && !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup)) { - return nil, fmt.Sprintf( - "%s PodClique %q does not match the current ordinary-scheduler identity", - roleName, name, - ), nil, nil - } - if !isCyborg { - if clique.Status.CurrentPodCliqueSetGenerationHash == nil || *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash { - return nil, fmt.Sprintf("Waiting for current conductor PodClique %q", clique.Name), nil, nil - } - return nil, "", nil, nil - } - - // Cyborg publication additionally binds device claims and exact Grove revision hashes. - if !hasCurrentLPXScalingGroupOwner(clique, snapshot.scalingGroup) { - return nil, fmt.Sprintf( - "Cyborg PodClique %q lacks the current LPU scaling-group owner identity", - clique.Name, - ), nil, nil - } - if !hasLPXCyborgDeviceIntent(clique.Spec.PodSpec) || !hasExpectedLPXCyborgClaimReferences(snapshot.podCliqueSet, clique, plan.CyborgTemplate) { - reason := fmt.Sprintf("Cyborg PodClique %q GPU intent or DRA Claim references changed during Grove materialization", clique.Name) - // Until Grove has accepted the PCS edit, differing references may still be its old engine. - pcs := snapshot.podCliqueSet - if pcs.Status.ObservedGeneration != nil && *pcs.Status.ObservedGeneration == pcs.Generation && - clique.Status.CurrentPodCliqueSetGenerationHash != nil && *clique.Status.CurrentPodCliqueSetGenerationHash == snapshot.generationHash { - return nil, "", &lpxRetiring{retirementReason: reason}, nil - } - return nil, reason, nil, nil - } - reference := &lpxv1alpha1.CyborgPodCliqueReference{Name: clique.Name, UID: string(clique.UID)} - - // Require Grove's selected PCS revision and pod-template identities. Deliberately - // do not require Cyborg ObservedGeneration: LPX must remain able to publish the - // provision request before Grove advances that compatibility signal. - if clique.Status.CurrentPodCliqueSetGenerationHash == nil || - *clique.Status.CurrentPodCliqueSetGenerationHash != snapshot.generationHash || - clique.Status.CurrentPodTemplateHash == nil || - *clique.Status.CurrentPodTemplateHash == "" { - return reference, fmt.Sprintf( - "Waiting for Grove to materialize the current PodCliqueSet generation in Cyborg PodClique %q", - clique.Name, - ), nil, nil - } - return reference, "", nil, nil -} - -func selectLPXPublicationPodGangs( - namespace string, - podGangs []groveschedulerv1alpha1.PodGang, - plan *lpx.MaterializationPlan, - allAgentGroups map[string]struct{}, -) (*groveschedulerv1alpha1.PodGang, *groveschedulerv1alpha1.PodGang, string) { - var lpxCandidate, ordinaryCandidate *groveschedulerv1alpha1.PodGang - var lpxCandidateCount, ordinaryCandidateCount int - ordinaryGroup := plan.ConductorClique - if ordinaryGroup == "" { - ordinaryGroup = plan.CyborgClique - } - // Count every matching gang before reporting either scheduler's ambiguity. - for index := range podGangs { - podGang := &podGangs[index] - switch podGang.Labels[grovecommon.LabelSchedulerName] { - case lpx.SchedulerName: - if hasExactLPXAgentRows(namespace, podGang, plan) { - lpxCandidate = podGang - lpxCandidateCount++ - } - case corev1.DefaultSchedulerName: - containsAgent, containsOrdinaryGroup := false, ordinaryGroup == "" - for _, group := range podGang.Spec.PodGroups { - if _, agent := allAgentGroups[group.Name]; agent { - containsAgent = true - } - if group.Name == ordinaryGroup { - containsOrdinaryGroup = true - } - } - if !containsAgent && containsOrdinaryGroup { - ordinaryCandidate = podGang - ordinaryCandidateCount++ - } - } - } - if lpxCandidateCount != 1 { - return nil, nil, fmt.Sprintf( - "Waiting for one replica-%d LPX PodGang; found %d candidates", - plan.ReplicaIndex, - lpxCandidateCount, - ) - } - if ordinaryCandidateCount != 1 { - return nil, nil, fmt.Sprintf( - "Waiting for one replica-%d ordinary-scheduler PodGang; found %d candidates", - plan.ReplicaIndex, - ordinaryCandidateCount, - ) - } - return lpxCandidate, ordinaryCandidate, "" -} - -func hasExactLPXAgentRows( - namespace string, - podGang *groveschedulerv1alpha1.PodGang, - plan *lpx.MaterializationPlan, -) bool { - if len(podGang.Spec.PodGroups) != len(plan.Agents) { - return false - } - groups := make(map[string]*groveschedulerv1alpha1.PodGroup, len(podGang.Spec.PodGroups)) - for index := range podGang.Spec.PodGroups { - group := &podGang.Spec.PodGroups[index] - if _, duplicate := groups[group.Name]; duplicate { - return false - } - groups[group.Name] = group - } - for _, agent := range plan.Agents { - group, found := groups[agent.CliqueName] - if !found || group.MinReplicas != int32(agent.Replicas) || len(group.PodReferences) != agent.Replicas { - return false - } - seen := make(map[types.NamespacedName]struct{}, agent.Replicas) - for _, reference := range group.PodReferences { - key := types.NamespacedName{Namespace: reference.Namespace, Name: reference.Name} - if reference.Namespace != namespace || strings.TrimSpace(reference.Name) == "" { - return false - } - if _, duplicate := seen[key]; duplicate { - return false - } - seen[key] = struct{}{} - } - } - return true -} - -func hasExpectedLPXCyborgClaimReferences( - pcs *grovev1alpha1.PodCliqueSet, - cyborg *grovev1alpha1.PodClique, - templateName string, -) bool { - var expected *corev1.PodSpec - for _, template := range pcs.Spec.Template.Cliques { - if template == nil || template.Name != templateName { - continue - } - if expected != nil { - return false - } - expected = &template.Spec.PodSpec - } - if expected == nil { - return false - } - observed := &cyborg.Spec.PodSpec - return apiequality.Semantic.DeepEqual(podLevelClaimReferences(expected), podLevelClaimReferences(observed)) && - apiequality.Semantic.DeepEqual(expected.ResourceClaims, observed.ResourceClaims) && - containerClaimReferencesEqual(expected.Containers, observed.Containers) && - containerClaimReferencesEqual(expected.InitContainers, observed.InitContainers) -} - -func podLevelClaimReferences(spec *corev1.PodSpec) []corev1.ResourceClaim { - if spec.Resources == nil { - return nil - } - return spec.Resources.Claims -} - -func containerClaimReferencesEqual(expected, observed []corev1.Container) bool { - if len(expected) != len(observed) { - return false - } - for index := range expected { - if expected[index].Name != observed[index].Name || - !apiequality.Semantic.DeepEqual(expected[index].Resources.Claims, observed[index].Resources.Claims) { - return false - } - } - return true -} - -func hasLPXCyborgDeviceIntent(podSpec corev1.PodSpec) bool { - if len(podSpec.ResourceClaims) != 0 { - return true - } - for _, container := range podSpec.Containers { - if container.Name != consts.MainContainerName && container.Name != "cyborg" { - continue - } - gpuCount, err := lpx.EffectiveCyborgGPUCount(container.Resources) - if err != nil { - return false - } - if gpuCount > 0 { - return true - } - } - return false -} - -// finalizeLPXRequests retires every exact owned publication for the shared Grove -// attempt. A wait error keeps the DGD finalizer until LPX's own cleanup -// finalizers have completed. -func (r *graphReconciler) finalizeLPXRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) error { - requests, _, err := r.listOwnedLPXRequestsIfAvailable(ctx, deployment) - - // Stop when listing fails or no publication remains. - if err != nil || len(requests) == 0 { - return err - } - - // Start or continue retirement from the authoritative first request. - if _, err := r.retireLPXRequest(ctx, deployment, "", &requests[0], "LPXGraphDeployment is being deleted"); err != nil { - return err - } - return fmt.Errorf("waiting for LPX request %q to finish fail-closed retirement", requests[0].Name) -} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 917ff1b66eae..5db4da600ee3 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -33,7 +33,6 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/require" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" @@ -42,7 +41,6 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -61,7 +59,7 @@ func newLPXTestScheme(t testing.TB) *k8sruntime.Scheme { scheme := k8sruntime.NewScheme() for _, add := range []func(*k8sruntime.Scheme) error{ corev1.AddToScheme, resourcev1.AddToScheme, nvidiacomv1alpha1.AddToScheme, nvidiacomv1beta1.AddToScheme, - grovev1alpha1.AddToScheme, groveschedulerv1alpha1.AddToScheme, lpxv1alpha1.AddToScheme, + grovev1alpha1.AddToScheme, lpxv1alpha1.AddToScheme, } { require.NoError(t, add(scheme)) } @@ -69,12 +67,12 @@ func newLPXTestScheme(t testing.TB) *k8sruntime.Scheme { } type snapshotFailureRegistry struct { - *lpx.ModelRegistry + lpx.ModelRegistry err error } type downloadOrderedLPXRegistry struct { - *lpx.ModelRegistry + lpx.ModelRegistry buildURL url.URL downloaded bool calls []string @@ -124,7 +122,7 @@ func requirePreparedLPX( func newPreparedLPXTestReconciler( t *testing.T, - registry *lpx.ModelRegistry, + registry lpx.ModelRegistry, ctx context.Context, dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, @@ -160,13 +158,10 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { require.NotContains(t, group.Spec.CliqueNames, "") cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) require.NotContains(t, cyborg.Spec.StartsAfter, "") - ordinaryPodGang := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) - require.Len(t, ordinaryPodGang.Spec.PodGroups, 1) - require.Equal(t, desired.plan.CyborgClique, ordinaryPodGang.Spec.PodGroups[0].Name) + require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) createLPXTestObjects(t, ctx, reconciler.Client, objects...) - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.IsType(t, &lpxOpen{}, classification) request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) require.NotNil(t, request.Spec.CyborgPodCliqueRef) @@ -225,8 +220,7 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) createLPXTestObjects(t, ctx, reconciler.Client, objects...) - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.IsType(t, &lpxOpen{}, classification) requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) require.NoError(t, err) @@ -236,13 +230,15 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( for _, request := range requests { model := request.Annotations[lpxModelAnnotation] requestByModel[model] = request + require.Equal(t, string(dgd.UID), request.Labels[lpxOwnerUIDLabel]) require.Equal(t, model, request.Spec.NodeLocal.Model) - require.Equal(t, "selected-dgd-0-lpx-scheduler", request.Spec.PodGangRef.Name) + require.Equal(t, desired.plan.LPXScalingGroup, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name) } for _, projection := range desired.requests { request, found := requestByModel[projection.modelProjection.Model()] require.True(t, found) require.Equal(t, projection.modelProjection.Digest().String(), request.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, projection.modelProjection.CompilerSnapshotDigest(), request.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) } for index, expected := range desired.plan.Agents { @@ -251,6 +247,7 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( require.Equal(t, int32(expected.Replicas), clique.Spec.Replicas) require.Equal(t, ptr.To(int32(expected.Replicas)), clique.Spec.MinAvailable) require.Equal(t, projection.modelProjection.Digest().String(), clique.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, projection.modelProjection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) require.Equal(t, projection.modelProjection.Model(), clique.Annotations[lpxv1alpha1.PodModelAnnotation]) require.NotContains(t, clique.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) require.NotContains(t, clique.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) @@ -268,244 +265,14 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( }}, } require.NoError(t, reconciler.Update(ctx, target)) - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) require.Equal(t, "LPXUnsupported", lpxResult(classification).Reason) require.Contains(t, lpxResult(classification).Message, "TargetUnsupported") } -func TestLPXPublicationWitnessReadsOnlyNamedChildrenAndNoPods(t *testing.T) { - t.Log("Materialize two runtime replicas and preserve their distinct publication identities") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - source.Spec.Components[0].Replicas = ptr.To(int32(2)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - require.Equal(t, int32(2), desired.plan.Replicas) - require.Len(t, desired.requests, 2) - require.NotEqual(t, desired.requests[0].requestName, desired.requests[1].requestName) - for replicaIndex := int32(0); replicaIndex < 2; replicaIndex++ { - require.Equal(t, replicaIndex, desired.requests[replicaIndex].replicaIndex) - } - - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - require.Equal(t, int32(2), scalingGroup.Spec.Replicas) - - t.Log("Leave Agent status unobserved and drift generated-child replica counts") - replicaOneAgent := findLPXTestClique(t, objects, desired.plan.ForReplica(1).Agents[0].CliqueName) - replicaOneAgent.Status.ObservedGeneration = nil - replicaOneAgent.Status.Replicas = replicaOneAgent.Spec.Replicas - replicaOneAgent.Status.UpdatedReplicas = replicaOneAgent.Spec.Replicas - replicaOneAgent.Status.ScheduleGatedReplicas = replicaOneAgent.Spec.Replicas - replicaOneAgent.Status.ScheduledReplicas = 0 - replicaOneAgent.Status.Conditions = nil - scalingGroup.Spec.Replicas = 9 - agentClique := findLPXTestClique(t, objects, desired.plan.Agents[0].CliqueName) - agentClique.Spec.Replicas = 9 - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - t.Log("Allow only deterministic engine-clique and parent-PCSG reads; reject Pods and broad clique lists") - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - cachedClient, ok := reconciler.Client.(client.WithWatch) - require.True(t, ok) - forbiddenReads := make([]string, 0) - podGangLists := 0 - lifecycleRequestListCalls, dependencyListCalls := 0, 0 - scalingGroupGets := 0 - conductorGets := 0 - cyborgGets := 0 - agentGets := 0 - forbidden := interceptor.Funcs{ - Get: func( - ctx context.Context, - delegated client.WithWatch, - key client.ObjectKey, - object client.Object, - opts ...client.GetOption, - ) error { - switch object.(type) { - case *corev1.Pod: - forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) - case *grovev1alpha1.PodCliqueScalingGroup: - if key.Name != desired.plan.LPXScalingGroup { - forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) - } else { - scalingGroupGets++ - } - case *grovev1alpha1.PodClique: - known := false - for _, request := range desired.requests { - plan := desired.plan.ForReplica(request.replicaIndex) - for _, agent := range plan.Agents { - if key.Name == agent.CliqueName { - agentGets++ - known = true - } - } - if key.Name == plan.ConductorClique { - conductorGets++ - known = true - break - } - if key.Name == plan.CyborgClique { - cyborgGets++ - known = true - break - } - } - if !known { - forbiddenReads = append(forbiddenReads, fmt.Sprintf("get %T %s", object, key)) - } - } - return delegated.Get(ctx, key, object, opts...) - }, - List: func( - ctx context.Context, - delegated client.WithWatch, - list client.ObjectList, - opts ...client.ListOption, - ) error { - if _, request := list.(*lpxv1alpha1.LPUPipelineRequestList); request { - lifecycleRequestListCalls++ - } else { - dependencyListCalls++ - } - switch list.(type) { - case *groveschedulerv1alpha1.PodGangList: - podGangLists++ - case *corev1.PodList, *grovev1alpha1.PodCliqueList, - *grovev1alpha1.PodCliqueScalingGroupList: - forbiddenReads = append(forbiddenReads, fmt.Sprintf("list %T", list)) - } - return delegated.List(ctx, list, opts...) - }, - } - reconciler.apiReader = interceptor.NewClient(reader, forbidden) - reconciler.Client = interceptor.NewClient(cachedClient, forbidden) - - t.Log("Publish directly from the PCS and PodGang witness") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - require.Empty(t, forbiddenReads) - require.Equal(t, 1, podGangLists) - require.Equal(t, 1, scalingGroupGets) - require.Zero(t, conductorGets) - require.Equal(t, 2, cyborgGets) - require.Equal(t, 2, agentGets) - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) - require.NoError(t, err) - require.Len(t, requests, 2) - require.NotEqual(t, requests[0].Spec.PodGangRef.Name, requests[1].Spec.PodGangRef.Name) - require.NotNil(t, requests[0].Spec.CyborgPodCliqueRef) - require.NotNil(t, requests[1].Spec.CyborgPodCliqueRef) - require.NotEqual(t, requests[0].Spec.CyborgPodCliqueRef.Name, requests[1].Spec.CyborgPodCliqueRef.Name) - published := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - - t.Log("Repeat publication using the same witnesses and preserve the request") - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - require.Equal(t, published.UID, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName).UID) - require.Empty(t, forbiddenReads) - require.Equal(t, 2, scalingGroupGets) - require.Zero(t, conductorGets) - require.Equal(t, 4, cyborgGets) - require.Equal(t, 4, agentGets) - - t.Log("Leave unsupported generated-child replica mutations untouched") - require.NoError(t, cachedClient.Get(ctx, client.ObjectKeyFromObject(scalingGroup), scalingGroup)) - require.Equal(t, int32(9), scalingGroup.Spec.Replicas) - require.NoError(t, cachedClient.Get(ctx, client.ObjectKeyFromObject(agentClique), agentClique)) - require.Equal(t, int32(9), agentClique.Spec.Replicas) - - t.Log("Retire the complete runtime while scheduler finalizers hold both requests") - for index := range requests { - requests[index].Finalizers = []string{"scheduling.lpu.nvidia.com/test-cleanup"} - require.NoError(t, reconciler.Update(ctx, &requests[index])) - } - _, err = reconciler.retireLPXRequest(ctx, dgd, desired.plan.PodCliqueSetName, &requests[0], "test group retirement") - require.NoError(t, err) - requests, err = reconciler.listOwnedLPXRequests(ctx, dgd) - require.NoError(t, err) - require.Len(t, requests, 2) - for index := range requests { - require.False(t, requests[index].DeletionTimestamp.IsZero()) - } - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, types.NamespacedName{ - Namespace: dgd.Namespace, - Name: desired.plan.PodCliqueSetName, - }, pcs)) - require.Zero(t, pcs.Spec.Replicas) - - t.Log("Restore nonzero Grove scale while scheduler finalizers still hold both requests in deletion") - pcs.Spec.Replicas = 1 - require.NoError(t, reconciler.Update(ctx, pcs)) - lifecycleRequestListCalls, dependencyListCalls = 0, 0 - - t.Log("Re-establish scale-to-zero using only the authoritative attempt-wide request observation") - classification, err = reconciler.retireLPXRequest(ctx, dgd, desired.plan.PodCliqueSetName, &requests[0], "test group retirement") - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - require.Positive(t, lifecycleRequestListCalls) - require.Zero(t, dependencyListCalls) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - require.Zero(t, pcs.Spec.Replicas) -} - -func TestSelectedLPXObservesLimitsOnlyCyborgGPUIntent(t *testing.T) { - t.Log("Materialize a hybrid Cyborg whose classic GPU intent exists only in limits") - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - cyborgPodSpec := &source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec - cyborgPodSpec.ResourceClaims = nil - cyborgPodSpec.Containers[0].Resources = corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), - }, - } - reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) - cyborg.Spec.PodSpec = *cyborgPodSpec.DeepCopy() - cyborg.Spec.PodSpec.SchedulerName = corev1.DefaultSchedulerName - pcs := findLPXTestPodCliqueSet(t, objects) - for _, template := range pcs.Spec.Template.Cliques { - if template.Name == desired.plan.CyborgTemplate { - template.Spec.PodSpec = *cyborg.Spec.PodSpec.DeepCopy() - } - } - container := cyborg.Spec.PodSpec.Containers[0] - require.Empty(t, container.Resources.Requests) - require.NotEmpty(t, container.Resources.Limits) - - t.Log("Observe the raw Grove PodSpec without relying on Kubernetes Pod request defaulting") - createLPXTestObjects(t, t.Context(), reconciler.Client, objects...) - classification, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - - t.Log("Retire the published request if the live classic GPU intent disappears") - storedCyborg := &grovev1alpha1.PodClique{} - require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(cyborg), storedCyborg)) - storedCyborg.Spec.PodSpec.ResourceClaims = nil - for index := range storedCyborg.Spec.PodSpec.Containers { - if storedCyborg.Spec.PodSpec.Containers[index].Name == consts.MainContainerName || - storedCyborg.Spec.PodSpec.Containers[index].Name == "cyborg" { - storedCyborg.Spec.PodSpec.Containers[index].Resources = corev1.ResourceRequirements{} - } - } - require.NoError(t, reconciler.Update(t.Context(), storedCyborg)) - classification, err = reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxClosed{}, classification) - requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) -} - -func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { +func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { for _, test := range []struct { name string snapshotError error @@ -520,8 +287,7 @@ func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.IsType(t, &lpxOpen{}, classification) requestBefore := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) key := client.ObjectKeyFromObject(dgd) @@ -550,7 +316,7 @@ func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { dgd.Spec.InputRevision = revision require.NoError(t, reconciler.Update(ctx, dgd)) } - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) pcs := &grovev1alpha1.PodCliqueSet{} t.Log("Failed desired input must preserve the exact published request and PCS") require.Equal(t, requestBefore, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name)) @@ -573,96 +339,6 @@ func TestLPXPublishedWorkloadFailurePreservesExistingEngines(t *testing.T) { } } -func TestLPXAuthoritativeLifecycleListConsumesContinuePages(t *testing.T) { - t.Log("Create lifecycle objects spanning multiple authoritative pages") - ctx := t.Context() - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - dgd := newLPXTestDeployment(t, source) - reconciler := newLPXTestReconciler(t, nil, dgd, source) - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - - expected := make([]lpxv1alpha1.LPUPipelineRequest, lpxLifecycleListPageSize+1) - for index := range expected { - expected[index] = lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("request-%02d", index), - Namespace: dgd.Namespace, - }} - } - const nextPageToken = "lpx-request-page-2" - - t.Log("Install a paginated reader that repopulates each supplied page in place") - requestGets := 0 - requestListCalls := make([]client.ListOptions, 0, 2) - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - Get: func( - ctx context.Context, - delegated client.WithWatch, - key client.ObjectKey, - object client.Object, - opts ...client.GetOption, - ) error { - if _, request := object.(*lpxv1alpha1.LPUPipelineRequest); request { - requestGets++ - } - return delegated.Get(ctx, key, object, opts...) - }, - List: func( - ctx context.Context, - delegated client.WithWatch, - list client.ObjectList, - opts ...client.ListOption, - ) error { - requestList, request := list.(*lpxv1alpha1.LPUPipelineRequestList) - if !request { - return delegated.List(ctx, list, opts...) - } - options := *(&client.ListOptions{}).ApplyOptions(opts) - requestListCalls = append(requestListCalls, options) - switch options.Continue { - case "": - requestList.Items = append(requestList.Items[:0], expected[:lpxLifecycleListPageSize]...) - requestList.Continue = nextPageToken - case nextPageToken: - requestList.Items = append(requestList.Items[:0], expected[lpxLifecycleListPageSize:]...) - requestList.Continue = "" - default: - return fmt.Errorf("unexpected LPX request continuation token %q", options.Continue) - } - return nil - }, - }) - - t.Log("Retain raw callback pointers until every continuation page is consumed") - retained := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expected)) - require.NoError(t, visitLifecycleObjectPages( - ctx, - reconciler.apiReader, - &lpxv1alpha1.LPUPipelineRequestList{}, - func(object k8sruntime.Object) error { - request := object.(*lpxv1alpha1.LPUPipelineRequest) - retained = append(retained, request) - return nil - }, - client.InNamespace(dgd.Namespace), - )) - - t.Log("Verify retained first-page objects and exact pagination options after traversal") - require.Len(t, retained, len(expected)) - for index := range expected { - require.Equal(t, expected[index], *retained[index]) - } - require.Zero(t, requestGets, "pagination must not fall back to per-object request Gets") - require.Len(t, requestListCalls, 2, "the authoritative list must make exactly one call per returned page") - require.Equal(t, "", requestListCalls[0].Continue) - require.Equal(t, nextPageToken, requestListCalls[1].Continue) - for _, call := range requestListCalls { - require.Equal(t, int64(lpxLifecycleListPageSize), call.Limit) - require.Equal(t, dgd.Namespace, call.Namespace) - require.Nil(t, call.LabelSelector) - } -} - func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { t.Log("Render initial root metadata and seed an exact-owned PCS with native Grove controls") ctx := t.Context() @@ -706,9 +382,10 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { }) t.Log("Only bookkeeping changes; root metadata additions, edits and removals are ignored") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, cached, desired) + modified, synced, err := commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, cached, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, updates) want := before.DeepCopy() @@ -719,18 +396,20 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { t.Log("Root metadata differences alone are a no-op once bookkeeping is current") updates = 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) + modified, synced, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.False(t, modified) require.Zero(t, updates) require.Equal(t, want, synced) t.Log("Template metadata still updates through Spec without changing root metadata or native controls") desired.Spec.Template.Cliques[0].Annotations = map[string]string{"test.example/template": "next"} - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) + modified, synced, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, updates) want.Spec = desired.Spec @@ -748,9 +427,10 @@ func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { require.NoError(t, base.Update(ctx, concurrent)) desired.Spec.Replicas += 2 updates = 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, cached, desired) + modified, synced, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, cached, desired, commoncontroller.WithPreservedListOrder(), + ) require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) - require.Nil(t, retiring) require.Nil(t, synced) require.False(t, modified) require.Equal(t, 1, updates) @@ -770,9 +450,10 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers = []corev1.Container{ {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, } - observed, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) + modified, observed, err := commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.True(t, modified) t.Log("Model API-defaulted live fields without changing the applied generation") @@ -786,9 +467,10 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("Apply authored ordering changes but ignore unchanged desired specs") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, observed, desired) + modified, synced, err := commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, observed, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.Equal(t, reorder, modified) require.Equal(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers, synced.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) require.Equal(t, before, observed, "synchronization must not mutate its observation") @@ -797,15 +479,16 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { } t.Log("The next reconciliation is a no-op") - _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, synced, desired) + modified, _, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.False(t, modified) }) } } -func TestLPXPodCliqueSetPublicationFence(t *testing.T) { +func TestLPXPublicationFencePrecedesPodCliqueSetWrite(t *testing.T) { t.Log("Prepare a render for publication and observe writes across the authoritative fence") ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -839,39 +522,25 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { }) t.Log("A failed publication observation prevents initial creation") - synced, modified, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, true) require.ErrorIs(t, err, fenceErr) - require.Nil(t, retiring) - require.False(t, modified) - require.Nil(t, synced) + require.Nil(t, classification) + require.Nil(t, currents) require.Zero(t, writes) require.Equal(t, 1, lists) require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) - t.Log("Retire an owned request without creating a PCS while its finalizer holds deletion") + t.Log("Recover the observation and create the exact child-owned PCS") failFence = false - request := deadlineTestRequest(dgd, "previous-attempt", time.Now(), lpxv1alpha1.RequestPhaseBound) - request.ResourceVersion = "" - request.Finalizers = []string{"test.example/scheduler"} - require.NoError(t, base.Create(ctx, request)) writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) + currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, true) require.NoError(t, err) - require.NotNil(t, retiring) - require.False(t, modified) - require.Nil(t, synced) - require.Zero(t, writes) - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(request), request)) - require.NotNil(t, request.DeletionTimestamp) - require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) - - t.Log("Finish request cleanup and create the exact child-owned PCS") - request.Finalizers = nil - require.NoError(t, base.Update(ctx, request)) - writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) + require.Nil(t, classification) + require.Empty(t, currents) + modified, synced, err := commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.True(t, modified) require.Equal(t, 1, lists) require.Equal(t, 1, writes) @@ -888,9 +557,10 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) t.Log("A stale missing observation does not adopt an already-created PCS") - _, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, nil, desired.DeepCopy()) + modified, _, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), + ) require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) - require.Nil(t, retiring) require.False(t, modified) t.Log("An in-place update does not require listing every scheduler request") @@ -901,9 +571,10 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" failFence = true writes, lists = 0, 0 - synced, modified, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX(ctx, dgd, lpx.ServingComponent(source).Replicas, live, desired) + modified, synced, err = commoncontroller.SyncObservedResource( + ctx, reconciler, dgd, live, desired, commoncontroller.WithPreservedListOrder(), + ) require.NoError(t, err) - require.Nil(t, retiring) require.True(t, modified) require.Equal(t, live.UID, synced.UID) require.Equal(t, 1, writes) @@ -913,6 +584,126 @@ func TestLPXPodCliqueSetPublicationFence(t *testing.T) { require.Equal(t, live.Annotations[lpx.WorkloadDigestAnnotation], stored.Annotations[lpx.WorkloadDigestAnnotation]) } +func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodCliqueSet(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) + require.NoError(t, reconciler.Delete(ctx, pcs)) + + t.Log("Keep the replacement PCS fenced while its predecessor's request remains persisted") + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + require.NoError(t, err) + require.Len(t, currents, 1) + require.IsType(t, &lpxRetiring{}, classification) + + t.Log("Garbage collection alone cannot authorize replacing a recorded scheduling attempt") + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.NoError(t, reconciler.Delete(ctx, request)) + currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + require.NoError(t, err) + require.Empty(t, currents) + require.IsType(t, &lpxRetiring{}, classification) +} + +func TestLPXScaleDownDeletesOnlyStaleRequests(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Components[0].Replicas = ptr.To[int32](3) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + for _, projection := range desired.requests { + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, projection.requestName) + request.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(request.Generation), + } + require.NoError(t, reconciler.Update(ctx, request)) + } + pcs := findLPXTestPodCliqueSet(t, objects) + pcsUID := pcs.UID + retainedUID := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName).UID + + t.Log("Scale from three replicas to one while every request remains pending") + scaledDown := withLPXTestReplicas(t, desired, 1) + _, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + require.NoError(t, err) + transition, ok := classification.(*lpxDeadlineTransition) + require.True(t, ok) + require.Len(t, transition.attempt.Requests, 1) + dgd.Status.Placement.LPXAttempt = transition.attempt + require.NoError(t, reconciler.Status().Update(ctx, dgd)) + + t.Log("Delete every stale LPR concurrently without replacing the shared PCS") + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + require.Len(t, currents, 1) + for _, projection := range desired.requests[1:] { + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, projection.requestName) + } + retained := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + require.Equal(t, retainedUID, retained.UID) + storedPCS := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, pcsUID, storedPCS.UID) +} + +func TestLPXScaleUpWaitsForTerminatingScaledDownRequest(t *testing.T) { + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Components[0].Replicas = ptr.To[int32](2) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + pcsUID := pcs.UID + removed := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[1].requestName) + removedUID := removed.UID + removed.Finalizers = []string{"scheduler.example/cleanup"} + require.NoError(t, reconciler.Update(ctx, removed)) + + t.Log("Scale down and begin direct request deletion") + scaledDown := withLPXTestReplicas(t, desired, 1) + _, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + require.NoError(t, err) + transition, ok := classification.(*lpxDeadlineTransition) + require.True(t, ok) + dgd.Status.Placement.LPXAttempt = transition.attempt + require.NoError(t, reconciler.Status().Update(ctx, dgd)) + _, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + terminating := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) + require.False(t, terminating.DeletionTimestamp.IsZero()) + + t.Log("Scale back up while the same request name is still terminating") + _, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, false) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + storedPCS := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, pcsUID, storedPCS.UID) + + t.Log("Recreate only the request after scheduler cleanup completes") + terminating.Finalizers = nil + require.NoError(t, reconciler.Update(ctx, terminating)) + requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) + classification = publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + require.IsType(t, &lpxOpen{}, classification) + recreated := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) + require.NotEqual(t, removedUID, recreated.UID) + require.Equal(t, pcsUID, metav1.GetControllerOf(recreated).UID) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, pcsUID, storedPCS.UID) +} + func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -920,10 +711,7 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) rendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, desired) - _, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) published := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} @@ -934,15 +722,15 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { require.NoError(t, reconciler.Update(ctx, drifted)) require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - _, _, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( + _, _, err := commoncontroller.SyncObservedResource( ctx, + reconciler, dgd, - lpx.ServingComponent(source).Replicas, drifted, rendered, + commoncontroller.WithPreservedListOrder(), ) require.NoError(t, err) - require.Nil(t, retiring) require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) t.Log("Correct the spec in place without stopping the existing PCS") @@ -956,7 +744,7 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { require.NotEmpty(t, foreignPCS.OwnerReferences) foreignPCS.OwnerReferences[0].UID = "foreign-owner" foreignReconciler := newLPXTestReconciler(t, registry, dgd, source, foreignPCS) - _, err = foreignReconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil, false) + _, err = foreignReconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil) require.EqualError(t, err, fmt.Sprintf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", foreignPCS.Name)) t.Log("An explicitly stopped PCS still cannot publish another request") @@ -978,23 +766,23 @@ func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { return errors.New("retired Grove observation must not read dependencies") }, }) - classification, err := reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil, false) + classification, err := reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil) require.NoError(t, err) closed := requireLPXClosed(t, classification) - require.Equal(t, "The retired Grove attempt is held at zero; successor publication waits for post-retirement Grove synchronization", closed.incomplete) + require.Equal(t, "The selected Grove PodCliqueSet has 0 replicas, want 1", closed.incomplete) require.Equal(t, 1, pcsGets) reconciler.apiReader = reader t.Log("Restore the explicit hold in place without rotating the existing request") - _, _, retiring, err = reconciler.reconcileGrovePodCliqueSetForLPX( + _, _, err = commoncontroller.SyncObservedResource( ctx, + reconciler, dgd, - lpx.ServingComponent(source).Replicas, drifted, rendered, + commoncontroller.WithPreservedListOrder(), ) require.NoError(t, err) - require.Nil(t, retiring) recreated := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) @@ -1014,7 +802,7 @@ func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { require.Nil(t, rejected) oldPCS := findLPXTestPodCliqueSet(t, lpxMaterializedObjects(t, oldPrepareReconciler, oldDGD, oldSource, oldDesired)) - published := deadlineTestRequest(oldDGD, oldDesired.requests[0].requestName, time.Now(), lpxv1alpha1.RequestPhaseBound) + published := deadlineTestRequest(oldDGD, oldPCS, oldDesired.requests[0].requestName, time.Now(), lpxv1alpha1.RequestPhaseBound) const successorBuildID = "build-v2-successor" successorRegistry := newLPXTestRegistryWithPartitionsAndMode( @@ -1058,15 +846,15 @@ func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { return delegated.Update(ctx, object, opts...) }, }) - synced, changed, retiring, err := reconciler.reconcileGrovePodCliqueSetForLPX( + changed, synced, err := commoncontroller.SyncObservedResource( ctx, + reconciler, successorDGD, - lpx.ServingComponent(successorSource).Replicas, pcs, successorRendered, + commoncontroller.WithPreservedListOrder(), ) require.ErrorIs(t, err, admissionErr) - require.Nil(t, retiring) require.Nil(t, synced) require.False(t, changed) @@ -1096,18 +884,12 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { require.Nil(t, source.Spec.Scheduling) require.NotContains(t, source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) - t.Log("Materialize the selected runtime alongside an unrelated ordinary frontend group") + t.Log("Materialize the selected runtime") objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - ordinary := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) - ordinary.Spec.PodGroups = append(ordinary.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{ - Name: "frontend", MinReplicas: 1, - PodReferences: []groveschedulerv1alpha1.NamespacedName{{Namespace: dgd.Namespace, Name: "frontend-0"}}, - }) createLPXTestObjects(t, ctx, reconciler.Client, objects...) t.Log("Keep an otherwise-ready runtime gated before the exact request is Bound") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.IsType(t, &lpxOpen{}, classification) runtimeReady := reconcileOutcome{ State: nvidiacomv1beta1.DGDStateSuccessful, @@ -1130,7 +912,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { require.Nil(t, rejected) wantDigest := desired.requests[0].attemptDigest desired = rederived - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) require.Equal(t, wantDigest, desired.requests[0].attemptDigest) @@ -1141,7 +923,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { request.Status = newLPXNodeLocalBoundStatus(7, 3, "sha256:bound-plan") require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) bound, ok := classification.(*lpxBound) require.True(t, ok) @@ -1155,19 +937,11 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { } require.Equal(t, runtimePending, overlayLPXResult(runtimePending, bound)) - t.Run("unknown backend arm is not bound", func(t *testing.T) { - t.Log("Reject an execution backend outside the scheduler API contract") - unknownBackend := request.DeepCopy() - unknownBackend.Spec.ExecutionBackend = "unknown" - unknownBackend.Status.Committed.Plan.Placement = lpxv1alpha1.PlanPlacement{ExecutionBackend: "unknown"} - require.IsType(t, &lpxSchedulerObserved{}, classifyPublishedLPX(unknownBackend)) - }) - t.Log("Keep runtime readiness gated when the Bound receipt belongs to an older generation") request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) request.Status.ObservedGeneration = ptr.To(int64(6)) require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) @@ -1175,89 +949,23 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { t.Log("Stale failure diagnostics are not attributed to the current request generation") request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) request.Status.Phase = lpxv1alpha1.RequestPhaseUnsupported - request.Status.Committed = nil request.Status.Diagnostics = []lpxv1alpha1.StatusDiagnostic{{ Code: "StaleUnsupported", Subject: "request/old", Detail: "belongs to an older generation", }} require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) require.NotContains(t, lpxResult(classification).Message, "StaleUnsupported") - t.Log("Delete the stored child while scheduler cleanup holds its published request") + t.Log("The owner chain is sufficient for deletion without Dynamo finalizers") request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Finalizers = []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"} - require.NoError(t, reconciler.Update(ctx, request)) - key := client.ObjectKeyFromObject(dgd) - require.NoError(t, reconciler.Get(ctx, key, dgd)) - require.NoError(t, reconciler.Delete(ctx, dgd)) - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - waiting := fmt.Sprintf("waiting for LPX request %q to finish fail-closed retirement", request.Name) - require.EqualError(t, err, waiting) - request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, request.Name) - require.False(t, request.DeletionTimestamp.IsZero()) - require.Equal(t, []string{lpxAttemptRecordingFinalizer, "scheduling.lpu.nvidia.com/lpx-cleanup"}, request.Finalizers) pcs := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) - require.Zero(t, pcs.Spec.Replicas, "child finalization must stop the workload before scheduler cleanup finishes") - - t.Log("Keep the child finalizer until the scheduler releases the exact request") - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.EqualError(t, err, waiting) - require.NoError(t, reconciler.Get(ctx, key, dgd)) - require.Contains(t, dgd.Finalizers, lpxGraphDeploymentFinalizer) - request.Finalizers = []string{lpxAttemptRecordingFinalizer} - require.NoError(t, reconciler.Update(ctx, request)) - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.EqualError(t, err, waiting) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, request.Name) - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(reconciler.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) -} - -func TestLPXFinalizationAfterRequestAPIRemoval(t *testing.T) { - resource := lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource() - for _, scenario := range []struct { - name string - listErr error - finalize bool - }{ - {"removed CRD", apierrors.NewNotFound(resource, ""), true}, - {"forbidden", apierrors.NewForbidden(resource, "", errors.New("denied")), false}, - {"transport error", errors.New("connection reset"), false}, - } { - t.Run(scenario.name, func(t *testing.T) { - t.Log("Return a request-list error after the operator has discovered the API") - ctx := t.Context() - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - child := newLPXTestDeployment(t, source) - r := newLPXTestReconciler(t, nil, child, source) - r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return scenario.listErr - }, - }) - - t.Log("Ordinary reconciliation must still fail when requests cannot be listed") - _, err := r.listOwnedLPXRequests(ctx, child) - require.Error(t, err) - - t.Log("Allow deletion only when the request API is gone, not on operational errors") - key := client.ObjectKeyFromObject(child) - require.NoError(t, r.Delete(ctx, child)) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - if scenario.finalize { - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, key, child))) - } else { - require.ErrorIs(t, err, scenario.listErr) - require.NoError(t, r.Get(ctx, key, child)) - require.Contains(t, child.Finalizers, lpxGraphDeploymentFinalizer) - } - }) - } + require.True(t, metav1.IsControlledBy(pcs, dgd)) + require.True(t, metav1.IsControlledBy(request, pcs)) + require.Empty(t, request.Finalizers) + require.Empty(t, dgd.Finalizers) } func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { @@ -1344,55 +1052,13 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { } } -func TestLPXRequestReadinessDecisions(t *testing.T) { - t.Log("Keep staged publication ahead of Grove pending and retain nil continuation") - pending := &lpxClosed{incomplete: "Grove is incomplete"} - staged := &lpxClosed{incomplete: "The Grove publication identities are staged; LPX publication waits for Grove synchronization"} - for _, test := range []struct { - name string - identity *lpxGroveIdentity - published bool - pending *lpxClosed - want *lpxClosed - }{ - {name: "staged before pending", identity: &lpxGroveIdentity{complete: true}, pending: pending, want: staged}, - {name: "staged without pending", identity: &lpxGroveIdentity{complete: true}, want: staged}, - {name: "published before pending", identity: &lpxGroveIdentity{complete: true}, published: true, pending: pending, want: pending}, - {name: "incomplete identity", identity: &lpxGroveIdentity{}, pending: pending, want: pending}, - {name: "missing identity", pending: pending, want: pending}, - {name: "published continues", identity: &lpxGroveIdentity{complete: true}, published: true}, - {name: "missing continues"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Resolve the requested replica without changing its current or desired request") - requests := []lpxModelMaterializing{{requestName: "engine", replicaIndex: 1}} - beforeRequests := slices.Clone(requests) - currents := map[string]*lpxv1alpha1.LPUPipelineRequest{} - if test.published { - currents["engine"] = deadlineTestRequest(&nvidiacomv1alpha1.LPXGraphDeployment{}, "engine", time.Now(), lpxv1alpha1.RequestPhaseBound) - } - before := currents["engine"].DeepCopy() - require.Equal(t, test.want, classifyLPXPublicationReadiness(requests, currents, []*lpxGroveIdentity{nil, test.identity}, test.pending)) - var want lpxClassification = &lpxBound{} - if test.pending != nil { - want = test.pending - } - require.Equal(t, want, classifyLPXCurrentRequests(requests, currents, test.pending)) - require.Equal(t, beforeRequests, requests) - require.Equal(t, before, currents["engine"]) - }) - } - require.Equal(t, &lpxClosed{incomplete: "Grove is incomplete"}, pending) -} - func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { tests := []struct { - name string - phase lpxv1alpha1.RequestPhase - releaseReason *lpxv1alpha1.ReleaseReason - wantState nvidiacomv1beta1.DGDState - wantReason string - wantMessage string + name string + phase lpxv1alpha1.RequestPhase + wantState nvidiacomv1beta1.DGDState + wantReason string + wantMessage string }{ {name: "pending", phase: lpxv1alpha1.RequestPhasePending, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXSchedulerPending"}, {name: "no fit", phase: lpxv1alpha1.RequestPhaseNoFit, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXNoFit"}, @@ -1400,42 +1066,7 @@ func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { {name: "planned", phase: lpxv1alpha1.RequestPhasePlanned, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXPlanned"}, {name: "reserving", phase: lpxv1alpha1.RequestPhaseReserving, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReserving"}, {name: "binding", phase: lpxv1alpha1.RequestPhaseBinding, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXBinding"}, - {name: "degraded", phase: lpxv1alpha1.RequestPhaseDegraded, wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXDegraded"}, - { - name: "failure release in progress", phase: lpxv1alpha1.RequestPhaseReleasing, - releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonBindingFailed), - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXReleasing", wantMessage: "bindingFailed", - }, - { - name: "request change release in progress", phase: lpxv1alpha1.RequestPhaseReleasing, - releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonRequestChanged), - wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReleasing", wantMessage: "requestChanged", - }, - { - name: "failure release complete", phase: lpxv1alpha1.RequestPhaseReleased, - releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonDependencyChanged), - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXReleased", wantMessage: "dependencyChanged", - }, - { - name: "deletion release complete", phase: lpxv1alpha1.RequestPhaseReleased, - releaseReason: ptr.To(lpxv1alpha1.ReleaseReasonRequestDeleted), - wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReleased", wantMessage: "requestDeleted", - }, - { - name: "release without journal", phase: lpxv1alpha1.RequestPhaseReleasing, - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", - wantMessage: "without the required release journal", - }, - { - name: "release with unknown reason", phase: lpxv1alpha1.RequestPhaseReleased, - releaseReason: ptr.To(lpxv1alpha1.ReleaseReason("futureReason")), - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", - wantMessage: "unknown release reason", - }, - { - name: "bound without exact proof", phase: lpxv1alpha1.RequestPhaseBound, - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", wantMessage: "without the exact current-generation plan proof", - }, + {name: "degraded", phase: lpxv1alpha1.RequestPhaseDegraded, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXDegraded"}, { name: "unknown phase", phase: lpxv1alpha1.RequestPhase("future"), wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", wantMessage: `unknown phase "future"`, @@ -1456,13 +1087,6 @@ func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { }}, }, } - if test.releaseReason != nil { - request.Status.Committed = &lpxv1alpha1.Committed{ - Execution: lpxv1alpha1.CommittedExecution{ - Release: &lpxv1alpha1.ReleaseJournal{Reason: *test.releaseReason, RequestedAtGeneration: 7}, - }, - } - } t.Log("Classify the current receipt and preserve its phase, journal and diagnostic") classification := classifyPublishedLPX(request) @@ -1485,14 +1109,13 @@ func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { later.Status.Diagnostics[0].Detail = "a later scheduler receipt" requests := []lpxModelMaterializing{{requestName: "open"}, {requestName: "observed"}, {requestName: "later"}} currents := map[string]*lpxv1alpha1.LPUPipelineRequest{"open": {}, "observed": request, "later": later} - pending := &lpxClosed{incomplete: "Grove is incomplete"} var want lpxClassification = &lpxOpen{} if test.wantState == nvidiacomv1beta1.DGDStateFailed { want = observed } - require.Equal(t, want, classifyLPXCurrentRequests(requests, currents, pending)) + require.Equal(t, want, classifyLPXCurrentRequests(requests, currents)) slices.Reverse(requests) - require.Equal(t, (*lpxSchedulerObserved)(later.Status), classifyLPXCurrentRequests(requests, currents, pending)) + require.Equal(t, (*lpxSchedulerObserved)(later.Status), classifyLPXCurrentRequests(requests, currents)) require.Equal(t, before, request) }) } @@ -1515,86 +1138,6 @@ func TestLPXSchedulerDiagnosticMessageRespectsConditionLimit(t *testing.T) { require.True(t, strings.HasSuffix(message, "…")) } -func TestLPXNativeRolloutPreservesOtherEngines(t *testing.T) { - for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { - t.Run(string(pipeline), func(t *testing.T) { - t.Log("Publish two engines and stage a third under the same PCS") - ctx := t.Context() - _, source, registry := newLPXTestDGD(t, pipeline) - source.Spec.Components[0].Replicas = ptr.To[int32](3) - deployment := newLPXTestDeployment(t, source) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) - objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - allRequests := desired.requests - desired.requests = allRequests[:2] - _, err := reconciler.reconcileSelectedLPX(ctx, deployment, desired) - require.NoError(t, err) - desired.requests = allRequests - old := &lpxv1alpha1.LPUPipelineRequest{} - kept := &lpxv1alpha1.LPUPipelineRequest{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[0].requestName}, old)) - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[1].requestName}, kept)) - old.Finalizers = []string{"test.example/scheduler"} - require.NoError(t, reconciler.Update(ctx, old)) - require.Nil(t, old.Status, "native identity changes must not require a scheduler receipt") - - t.Log("Grove replaces engine zero; engine one remains old, while engine two is ready to publish") - pcs := findLPXTestPodCliqueSet(t, objects) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - pcs.Status.CurrentGenerationHash = ptr.To("next-hash") - pcs.Status.ObservedGeneration = ptr.To[int64](0) - require.NoError(t, reconciler.Update(ctx, pcs)) - for _, object := range objects { - clique, ok := object.(*grovev1alpha1.PodClique) - if !ok || clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] == "1" { - continue - } - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(clique), clique)) - require.NoError(t, reconciler.Delete(ctx, clique)) - clique.UID += "-replacement" - clique.ResourceVersion = "" - clique.Status.CurrentPodCliqueSetGenerationHash = ptr.To("next-hash") - require.NoError(t, reconciler.Create(ctx, clique)) - } - _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) - require.NoError(t, err) - live := &lpxv1alpha1.LPUPipelineRequest{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(old), live)) - require.NotNil(t, live.DeletionTimestamp) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(kept), kept)) - require.Nil(t, kept.DeletionTimestamp) - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: allRequests[2].requestName}, &lpxv1alpha1.LPUPipelineRequest{})) - - t.Log("Publish the replacement under its stable name after its own finalizer finishes") - live.Finalizers = nil - require.NoError(t, reconciler.Update(ctx, live)) - _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) - require.NoError(t, err) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(old), live)) - require.NotEqual(t, old.UID, live.UID) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - require.Equal(t, types.UID("pcs-uid"), pcs.UID) - - t.Log("An owned deleting Agent retires its request even while its PodGang is absent") - live.Finalizers = []string{"test.example/scheduler"} - require.NoError(t, reconciler.Update(ctx, live)) - agent := findLPXTestClique(t, objects, desired.plan.Agents[0].CliqueName) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(agent), agent)) - agent.Finalizers = []string{"test.example/grove"} - require.NoError(t, reconciler.Update(ctx, agent)) - require.NoError(t, reconciler.Delete(ctx, agent)) - require.NoError(t, reconciler.Delete(ctx, &groveschedulerv1alpha1.PodGang{ObjectMeta: metav1.ObjectMeta{ - Name: live.Spec.PodGangRef.Name, Namespace: deployment.Namespace, - }})) - _, err = reconciler.reconcileSelectedLPX(ctx, deployment, desired) - require.NoError(t, err) - require.False(t, getLPXRequest(t, ctx, reconciler.Client, deployment.Namespace, live.Name).DeletionTimestamp.IsZero()) - require.Equal(t, kept.UID, getLPXRequest(t, ctx, reconciler.Client, deployment.Namespace, kept.Name).UID) - }) - } -} - func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { type digestInput struct { namespace string @@ -1637,87 +1180,20 @@ func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - t.Log("Fence exact PCS and PodGang UID annotations") + t.Log("Fence the exact PCS owner and immutable attempt annotations") projection := &desired.requests[0] - identity := &lpxGroveIdentity{pcsUID: "pcs-uid", podGangName: "podgang", podGangUID: "podgang-uid"} + identity := &lpxGroveIdentity{pcsUID: "pcs-uid", pcs: deadlineTestPCS(dgd, "pcs-uid"), plan: desired.plan} current := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection, identity)}, - Spec: projection.modelProjection.RequestSpec(dgd.Namespace, identity.podGangName, identity.cyborgClique), + ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection, identity), OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(identity.pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}}, + Spec: projection.modelProjection.RequestSpec(identity.plan, projection.agentCliqueName), } require.NoError(t, validateCurrentLPXRequest(dgd, projection, identity, current)) changed := *identity changed.pcsUID = "replacement-pcs-uid" require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") - changed = *identity - changed.podGangUID = "replacement-podgang-uid" - require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") - t.Log("Resolve current intent from complete identities and preserve inputs while Grove is incomplete") - complete := *identity - complete.complete, complete.agentUIDs = true, []types.UID{"agent-uid"} - computed, intentChanged, err := resolveLPXCurrentRequestIntent(dgd, *projection, &complete, nil) - require.NoError(t, err) - require.False(t, intentChanged) - require.True(t, strings.HasPrefix(computed, "sha256:")) - intent := *projection - intent.attemptDigest = computed - current.Annotations = lpxRequestAnnotations(dgd, &intent, &complete) - intent.attemptDigest = "supplied-digest" - replacement := complete - replacement.agentUIDs = []types.UID{"replacement-agent-uid"} - for _, test := range []struct { - name string - identity *lpxGroveIdentity - live *lpxv1alpha1.LPUPipelineRequest - drift bool - wantDigest string - wantChanged bool - }{ - {name: "match", identity: &complete, live: current, wantDigest: computed}, - {name: "new request", identity: &complete, wantDigest: computed}, - {name: "spec drift", identity: &complete, live: current, drift: true, wantDigest: computed, wantChanged: true}, - {name: "agent replacement", identity: &replacement, live: current, wantChanged: true}, - {name: "incomplete with live request", identity: identity, live: current, drift: true, wantDigest: computed}, - {name: "missing identity with live request", live: current, drift: true, wantDigest: computed}, - {name: "incomplete without request", identity: identity, wantDigest: intent.attemptDigest}, - {name: "missing identity and request", wantDigest: intent.attemptDigest}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Compare immutable intent without changing the supplied digest or observed request") - live := test.live.DeepCopy() - if test.drift { - live.Spec.PodGangRef.Name = "other-gang" - } - beforeLive, beforeDGD, beforeIntent := live.DeepCopy(), dgd.DeepCopy(), intent - beforeSpec := intent.modelProjection.RequestSpec(dgd.Namespace, complete.podGangName, nil) - var beforeIdentity *lpxGroveIdentity - if test.identity != nil { - copy := *test.identity - copy.agentUIDs = slices.Clone(copy.agentUIDs) - beforeIdentity = © - } - got, changed, err := resolveLPXCurrentRequestIntent(dgd, intent, test.identity, live) - require.NoError(t, err) - require.Equal(t, test.wantChanged, changed) - if test.wantDigest == "" { - require.NotEqual(t, computed, got, "replacement Agent identity changes the attempt digest") - } else { - require.Equal(t, test.wantDigest, got) - } - require.Equal(t, beforeLive, live) - require.Equal(t, beforeDGD, dgd) - require.Equal(t, beforeIntent, intent) - require.Equal(t, beforeIdentity, test.identity) - require.Equal(t, beforeSpec, intent.modelProjection.RequestSpec(dgd.Namespace, complete.podGangName, nil)) - }) - } - - t.Log("Keep published requests when the deployment generation changes without intent drift") createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) dgd.Generation++ classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) @@ -1813,390 +1289,75 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { require.Equal(t, list, current) } - t.Log("Explicit deletion still retires requests and releases the child finalizer while disabled") + t.Log("Deletion remains available while disabled and needs no controller finalizer") if scenario.name == "LPX" { r.runtimeConfig.Gate.LPX = false } else { r.runtimeConfig.Gate.Grove = false } r.modelRegistry = nil + require.Empty(t, child.Finalizers) require.NoError(t, r.Delete(ctx, child)) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.ErrorContains(t, err, "waiting for LPX request") - request := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - require.False(t, request.DeletionTimestamp.IsZero()) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: dynamo.PCSNameForLPX(child)}, pcs)) - require.Zero(t, pcs.Spec.Replicas) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.ErrorContains(t, err, "waiting for LPX request") - requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.NoError(t, err) require.True(t, apierrors.IsNotFound(r.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) }) } } +func TestLPXFinalizationAfterRequestAPIRemoval(t *testing.T) { + resource := lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource() + for _, scenario := range []struct { + name string + listErr error + }{ + {"removed CRD", apierrors.NewNotFound(resource, "")}, + {"forbidden", apierrors.NewForbidden(resource, "", errors.New("denied"))}, + {"transport error", errors.New("connection reset")}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Keep a legacy controller finalizer while the request API cannot be read") + ctx := t.Context() + source := newLPXTestSource(lpx.PipelineSingle, "build-v2") + child := newLPXTestDeployment(t, source) + child.Finalizers = []string{legacyLPXGraphDeploymentFinalizer} + r := newLPXTestReconciler(t, nil, child, source) + r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return scenario.listErr + }, + }) + + t.Log("Ordinary reconciliation still fails when requests cannot be listed") + _, err := r.listOwnedLPXRequests(ctx, child) + require.ErrorIs(t, err, scenario.listErr) + + t.Log("Release the legacy finalizer without reading requests; owner garbage collection owns cleanup") + key := client.ObjectKeyFromObject(child) + require.NoError(t, r.Delete(ctx, child)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, key, child))) + }) + } +} + func TestLPXControllerPreservesPublishedWorkWhenProviderEditIsInvalid(t *testing.T) { t.Log("Reject an incompatible provider edit without deleting the existing engine") dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) createLPXTestObjects(t, t.Context(), reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - _, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) - require.NoError(t, err) + publishSelectedLPXForTest(t, t.Context(), reconciler, dgd, desired) require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(source), source)) source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent require.NoError(t, reconciler.Update(t.Context(), source)) - _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) require.ErrorContains(t, err, "LPX requires the Grove workload provider") request := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) require.Nil(t, request.DeletionTimestamp) } -func TestLPUOnlyWaitsForCurrentConductorWitness(t *testing.T) { - tests := []struct{ name, defect, want string }{ - {name: "stale owner", defect: "owner", want: "ordinary-scheduler identity"}, - {name: "wrong PodGang", defect: "podgang", want: "ordinary-scheduler identity"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - conductor := findLPXTestClique(t, objects, desired.plan.ConductorClique) - switch test.defect { - case "owner": - conductor.OwnerReferences[0].UID = "previous-scaling-group-uid" - case "podgang": - conductor.Labels[grovecommon.LabelPodGang] = "other-podgang" - } - createLPXTestObjects(t, t.Context(), reconciler.Client, objects...) - classification, err := reconciler.reconcileSelectedLPX(t.Context(), dgd, desired) - require.NoError(t, err) - closed := requireLPXClosed(t, classification) - require.Contains(t, closed.incomplete, test.want) - requireLPXRequestNotFound(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - }) - } -} - -func TestNodeLocalHybridPublishesAndFencesCurrentCyborgMaterialization(t *testing.T) { - tests := []struct { - name string - generation int64 - observedGeneration *int64 - }{ - { - name: "absent generation then claim drift", - generation: 1, - observedGeneration: nil, - }, - { - name: "lagging generation then UID replacement", - generation: 2, - observedGeneration: ptr.To(int64(1)), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Arrange current Cyborg materialization evidence without an observed generation") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) - cyborg.Generation = test.generation - cyborg.Status.ObservedGeneration = test.observedGeneration - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - t.Log("Publish the NodeLocal request with the exact Cyborg name and UID") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - request := getLPXRequest( - t, - ctx, - reconciler.Client, - dgd.Namespace, - desired.requests[0].requestName, - ) - require.NotNil(t, request.Spec.CyborgPodCliqueRef) - require.Equal(t, cyborg.Name, request.Spec.CyborgPodCliqueRef.Name) - require.Equal(t, string(cyborg.UID), request.Spec.CyborgPodCliqueRef.UID) - publishedUID := request.UID - - storedCyborg := &grovev1alpha1.PodClique{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(cyborg), storedCyborg)) - if test.observedGeneration == nil { - t.Log("Drift a live Claim reference without an observed generation") - require.NotEmpty(t, storedCyborg.Spec.PodSpec.ResourceClaims) - storedCyborg.Generation++ - storedCyborg.Spec.PodSpec.ResourceClaims[0].ResourceClaimTemplateName = ptr.To("stale-gpu") - require.NoError(t, reconciler.Update(ctx, storedCyborg)) - - t.Log("An unobserved PCS edit permits temporary old-child references without retiring the engine") - pcs := findLPXTestPodCliqueSet(t, objects) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - pcs.Generation++ - require.NoError(t, reconciler.Update(ctx, pcs)) - _, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.Equal(t, publishedUID, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, request.Name).UID) - pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) - require.NoError(t, reconciler.Update(ctx, pcs)) - } else { - t.Log("Advance only the global generation without rotating the published request") - storedCyborg.Status.ObservedGeneration = ptr.To(test.generation) - require.NoError(t, reconciler.Update(ctx, storedCyborg)) - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - require.Equal(t, publishedUID, request.UID) - - t.Log("Replace the Cyborg clique at the same name with a different UID") - require.NoError(t, reconciler.Delete(ctx, storedCyborg)) - replacement := storedCyborg.DeepCopy() - replacement.ResourceVersion = "" - replacement.UID = "replacement-cyborg-uid" - replacement.DeletionTimestamp = nil - replacement.CreationTimestamp = metav1.Time{} - require.NoError(t, reconciler.Create(ctx, replacement)) - } - - t.Log("Retire the request when its exact published materialization no longer holds") - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxClosed{}, classification) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - }) - } -} - -func TestNodeLocalHybridWaitsForCurrentCyborgWitness(t *testing.T) { - tests := []struct { - name string - defect string - wantMessage string - }{ - { - name: "missing current scaling group", - defect: "missing-scaling-group", - wantMessage: "current LPU PodCliqueScalingGroup", - }, - { - name: "stale scaling-group owner", - defect: "stale-scaling-group-owner", - wantMessage: "current PodCliqueSet owner identity", - }, - { - name: "stale Cyborg owner", - defect: "stale-cyborg-owner", - wantMessage: "current LPU scaling-group owner identity", - }, - { - name: "missing PCS generation hash", - defect: "missing-pcs-hash", - wantMessage: "current PodCliqueSet generation", - }, - { - name: "mismatched PCS generation hash", - defect: "mismatched-pcs-hash", - wantMessage: "current PodCliqueSet generation", - }, - { - name: "missing Pod template hash", - defect: "missing-template-hash", - wantMessage: "current PodCliqueSet generation", - }, - { - name: "empty Pod template hash", - defect: "empty-template-hash", - wantMessage: "current PodCliqueSet generation", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Arrange one incomplete hybrid Cyborg witness") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) - switch test.defect { - case "missing-scaling-group": - scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - objects = slices.DeleteFunc(objects, func(object client.Object) bool { return object == scalingGroup }) - case "stale-scaling-group-owner": - scalingGroup := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - require.NotEmpty(t, scalingGroup.OwnerReferences) - scalingGroup.OwnerReferences[0].UID = "previous-pcs-uid" - case "stale-cyborg-owner": - require.NotEmpty(t, cyborg.OwnerReferences) - cyborg.OwnerReferences[0].UID = "previous-scaling-group-uid" - case "missing-pcs-hash": - cyborg.Status.CurrentPodCliqueSetGenerationHash = nil - case "mismatched-pcs-hash": - cyborg.Status.CurrentPodCliqueSetGenerationHash = ptr.To("previous-pcs-generation") - case "missing-template-hash": - cyborg.Status.CurrentPodTemplateHash = nil - case "empty-template-hash": - cyborg.Status.CurrentPodTemplateHash = ptr.To("") - default: - require.FailNow(t, "unknown materialization defect", "defect: %s", test.defect) - } - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - t.Log("Keep publication pending until a dependency watch supplies current evidence") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - closed := requireLPXClosed(t, classification) - require.Contains(t, closed.incomplete, test.wantMessage) - require.Equal(t, nvidiacomv1beta1.DGDStatePending, lpxResult(closed).State) - requireLPXRequestNotFound( - t, - ctx, - reconciler.Client, - dgd.Namespace, - desired.requests[0].requestName, - ) - }) - } -} - -func TestLPXSchedulerScopedGangAndStartupWitnessesAreDisjoint(t *testing.T) { - const ( - crossReplicaAgent = "other replica Agent enters ordinary gang" - missingLPXGang = "Waiting for one replica-0 LPX PodGang; found 0 candidates" - missingOrdinary = "Waiting for one replica-0 ordinary-scheduler PodGang; found 0 candidates" - ) - tests := []struct { - name, wantMessage string - }{ - {"LPX gang unavailable", missingLPXGang}, - {"LPX gang is stale", "Waiting for Grove to publish current-generation PodGangs"}, - {"LPX gang has incomplete references", missingLPXGang}, - {"LPX gang has an extra group", missingLPXGang}, - {"LPX gang is ambiguous", "Waiting for one replica-0 LPX PodGang; found 2 candidates"}, - {"Agent enters ordinary gang", missingOrdinary}, - {crossReplicaAgent, missingOrdinary}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Materialize the complete hybrid runtime and its scheduler-scoped gangs") - ctx := context.Background() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - if test.name == crossReplicaAgent { - source.Spec.Components[0].Replicas = ptr.To(int32(2)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(1)) - } - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - - t.Log("Introduce only the selected gang or startup-witness defect") - gang := findLPXTestPodGang(t, objects, lpx.SchedulerName) - switch test.name { - case "LPX gang unavailable": - gang.Labels[grovecommon.LabelPartOfKey] = lpxTestOtherName - case "LPX gang is stale": - gang.Annotations[lpxDeploymentUIDAnnotation] = "different-lgd" - case "LPX gang has incomplete references": - gang.Spec.PodGroups[0].PodReferences = nil - case "LPX gang has an extra group": - gang.Spec.PodGroups = append(gang.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{Name: "unexpected"}) - case "LPX gang is ambiguous": - duplicate := gang.DeepCopy() - duplicate.Name, duplicate.UID, duplicate.ResourceVersion = duplicate.Name+"-duplicate", "duplicate-uid", "" - objects = append(objects, duplicate) - case "Agent enters ordinary gang", crossReplicaAgent: - ordinary := findLPXTestPodGang(t, objects, corev1.DefaultSchedulerName) - agent := desired.plan.Agents[0] - if test.name == crossReplicaAgent { - agent = desired.plan.ForReplica(1).Agents[0] - } - ordinary.Spec.PodGroups = append(ordinary.Spec.PodGroups, groveschedulerv1alpha1.PodGroup{ - Name: agent.CliqueName, MinReplicas: 1, - PodReferences: []groveschedulerv1alpha1.NamespacedName{{Namespace: ordinary.Namespace, Name: agent.CliqueName + "-0"}}, - }) - } - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - t.Log("Keep the invalid engine unpublished while allowing an independent sibling") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) - require.NoError(t, err) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - if test.name == crossReplicaAgent { - require.IsType(t, &lpxOpen{}, classification) - require.Len(t, requests, 1) - require.Equal(t, desired.requests[1].requestName, requests[0].Name) - } else { - require.IsType(t, &lpxClosed{}, classification) - require.Equal(t, test.wantMessage, classification.(*lpxClosed).incomplete) - require.Empty(t, requests) - } - }) - } -} - -func TestLPXLiveRequestSizeGrowthRetiresPublication(t *testing.T) { - ctx := context.Background() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - require.NotNil(t, desired) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - - t.Log("Publish the request, then model API-server metadata growth") - classification, err := reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - storedClient := reconciler.Client - storedWithWatch, ok := storedClient.(client.WithWatch) - require.True(t, ok) - reconciler.apiReader = interceptor.NewClient(storedWithWatch, interceptor.Funcs{ - List: func( - ctx context.Context, - delegated client.WithWatch, - list client.ObjectList, - opts ...client.ListOption, - ) error { - if err := delegated.List(ctx, list, opts...); err != nil { - return err - } - requests, ok := list.(*lpxv1alpha1.LPUPipelineRequestList) - if !ok { - return nil - } - for index := range requests.Items { - request := &requests.Items[index] - if request.Name != desired.requests[0].requestName { - continue - } - request.ManagedFields = []metav1.ManagedFieldsEntry{{ - Manager: "metadata-growth-test", - FieldsType: "FieldsV1", - FieldsV1: &metav1.FieldsV1{ - Raw: []byte(`{"padding":"` + strings.Repeat("x", 850*1024) + `"}`), - }, - }} - } - return nil - }, - }) - - t.Log("Retire the now-oversized publication") - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxClosed{}, classification) - requireLPXRequestNotFound(t, ctx, storedClient, dgd.Namespace, desired.requests[0].requestName) -} - func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t *testing.T) { t.Log("Give foreign requests the desired names in the opposite lexical order") ctx := t.Context() @@ -2207,6 +1368,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t Name: "z-foreign-request", Namespace: dgd.Namespace, UID: "foreign-collision-uid", + Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, OwnerReferences: []metav1.OwnerReference{{ APIVersion: nvidiacomv1beta1.GroupVersion.String(), Kind: nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind, @@ -2224,7 +1386,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t reconciler := newLPXTestReconciler(t, nil, dgd, source) t.Log("Complete an empty preflight fence before foreign requests occupy the desired names") - currents, retiring, err := reconciler.reconcileLPXAttemptFence(ctx, dgd, desired) + currents, retiring, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) require.NoError(t, err) require.Empty(t, currents) require.Nil(t, retiring) @@ -2240,7 +1402,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t require.NoError(t, err) require.Nil(t, classification) - t.Log("Observe exact-name collisions with one bounded authoritative list and no per-request Gets") + t.Log("Observe labeled exact-name collisions with one scoped authoritative list and no per-request Gets") reader, ok := reconciler.apiReader.(client.WithWatch) require.True(t, ok) requestGets := 0 @@ -2271,18 +1433,19 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t }, }) - currents, retiring, err = reconciler.reconcileLPXAttemptFence(ctx, dgd, desired) + currents, retiring, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) require.EqualError(t, err, fmt.Sprintf("LPX request name %q is occupied by a foreign owner", foreign.Name)) require.Nil(t, currents) require.Nil(t, retiring) require.Zero(t, requestGets, "exact-name collision detection must not issue one request Get per desired projection") require.Len(t, requestListCalls, 1) - require.Equal(t, int64(lpxLifecycleListPageSize), requestListCalls[0].Limit) + require.Zero(t, requestListCalls[0].Limit) + require.Empty(t, requestListCalls[0].Continue) require.Equal(t, dgd.Namespace, requestListCalls[0].Namespace) - require.True(t, requestListCalls[0].LabelSelector == nil || requestListCalls[0].LabelSelector.Empty()) + require.Equal(t, lpxOwnerUIDLabel+"="+string(dgd.UID), requestListCalls[0].LabelSelector.String()) t.Log("Refresh publication state and refuse the foreign requests created after the empty preflight") - classification, err = reconciler.reconcileSelectedLPX(ctx, dgd, desired) + classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) require.ErrorContains(t, err, "occupied by a foreign owner") require.Nil(t, classification) require.Len(t, requestListCalls, 2, "final publication must refresh the authoritative request snapshot") @@ -2305,28 +1468,8 @@ func requireLPXClosed(t *testing.T, classification lpxClassification) *lpxClosed return closed } -func newLPXNodeLocalBoundStatus(generation, revision int64, planDigest lpxv1alpha1.PlanDigest) *lpxv1alpha1.LPUPipelineRequestStatus { - // Construct a fresh exact-generation NodeLocal scheduler receipt. - return &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: lpxv1alpha1.RequestPhaseBound, - ObservedGeneration: ptr.To(generation), - LastPlanRevision: revision, - Committed: &lpxv1alpha1.Committed{ - Execution: lpxv1alpha1.CommittedExecution{ - AcceptedGeneration: ptr.To(generation), - NodeLocal: &lpxv1alpha1.NodeLocalExecution{}, - }, - Plan: lpxv1alpha1.CommittedPlan{ - PlannedFromGeneration: generation, - Revision: revision, - PlanDigest: planDigest, - Placement: lpxv1alpha1.PlanPlacement{ - ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, - NodeLocal: &lpxv1alpha1.NodeLocalPlacement{}, - }, - }, - }, - } +func newLPXNodeLocalBoundStatus(generation, _ int64, _ lpxv1alpha1.PlanDigest) *lpxv1alpha1.LPUPipelineRequestStatus { + return &lpxv1alpha1.LPUPipelineRequestStatus{Phase: lpxv1alpha1.RequestPhaseBound, ObservedGeneration: &generation} } func findLPXTestClique(t *testing.T, objects []client.Object, name string) *grovev1alpha1.PodClique { @@ -2351,17 +1494,6 @@ func findLPXTestScalingGroup(t *testing.T, objects []client.Object, name string) return nil } -func findLPXTestPodGang(t *testing.T, objects []client.Object, schedulerName string) *groveschedulerv1alpha1.PodGang { - t.Helper() - for _, object := range objects { - if gang, ok := object.(*groveschedulerv1alpha1.PodGang); ok && gang.Labels[grovecommon.LabelSchedulerName] == schedulerName { - return gang - } - } - t.Fatalf("PodGang for scheduler %q not found", schedulerName) - return nil -} - func findLPXTestPodCliqueSet(t *testing.T, objects []client.Object) *grovev1alpha1.PodCliqueSet { t.Helper() for _, object := range objects { @@ -2384,7 +1516,7 @@ func renderLPXTestPodCliqueSet( return rendered } -func newLPXTestDGD(t *testing.T, pipeline lpx.Pipeline) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, *lpx.ModelRegistry) { +func newLPXTestDGD(t *testing.T, pipeline lpx.Pipeline) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { t.Helper() compilationMode := manifestcapnpv2.CompilationMode_lpuOnly if pipeline == lpx.PipelineLPX { @@ -2405,7 +1537,6 @@ func newLPXTestDeployment(t *testing.T, source *nvidiacomv1beta1.DynamoGraphDepl ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: types.UID("lpx-" + string(source.UID)), Generation: 1, Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - Finalizers: []string{lpxGraphDeploymentFinalizer}, }, Spec: nvidiacomv1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, } @@ -2481,7 +1612,7 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D return source } -func newLPXSpecDecodeTestDGD(t *testing.T) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, *lpx.ModelRegistry) { +func newLPXSpecDecodeTestDGD(t *testing.T) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { t.Helper() source := newLPXSpecDecodeTestSource() root := t.TempDir() @@ -2508,7 +1639,7 @@ func newLPXTestRegistryWithPartitionsAndMode( buildID string, partitionIDs []int, compilationMode manifestcapnpv2.CompilationMode, -) *lpx.ModelRegistry { +) lpx.ModelRegistry { t.Helper() root := t.TempDir() writeLPXTestBuild(t, root, buildID, partitionIDs, compilationMode) @@ -2588,7 +1719,7 @@ func writeLPXTestBuild( func newLPXTestReconciler( t *testing.T, - registry lpxModelRegistry, + registry lpx.ModelRegistry, dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, objects ...client.Object, @@ -2605,11 +1736,6 @@ func newLPXTestReconciler( base := fake.NewClientBuilder(). WithScheme(scheme). WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences). - WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). - WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimRefIndex, lpxDRAClaimReferences(false)). - WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimTemplateRefIndex, lpxDRAClaimReferences(true)). - WithIndex(&resourcev1.ResourceClaim{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences). - WithIndex(&resourcev1.ResourceClaimTemplate{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences). WithStatusSubresource(&nvidiacomv1beta1.DynamoGraphDeployment{}, &nvidiacomv1alpha1.LPXGraphDeployment{}). WithObjects(seed...). Build() @@ -2696,27 +1822,12 @@ func lpxMaterializedObjects( group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" objects := []client.Object{pcs, group} - // Each Grove replica owns independent cliques and one gang per scheduler. + // Each Grove replica owns independent cliques. for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { - gangs := make(map[string]*groveschedulerv1alpha1.PodGang, 2) - for _, scheduler := range []string{corev1.DefaultSchedulerName, lpx.SchedulerName} { - name := fmt.Sprintf("selected-dgd-%d-%s", replicaIndex, scheduler) - gangs[scheduler] = &groveschedulerv1alpha1.PodGang{ - TypeMeta: metav1.TypeMeta{APIVersion: groveschedulerv1alpha1.SchemeGroupVersion.String(), Kind: "PodGang"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), - Labels: maps.Clone(pcs.Labels), Annotations: maps.Clone(pcs.Annotations), - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }, - } - gangs[scheduler].Labels[grovecommon.LabelPartOfKey] = pcs.Name - gangs[scheduler].Labels[grovecommon.LabelSchedulerName] = scheduler - } parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} for _, template := range pcs.Spec.Template.Cliques { rendered := template.DeepCopy() name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) - gang := gangs[rendered.Spec.PodSpec.SchedulerName] clique := &grovev1alpha1.PodClique{ TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, ObjectMeta: metav1.ObjectMeta{ @@ -2731,28 +1842,34 @@ func lpxMaterializedObjects( }, } clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name - clique.Labels[grovecommon.LabelPodGang] = gang.Name clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) for index, dependency := range clique.Spec.StartsAfter { clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) } objects = append(objects, clique) - - // Gang references, not fake Pods, witness each clique's rendered cardinality. - podGroup := groveschedulerv1alpha1.PodGroup{Name: name, MinReplicas: ptr.Deref(clique.Spec.MinAvailable, clique.Spec.Replicas)} - for podIndex := int32(0); podIndex < clique.Spec.Replicas; podIndex++ { - podGroup.PodReferences = append(podGroup.PodReferences, groveschedulerv1alpha1.NamespacedName{ - Namespace: pcs.Namespace, Name: fmt.Sprintf("%s-%d", name, podIndex), - }) - } - gang.Spec.PodGroups = append(gang.Spec.PodGroups, podGroup) } - objects = append(objects, gangs[corev1.DefaultSchedulerName], gangs[lpx.SchedulerName]) } return objects } +func withLPXTestReplicas(t *testing.T, desired *lpxMaterializing, replicas int32) *lpxMaterializing { + t.Helper() + require.NotNil(t, desired) + require.NotNil(t, desired.plan) + require.Positive(t, desired.plan.Replicas) + require.GreaterOrEqual(t, replicas, int32(0)) + require.LessOrEqual(t, replicas, desired.plan.Replicas) + require.Zero(t, len(desired.requests)%int(desired.plan.Replicas)) + requestsPerReplica := len(desired.requests) / int(desired.plan.Replicas) + next := *desired + next.requests = slices.Clone(desired.requests[:int(replicas)*requestsPerReplica]) + plan := *desired.plan + plan.Replicas = replicas + next.plan = &plan + return &next +} + func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.Client, objects ...client.Object) { t.Helper() for _, object := range objects { @@ -2760,6 +1877,53 @@ func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.C } } +func reconcileSelectedLPXForTest( + ctx context.Context, + reconciler *graphReconciler, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, +) (lpxClassification, error) { + podCliqueSetMissing := false + if desired.plan != nil { + pcs := &grovev1alpha1.PodCliqueSet{} + err := reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + podCliqueSetMissing = apierrors.IsNotFound(err) + } + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, deployment, desired, podCliqueSetMissing) + if classification != nil || err != nil { + return classification, err + } + return reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents) +} + +func publishSelectedLPXForTest( + t *testing.T, + ctx context.Context, + reconciler *graphReconciler, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, +) lpxClassification { + t.Helper() + for range len(desired.requests) + 2 { + classification, err := reconcileSelectedLPXForTest(ctx, reconciler, deployment, desired) + require.NoError(t, err) + transition, preparing := classification.(*lpxDeadlineTransition) + if !preparing { + return classification + } + if deployment.Status.Placement == nil { + deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{} + } + deployment.Status.Placement.LPXAttempt = transition.attempt + require.NoError(t, reconciler.Status().Update(ctx, deployment)) + } + t.Fatal("LPX publication did not converge") + return nil +} + func getLPXRequest(t *testing.T, ctx context.Context, kubeClient client.Reader, namespace, name string) *lpxv1alpha1.LPUPipelineRequest { t.Helper() request := &lpxv1alpha1.LPUPipelineRequest{} diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index 3388634c567c..aac2cc609f7d 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -32,14 +32,8 @@ const ( modelDownloadCheckTimeout = 30 * time.Second ) -type lpxModelRegistry interface { - BuildURL(id string) (*url.URL, error) - EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) - AcquireBuildSnapshot(ctx context.Context, id string) (*lpx.BuildSnapshot, error) -} - // newLPXModelRegistry constructs a registry from the non-nil operator configuration. -func newLPXModelRegistry(config *configv1alpha1.OperatorConfiguration) (*lpx.ModelRegistry, error) { +func newLPXModelRegistry(config *configv1alpha1.OperatorConfiguration) (lpx.ModelRegistry, error) { // Configure the optional Model Express client. var mxClient modelpb.ModelServiceClient var err error @@ -102,7 +96,7 @@ func (r *graphReconciler) reconcileModelDownloads( func ensureModelsDownloaded( ctx context.Context, dgd *v1beta1.DynamoGraphDeployment, - registry lpxModelRegistry, + registry lpx.ModelRegistry, ) ([]string, bool, error) { builds, err := collectBuilds(dgd, registry) if err != nil { @@ -115,7 +109,7 @@ func ensureModelsDownloaded( build := buildURL.String() buildCtx := ctx cancel := func() {} - if deadline, ok := ctx.Deadline(); ok { + if deadline, ok := ctx.Deadline(); ok && i < len(builds)-1 { perBuildTimeout := time.Until(deadline) / time.Duration(len(builds)-i) buildCtx, cancel = context.WithTimeout(ctx, perBuildTimeout) } @@ -135,7 +129,7 @@ func ensureModelsDownloaded( return downloaded, len(downloaded) == len(builds), errors.Join(downloadErrors...) } -func collectBuilds(dgd *v1beta1.DynamoGraphDeployment, registry lpxModelRegistry) ([]url.URL, error) { +func collectBuilds(dgd *v1beta1.DynamoGraphDeployment, registry lpx.ModelRegistry) ([]url.URL, error) { buildsByKey := make(map[string]url.URL) for _, component := range dgd.Spec.Components { diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 7d8acfecc140..5c3ef3ece676 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -53,7 +53,7 @@ func TestEnsureModelsDownloaded(t *testing.T) { tests := []struct { name string dgd *v1beta1.DynamoGraphDeployment - registry lpxModelRegistry + registry lpx.ModelRegistry wantReady bool wantErr string wantCalls []string @@ -322,7 +322,7 @@ func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { } type fakeModelDownloadRegistry struct { - *lpx.ModelRegistry + lpx.ModelRegistry ready map[string]bool err map[string]error calls []string diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index cefa4d5e7c92..f996a714988f 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -20,10 +20,8 @@ package lpx import ( "context" "fmt" - "maps" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -72,53 +70,6 @@ func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( return nil, rejected, nil } + // Grove accepts the desired spec before individual engine requests are reconciled. return selectedLPX, nil, nil } - -// retireChangedLPXRequests withdraws changed intent before Grove can consume it. -// Deleting requests may still need Grove to remove their Pods to finish cleanup. -func (r *graphReconciler) retireChangedLPXRequests(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, rosterChanged bool) (*lpxDeadlineTransition, error) { - requests, err := r.listOwnedLPXRequests(ctx, deployment) - if err != nil || len(requests) == 0 { - return nil, err - } - desiredByName := make(map[string]*lpxModelMaterializing, len(desired.requests)) - for index := range desired.requests { - request := &desired.requests[index] - desiredByName[request.requestName] = request - } - - // Cancel removed batch members before deletion can be mistaken for a timeout. - if attempt := currentLPXAttemptStatus(deployment); attempt != nil && attempt.ExceededAt == nil { - currents := lpxRequestsByName(requests) - maps.DeleteFunc(currents, func(name string, _ *lpxv1alpha1.LPUPipelineRequest) bool { return desiredByName[name] == nil }) - if len(currents) != len(requests) { - want := desiredLPXAttemptStatus(desired, deployment.Generation, attempt.PodCliqueSetUID) - if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, want, currents); transition != nil { - return transition, nil - } - } - } - - // Keep unchanged engines, including those surviving scale and native image rollouts. - for index := range requests { - request := &requests[index] - wanted := desiredByName[request.Name] - if !request.DeletionTimestamp.IsZero() || !rosterChanged && wanted != nil && - wanted.modelProjection.Digest().String() == request.Annotations[dynamolpx.WorkloadDigestAnnotation] { - continue - } - classification, err := r.retireLPXEngineRequest(ctx, deployment, request, "The engine's scheduler intent changed or was removed", true) - if err != nil { - return nil, err - } - if transition, recording := classification.(*lpxDeadlineTransition); recording { - if _, completed, _ := exactLPXAttemptState(currentLPXAttemptStatus(deployment), lpxRequestsByName(requests)); completed { - disarmedAt := metav1.Now() - transition.attempt.DisarmedAt = &disarmedAt - } - return transition, nil - } - } - return nil, nil -} diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index d66cf09aca61..71de28ddccec 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -83,7 +83,7 @@ func renderPodCliqueSet( return pcs, resources, nil } -// stampLPXIdentity propagates stable ownership, never source revision, into Pod templates. +// stampLPXIdentity propagates stable ownership labels and annotations, never source revision. func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { // Preflight validates the exact source controller owner before rendering resources. sourceOwner := metav1.GetControllerOf(deployment) @@ -99,7 +99,16 @@ func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha delete(*annotations, dynamolpx.DGDGenerationAnnotation) maps.Copy(*annotations, identity) } + stampOwnerLabel := func(object client.Object) { + labels := object.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels[lpxOwnerUIDLabel] = string(deployment.UID) + object.SetLabels(labels) + } stamp(&pcs.Annotations) + stampOwnerLabel(pcs) for _, clique := range pcs.Spec.Template.Cliques { stamp(&clique.Annotations) } @@ -110,5 +119,6 @@ func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha annotations := resource.GetAnnotations() stamp(&annotations) resource.SetAnnotations(annotations) + stampOwnerLabel(resource) } } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 4ef715589788..bb5f4c897d34 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -54,7 +54,7 @@ func newLPXRenderDeployment(t *testing.T, source *v1beta1.DynamoGraphDeployment) func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { t.Log("Resolve a real LPU workload including inherited scheduler metadata") - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -100,7 +100,7 @@ func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { for _, fixture := range []string{"node-local-v2-hybrid", "node-local-v3-hx-hybrid"} { t.Run(fixture, func(t *testing.T) { t.Log("Resolve a hybrid engine using a real ClusterTopologyBinding for KV transfer") - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + fixture + ".input.yaml") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + fixture + ".input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -178,7 +178,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Resolve the authored engine and its exact child without changing either input") fixture := test.name - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/" + fixture + ".input.yaml") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + fixture + ".input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -250,16 +250,12 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { t.Log("Project every model into the expected scheduler family, mode and physical Agent shape") require.Equal(t, lpx.BuildFamily(test.wantFamily), selected.BuildFamily()) - var cyborgRef *lpxv1alpha1.CyborgPodCliqueReference - if hybrid { - cyborgRef = &lpxv1alpha1.CyborgPodCliqueReference{Name: "cyborg", UID: "uid"} - } projections := selected.ModelProjections() require.Len(t, plan.Agents, len(projections)) projectionIndices := make(map[string]int, len(projections)) for index, projection := range projections { projectionIndices[plan.Agents[index].TemplateName] = index - request := projection.RequestSpec(source.Namespace, "agents", cyborgRef) + request := projection.RequestSpec(plan, "agents") require.Equal(t, lpxv1alpha1.ExecutionBackendNodeLocal, request.ExecutionBackend) require.NotNil(t, request.NodeLocal) require.Equal(t, test.wantFamily, request.TargetFamily) @@ -453,7 +449,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { func TestLPXSpecDecodeConductorTemplate(t *testing.T) { t.Log("Give the speculative components distinct authored Agent and conductor commands") registry := newTestDataModelRegistry(t, t.TempDir()) - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -521,7 +517,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { require.Equal(t, before, source) } -func newTestDataModelRegistry(t *testing.T, registryRoot string) *lpx.ModelRegistry { +func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegistry { t.Helper() v2Builds := map[string]testV2GraphManifestFixture{ @@ -812,7 +808,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { t.Log("Render manifest-derived, equivalent explicit, and overridden runtime settings") configs := make([]renderedConfig, 0, 3) for _, settingsJSON := range []string{"", `{"batch_size":1,"swa":{"chunked":false}}`, `{"sequence_length":65536}`} { - payload, err := os.ReadFile("../../dynamo/testdata/from_dgd_yaml/single_v2.input.yaml") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml") require.NoError(t, err) var deployment v1beta1.DynamoGraphDeployment @@ -995,7 +991,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { for _, name := range tests { t.Run(name, func(t *testing.T) { t.Log("Load the authored input before selection") - b, err := os.ReadFile("../../dynamo/testdata/" + name + ".input.yaml") + b, err := os.ReadFile("../../dynamo/lpx/testdata/" + name + ".input.yaml") require.NoError(t, err) var dynamoDeployment v1beta1.DynamoGraphDeployment @@ -1082,7 +1078,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { normalizedHash := lpx.LPUConfigMapHash(normalized) replacements = append(replacements, actualHash, normalizedHash, actualHash[:16], normalizedHash[:16]) } - goldenPath, err := filepath.Abs("../../dynamo/testdata/" + name + ".yaml") + goldenPath, err := filepath.Abs("../../dynamo/lpx/testdata/" + name + ".yaml") require.NoError(t, err) const header = "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0\n\n" @@ -1093,7 +1089,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *testing.T) { t.Log("Load the pre-scheduler V2 hybrid DGD shape without scheduler intent and with an authored runtime command") - payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) require.NoError(t, err) dgd := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, dgd)) @@ -1161,7 +1157,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Empty(t, plan.ConductorTemplate) require.Len(t, plan.Agents, 1) require.Equal(t, 6, plan.Agents[0].Replicas) - request := projection.RequestSpec(dgd.Namespace, "test", nil) + request := projection.RequestSpec(plan, "test") require.Len(t, request.Partitions, 3) for index, partition := range request.Partitions { require.Equal(t, int64(index), partition.CompilerPartitionID) @@ -1404,7 +1400,7 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. for _, mode := range modes { t.Run(mode.name, func(t *testing.T) { - payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", mode.file)) + payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", mode.file)) require.NoError(t, err) for _, intent := range intents { @@ -1547,7 +1543,7 @@ func getResource[T any](t *testing.T, resources []client.Object, name string) T func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { t.Log("Load an authored hybrid engine and override its independent leader template") - payload, err := os.ReadFile(filepath.Join("../../dynamo/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 92f8a3abe5a2..94117cea7dde 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -13,11 +13,8 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -32,7 +29,6 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" ) const lpxSourceOwnerIndex = "lpx.sourceController" @@ -46,43 +42,20 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { For(&nvidiacomv1alpha1.LPXGraphDeployment{}, builder.WithPredicates(lpxDeploymentPrimaryPredicate())). Named("lpxgraphdeployment"). Watches(&nvidiacomv1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(r.mapLPXSourceToRequests), builder.WithPredicates(lpxSourcePredicate())). - WithEventFilter(r.lpxControllerEventFilter()) - // Primary/source events and finalizer retries are sufficient while disabled. + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.runtimeConfig)) + // Primary and source events are sufficient while disabled. if r.unavailableReason() != "" { return ctrlBuilder.Complete(r) } // Active integration observes its workload and scheduling dependencies. - ctrlBuilder = ctrlBuilder.Owns(&corev1.ConfigMap{}). + return ctrlBuilder.Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). Owns(&grovev1alpha1.PodCliqueSet{}). Watches(&grovev1alpha1.PodClique{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxPodCliqueEventPredicates())). - Watches(&grovev1alpha1.PodCliqueScalingGroup{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxScalingGroupEventPredicates())) - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences); err != nil { - return fmt.Errorf("register LPX topology reference index: %w", err) - } - ctrlBuilder = r.addLPXWatches(ctrlBuilder) - if r.runtimeConfig.Gate.Enabled(features.DRA) { - // Index authored references at each hop without reading other cached objects. - for _, index := range []struct { - object client.Object - name string - values client.IndexerFunc - }{ - {&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimRefIndex, lpxDRAClaimReferences(false)}, - {&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxResourceClaimTemplateRefIndex, lpxDRAClaimReferences(true)}, - {&resourcev1.ResourceClaim{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences}, - {&resourcev1.ResourceClaimTemplate{}, lpxDeviceClassRefIndex, lpxDeviceClassReferences}, - } { - if err := mgr.GetFieldIndexer().IndexField(context.Background(), index.object, index.name, index.values); err != nil { - return fmt.Errorf("register LPX %T index %s: %w", index.object, index.name, err) - } - } - for _, dependency := range []client.Object{&resourcev1.ResourceClaim{}, &resourcev1.ResourceClaimTemplate{}, &resourcev1.DeviceClass{}} { - ctrlBuilder = ctrlBuilder.Watches(dependency, handler.EnqueueRequestsFromMapFunc(r.mapLPXDRADependencyToRequests), builder.WithPredicates(predicate.GenerationChangedPredicate{})) - } - } - return ctrlBuilder.Complete(r) + Watches(&grovev1alpha1.PodCliqueScalingGroup{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxScalingGroupEventPredicates())). + Watches(&lpxv1alpha1.LPUPipelineRequest{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests)). + Complete(r) } func lpxSourceOwnerReferences(obj client.Object) []string { @@ -109,58 +82,13 @@ func (r *graphReconciler) mapLPXSourceToRequests(ctx context.Context, obj client // lpxDeploymentPrimaryPredicate observes every field that grants publication authority. func lpxDeploymentPrimaryPredicate() predicate.Predicate { - ownerChanged := predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { - oldDeployment := e.ObjectOld.(*nvidiacomv1alpha1.LPXGraphDeployment) - newDeployment := e.ObjectNew.(*nvidiacomv1alpha1.LPXGraphDeployment) - return !apiequality.Semantic.DeepEqual(oldDeployment.OwnerReferences, newDeployment.OwnerReferences) - }} - return predicate.Or(commoncontroller.GenerationOrDeletionChangedPredicate(), predicate.AnnotationChangedPredicate{}, ownerChanged) -} - -func (r *graphReconciler) addLPXWatches(ctrlBuilder *builder.Builder) *builder.Builder { - ctrlBuilder = ctrlBuilder. - Owns(&lpxv1alpha1.LPUPipelineRequest{}). - Watches( - &grovev1alpha1.ClusterTopologyBinding{}, - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request { - return r.indexedLPXDependencyRequests(ctx, obj, lpxTopologyBindingRefIndex) - }), - builder.WithPredicates(predicate.GenerationChangedPredicate{}), - ). - Watches( - &groveschedulerv1alpha1.PodGang{}, - handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), - builder.WithPredicates(lpxPodGangPredicate()), - ) - return ctrlBuilder -} - -func (r *graphReconciler) lpxControllerEventFilter() predicate.Predicate { - // The topology mapper applies namespace filtering to each consuming DGD. - return predicate.NewPredicateFuncs(func(obj client.Object) bool { - if _, deviceClass := obj.(*resourcev1.DeviceClass); deviceClass { - return true - } - if _, topology := obj.(*grovev1alpha1.ClusterTopologyBinding); topology && obj.GetNamespace() == "" { - return true - } - return commoncontroller.NamespaceAllowed(r.Config, r.runtimeConfig, obj, obj.GetNamespace()) - }) -} - -// mapLPXDRADependencyToRequests receives non-nil claim, template and DeviceClass -// events from their registered informers, keeping LPX watches out of DGD. -func (r *graphReconciler) mapLPXDRADependencyToRequests(ctx context.Context, obj client.Object) []ctrl.Request { - switch dependency := obj.(type) { - case *resourcev1.ResourceClaim: - return r.indexedLPXDependencyRequests(ctx, dependency, lpxResourceClaimRefIndex) - case *resourcev1.ResourceClaimTemplate: - return r.indexedLPXDependencyRequests(ctx, dependency, lpxResourceClaimTemplateRefIndex) - case *resourcev1.DeviceClass: - return r.mapLPXDeviceClassToRequests(ctx, dependency) - default: - return nil - } + return predicate.Or( + commoncontroller.GenerationOrDeletionChangedPredicate(), + predicate.AnnotationChangedPredicate{}, + predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { + return !apiequality.Semantic.DeepEqual(e.ObjectOld.GetOwnerReferences(), e.ObjectNew.GetOwnerReferences()) + }}, + ) } // mapLPXChildToRequests receives non-nil, namespaced Grove informer objects. @@ -172,35 +100,6 @@ func mapLPXChildToRequests(_ context.Context, obj client.Object) []ctrl.Request return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: deploymentName}}} } -// lpxPodGangPredicate receives non-nil PodGangs from the registered informer. -func lpxPodGangPredicate() predicate.Predicate { - isSchedulerWitness := func(obj client.Object) bool { - if obj.GetAnnotations()[dynamolpx.DeploymentNameAnnotation] == "" { - return false - } - schedulerName := obj.GetLabels()[grovecommon.LabelSchedulerName] - return schedulerName == dynamolpx.SchedulerName || schedulerName == corev1.DefaultSchedulerName - } - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { return isSchedulerWitness(e.Object) }, - DeleteFunc: func(e event.DeleteEvent) bool { return isSchedulerWitness(e.Object) }, - UpdateFunc: func(e event.UpdateEvent) bool { - oldGang := e.ObjectOld.(*groveschedulerv1alpha1.PodGang) - newGang := e.ObjectNew.(*groveschedulerv1alpha1.PodGang) - if !isSchedulerWitness(oldGang) && !isSchedulerWitness(newGang) { - return false - } - return oldGang.UID != newGang.UID || - oldGang.Generation != newGang.Generation || - !maps.Equal(oldGang.Labels, newGang.Labels) || - !maps.Equal(oldGang.Annotations, newGang.Annotations) || - !apiequality.Semantic.DeepEqual(oldGang.OwnerReferences, newGang.OwnerReferences) || - !apiequality.Semantic.DeepEqual(oldGang.DeletionTimestamp, newGang.DeletionTimestamp) - }, - GenericFunc: func(event.GenericEvent) bool { return false }, - } -} - func lpxPodCliqueMaterializationEvent(obj client.Object) bool { clique := obj.(*grovev1alpha1.PodClique) role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 1643a0a93c2c..21817ca130d6 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -15,16 +15,11 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -93,136 +88,6 @@ func TestLPXSourceWatchMapsControllerOwnedMaterializations(t *testing.T) { require.Contains(t, logs.String(), `"sourceUID":"source-uid"`) } -func TestLPXTopologyBindingWatchMapsMatchingPoliciesWithinScope(t *testing.T) { - t.Log("Build matching, unrelated, and out-of-scope graph policies") - scheme := runtime.NewScheme() - require.NoError(t, nvidiacomv1alpha1.AddToScheme(scheme)) - require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) - matching := topologyPolicyDGD("matching", "workloads", "fabric") - unrelated := topologyPolicyDGD("unrelated", "workloads", "other-fabric") - foreign := topologyPolicyDGD("foreign", "other", "fabric") - ordinary := topologyPolicyDGD("ordinary", "workloads", "fabric") - for _, source := range []*nvidiacomv1beta1.DynamoGraphDeployment{matching, unrelated, foreign} { - source.Spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX}} - } - - t.Log("Give each source a differently named materialization and retain a stale same-name child") - objects := []client.Object{matching, unrelated, foreign, ordinary} - for _, source := range []*nvidiacomv1beta1.DynamoGraphDeployment{matching, unrelated, foreign, ordinary} { - source.UID = types.UID(source.Name + "-uid") - objects = append(objects, &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ - Name: "materialization-" + source.Name, Namespace: source.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - }}) - } - stale := matching.DeepCopy() - stale.UID = "previous-source-uid" - objects = append(objects, &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ - Name: matching.Name, Namespace: matching.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(stale, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - }}) - - t.Log("Index dependency references and exact controller ownership without scanning") - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). - WithIndex(&nvidiacomv1beta1.DynamoGraphDeployment{}, lpxTopologyBindingRefIndex, lpxTopologyBindingReferences). - WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences). - WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - options := (&client.ListOptions{}).ApplyOptions(opts) - require.NotNil(t, options.FieldSelector, "topology events must use an index, not scan DGDs") - return delegated.List(ctx, list, opts...) - }}).Build() - reconciler := &graphReconciler{ - Client: kube, - runtimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{LPX: true}}, - Config: &configv1alpha1.OperatorConfiguration{ - Namespace: configv1alpha1.NamespaceConfiguration{Restricted: "workloads"}, - LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - }, - } - - t.Log("Accept namespaced graph events only in the restricted namespace") - filter := reconciler.lpxControllerEventFilter() - require.True(t, filter.Create(event.CreateEvent{Object: matching})) - require.False(t, filter.Create(event.CreateEvent{Object: foreign})) - - t.Log("Admit cluster-scoped topology bindings for namespace-aware mapping") - binding := &grovev1alpha1.ClusterTopologyBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "fabric"}, - } - require.True(t, filter.Create(event.CreateEvent{Object: binding})) - require.False(t, filter.Create(event.CreateEvent{Object: &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "foreign-cluster-object"}, - }})) - - t.Log("Requeue only the allowed materialization owned by the current matching source UID") - requests := reconciler.indexedLPXDependencyRequests(context.Background(), binding, lpxTopologyBindingRefIndex) - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization-matching"}}}, requests) - - t.Log("Ignore bindings no graph consumes") - require.Empty(t, reconciler.indexedLPXDependencyRequests(context.Background(), &grovev1alpha1.ClusterTopologyBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "unused"}, - }, lpxTopologyBindingRefIndex)) - - t.Log("Editing a source policy removes the old dependency and indexes the new one") - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(matching), matching)) - matching.Spec.Experimental.KvTransferPolicy.ClusterTopologyName = "new-fabric" - require.NoError(t, kube.Update(t.Context(), matching)) - require.Empty(t, reconciler.indexedLPXDependencyRequests(t.Context(), &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "fabric"}}, lpxTopologyBindingRefIndex)) - require.Len(t, reconciler.indexedLPXDependencyRequests(t.Context(), &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "new-fabric"}}, lpxTopologyBindingRefIndex), 1) -} - -func topologyPolicyDGD(name, namespace, bindingName string) *nvidiacomv1beta1.DynamoGraphDeployment { - return &nvidiacomv1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ - Experimental: &nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec{ - KvTransferPolicy: &nvidiacomv1beta1.KvTransferPolicy{ClusterTopologyName: bindingName}, - }, - }, - } -} - -func TestLPXPodGangPredicateTracksOnlyMaterializationIdentity(t *testing.T) { - t.Log("Accept scheduler-scoped publication witnesses and map their exact namespaced graph") - predicate := lpxPodGangPredicate() - gang := &groveschedulerv1alpha1.PodGang{ - ObjectMeta: metav1.ObjectMeta{ - Name: "agents", Namespace: "workloads", UID: "uid", Generation: 1, - Annotations: map[string]string{dynamolpx.DeploymentNameAnnotation: "materialization"}, - Labels: map[string]string{ - grovecommon.LabelSchedulerName: dynamolpx.SchedulerName, - consts.KubeLabelDynamoGraphDeploymentName: "graph", - }, - }, - } - require.True(t, predicate.Create(event.CreateEvent{Object: gang})) - require.True(t, predicate.Delete(event.DeleteEvent{Object: gang})) - require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization"}}}, - mapLPXChildToRequests(t.Context(), gang)) - - t.Log("Ignore status-only scheduler updates") - statusOnly := gang.DeepCopy() - statusOnly.Status.Phase = groveschedulerv1alpha1.PodGangPhase("scheduled") - require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: statusOnly})) - - t.Log("Wake on PodGang spec and mirrored workload-annotation convergence") - specChange := gang.DeepCopy() - specChange.Generation++ - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: specChange})) - annotationChange := gang.DeepCopy() - annotationChange.Annotations = map[string]string{dynamolpx.WorkloadDigestAnnotation: "sha256:updated"} - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: gang, ObjectNew: annotationChange})) - - t.Log("Accept the ordinary scheduler witness and reject unrelated PodGangs") - ordinary := gang.DeepCopy() - ordinary.Labels[grovecommon.LabelSchedulerName] = corev1.DefaultSchedulerName - require.True(t, predicate.Create(event.CreateEvent{Object: ordinary})) - unrelated := ordinary.DeepCopy() - delete(unrelated.Annotations, dynamolpx.DeploymentNameAnnotation) - require.False(t, predicate.Create(event.CreateEvent{Object: unrelated})) - require.Empty(t, mapLPXChildToRequests(t.Context(), unrelated)) -} - func TestLPXWorkloadEventPredicates(t *testing.T) { for _, test := range []struct { name string @@ -367,115 +232,3 @@ func BenchmarkLPXSourceWatchStatusChurn(b *testing.B) { } } } - -func TestLPXDRAWatchMapsIndexedConsumers(t *testing.T) { - t.Log("Index consumed regular and init-container references, deduplicating shared aliases") - source := newLPXTestSource(dynamolpx.PipelineLPX, "build-v2") - child := newLPXTestDeployment(t, source) - child.Name = "materialization" - pod := &dynamolpx.ServingComponent(source).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec - pod.Containers[0].Resources.Claims = []corev1.ResourceClaim{{Name: "direct"}, {Name: "template"}} - pod.Containers = append(pod.Containers, corev1.Container{Name: "sidecar", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "direct"}}}}) - pod.InitContainers = []corev1.Container{ - {Name: "prepare", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "init"}}}}, - {Name: "native-sidecar", RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "init-template"}}}}, - } - pod.ResourceClaims = []corev1.PodResourceClaim{ - {Name: "direct", ResourceClaimName: ptr.To("shared")}, - {Name: "template", ResourceClaimTemplateName: ptr.To("shared")}, - {Name: "init", ResourceClaimName: ptr.To("init-claim")}, - {Name: "init-template", ResourceClaimTemplateName: ptr.To("init-template")}, - {Name: "unused", ResourceClaimName: ptr.To("unused-claim"), ResourceClaimTemplateName: ptr.To("unused-template")}, - } - require.ElementsMatch(t, []string{"shared", "init-claim"}, lpxDRAClaimReferences(false)(source)) - require.ElementsMatch(t, []string{"shared", "init-template"}, lpxDRAClaimReferences(true)(source)) - - t.Log("Reference same-named claims and templates from two namespaces, excluding unused aliases") - foreign := source.DeepCopy() - foreign.Name, foreign.Namespace = "foreign", "other-namespace" - foreign.UID = "foreign-source-uid" - unused := source.DeepCopy() - unused.Name = "unused" - unused.UID = "unused-source-uid" - unusedPod := &dynamolpx.ServingComponent(unused).ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate.Spec - unusedPod.Containers = unusedPod.Containers[:1] - unusedPod.Containers[0].Resources.Claims, unusedPod.InitContainers = nil, nil - claim := &resourcev1.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: source.Namespace}, - Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ - Name: "gpu", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: "gpu"}, - }}}}, - } - claimTemplate := &resourcev1.ResourceClaimTemplate{ - ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: source.Namespace}, - Spec: resourcev1.ResourceClaimTemplateSpec{Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ - Name: "gpu", FirstAvailable: []resourcev1.DeviceSubRequest{{Name: "first", DeviceClassName: "gpu"}, {Name: "second", DeviceClassName: "alternative-gpu"}}, - }}}}}, - } - foreignTemplate := claimTemplate.DeepCopy() - foreignTemplate.Namespace = foreign.Namespace - - t.Log("Route dependencies to current materializations, not same-name children from old source UIDs") - foreignChild := newLPXTestDeployment(t, foreign) - foreignChild.Name = "foreign-materialization" - unusedChild := newLPXTestDeployment(t, unused) - unusedChild.Name = "unused-materialization" - staleChild := child.DeepCopy() - staleChild.Name, staleChild.UID = source.Name, "stale-materialization-uid" - staleChild.OwnerReferences[0].UID = "previous-source-uid" - r := newLPXTestReconciler(t, nil, child, source, foreign, unused, foreignChild, unusedChild, staleChild, claim, claimTemplate, foreignTemplate) - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - options := (&client.ListOptions{}).ApplyOptions(opts) - require.NotNil(t, options.FieldSelector, "DRA events must use indexes at both hops") - require.False(t, options.FieldSelector.Empty()) - return delegated.List(ctx, list, opts...) - }}) - childRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - foreignRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(foreignChild)} - - t.Log("Claim events remain namespace-local and unused aliases never enqueue a graph") - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claim)) - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) - require.Equal(t, []ctrl.Request{foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), foreignTemplate)) - require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.ResourceClaimTemplate{ObjectMeta: metav1.ObjectMeta{Namespace: source.Namespace, Name: "unrelated"}})) - require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "unrelated"}})) - - t.Log("Class events join exact and alternative references and enqueue each graph once") - for _, class := range []string{"gpu", "alternative-gpu"} { - require.ElementsMatch(t, []ctrl.Request{childRequest, foreignRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) - } - r.Config.Namespace.Restricted = "unwatched-namespace" - require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) - r.Config.Namespace.Restricted = source.Namespace - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) - - t.Log("Editing only a direct claim updates its DeviceClass dependency") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claim), claim)) - claim.Spec.Devices.Requests[0].Exactly.DeviceClassName = "replacement-claim-gpu" - require.NoError(t, r.Update(t.Context(), claim)) - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-claim-gpu"}})) - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu"}})) - - t.Log("A template-only edit removes its old class references without a DGD edit") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(claimTemplate), claimTemplate)) - claimTemplate.Spec.Spec.Devices.Requests[0].FirstAvailable = []resourcev1.DeviceSubRequest{{Name: "replacement", DeviceClassName: "replacement-template-gpu"}} - require.NoError(t, r.Update(t.Context(), claimTemplate)) - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) - for _, class := range []string{"gpu", "alternative-gpu"} { - require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: class}})) - } - - t.Log("Missing templates remain indexed, but no longer contribute a DeviceClass dependency") - require.NoError(t, r.Delete(t.Context(), claimTemplate)) - require.Equal(t, []ctrl.Request{childRequest}, r.mapLPXDRADependencyToRequests(t.Context(), claimTemplate)) - require.Empty(t, r.mapLPXDRADependencyToRequests(t.Context(), &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "replacement-template-gpu"}})) - - t.Log("Dependency events never mutate the source or its revision; ordinary components are not indexed") - stored := &nvidiacomv1beta1.DynamoGraphDeployment{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), stored)) - require.Equal(t, source.Spec, stored.Spec) - require.Equal(t, source.Generation, stored.Generation) - source.Spec.Components[0].ComponentType = nvidiacomv1beta1.ComponentTypeDecode - require.Empty(t, lpxDRAClaimReferences(false)(source)) - require.Empty(t, lpxDRAClaimReferences(true)(source)) -} diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index cea496aec603..f824755479e6 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -51,6 +51,14 @@ func RenderLPXBasePodCliqueSet( return nil, nil, err } + // Publish the validated queue for the LPX backend's KAI fallback. + if queue != "" { + if pcs.Labels == nil { + pcs.Labels = make(map[string]string) + } + pcs.Labels[commonconsts.KubeLabelKaiSchedulerQueue] = queue + } + // Cyborg uses ordinary worker KV-transfer defaults, including the binding's domains. var topologyDomains []v1beta1.TopologyDomain if dynamoDeployment.Spec.Experimental != nil { diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index ffb7f2a84c1c..2be5765852bd 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -364,7 +364,7 @@ func TestLPXInputRevisionTracksIndirectRenderMetadata(t *testing.T) { for _, change := range []string{"epp-presence", "alpha-label", "alpha-annotation", "alpha-subtype"} { t.Run(change, func(t *testing.T) { t.Log("Capture the GPU role metadata derived from the complete source graph") - payload, err := os.ReadFile("testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") + payload, err := os.ReadFile("lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) @@ -428,7 +428,7 @@ func TestLPXRestartTokenPreservesDeliveredTokenOnCancellation(t *testing.T) { func TestLPXInputRevisionIgnoresUnrelatedConvertedMetadata(t *testing.T) { t.Log("Keep EPP discovery enabled while editing only its independent workload") - payload, err := os.ReadFile("testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") + payload, err := os.ReadFile("lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml") require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot.go b/deploy/operator/internal/dynamo/lpx/build_snapshot.go index 5b4440adcf79..f2155a337d37 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot.go @@ -52,7 +52,7 @@ type BuildSnapshot struct { // AcquireBuildSnapshot fences the required manifest-v2 compiler metadata with inventories // and duplicate reads. The receiver must be non-nil and is not mutated. -func (r *ModelRegistry) AcquireBuildSnapshot(ctx context.Context, id string) (*BuildSnapshot, error) { +func (r *defaultModelRegistry) AcquireBuildSnapshot(ctx context.Context, id string) (*BuildSnapshot, error) { // Bound all metadata RPCs for one snapshot acquisition. ctx, cancel := context.WithTimeout(ctx, buildSnapshotTimeout) defer cancel() diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 124aa2cd5dc2..2c683cdde0d3 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -49,8 +49,6 @@ func configureHybridCyborg( return err } - cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" - delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) cyborg.Annotations = roleAnnotations( cyborg.Annotations, lpxv1alpha1.PodRoleCyborgWorker, diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 9e761fb01ea5..8fe1ee0f6b8d 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -128,14 +128,12 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineLPX, `{"prop_sync":false}`) projection.stage = testRenderComponentName - t.Log("Keep every physical scheduler partition and Agent, including chain followers") - require.Len(t, projection.RequestSpec("test", "agents", nil).Partitions, test.partitions) - require.Equal(t, 2*test.partitions, projection.agentReplicas) - t.Log("Render only projected endpoints without compressing their physical Agent offsets") workload := &SelectedWorkload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1} plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) + require.Len(t, projection.RequestSpec(plan, "agents").Partitions, test.partitions) + require.Equal(t, 2*test.partitions, projection.agentReplicas) configMap, err := workload.RenderCyborgConfigMap("test", plan, renderTestPodSpec()) require.NoError(t, err) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" diff --git a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go index 7d62cdd6f5c0..fd71fbbc7640 100644 --- a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go @@ -22,7 +22,7 @@ const registryTestTopology = "URSA_V2_1__Q8__8C__G_106__KP_FEC__GHZ_1_0__NO_FPGA const gbuildManifestJSONFile = "manifest.json" -func normalizeRegistryFixtureBuild(ctx context.Context, registry *ModelRegistry, id string) (*Build, error) { +func normalizeRegistryFixtureBuild(ctx context.Context, registry ModelRegistry, id string) (*Build, error) { snapshot, err := registry.AcquireBuildSnapshot(ctx, id) if err != nil { return nil, err diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index a584b37c176f..6a6a7dc3d32d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -55,13 +55,14 @@ type ModelProjectionInput struct { // derived for one model. Completed component data is shared read-only; only the // resolver assigns the stage before publishing the immutable projections. type ModelProjection struct { - digest WorkloadDigest - runtimeBuildRef string - model string - stage string - pipeline Pipeline - configuredBuild Build - allocationMetadata json.RawMessage + digest WorkloadDigest + compilerSnapshotDigest string + runtimeBuildRef string + model string + stage string + pipeline Pipeline + configuredBuild Build + allocationMetadata json.RawMessage // partitions retains immutable physical build evidence before runtime collapse. partitions []BuildPartition connectors []lpxv1alpha1.PropSyncConnectorRequest @@ -73,19 +74,21 @@ func (p *ModelProjection) Digest() WorkloadDigest { return p.digest } +// CompilerSnapshotDigest returns the immutable compiler snapshot identity. +func (p *ModelProjection) CompilerSnapshotDigest() string { + return p.compilerSnapshotDigest +} + // Model returns the logical model identity. The receiver must be non-nil. func (p *ModelProjection) Model() string { return p.model } -// RequestSpec returns a fresh node-local LPR spec. The materialization owner -// supplies only workload references; scheduler-owned placement fields do not -// exist in this request arm. The receiver must be non-nil. A nil cyborgRef is -// supported and omits the Cyborg reference. Neither input is mutated. +// RequestSpec returns a fresh node-local request for one Grove scaling-group +// replica. The receiver and plan must be non-nil; neither input is mutated. func (p *ModelProjection) RequestSpec( - namespace string, - podGangName string, - cyborgRef *lpxv1alpha1.CyborgPodCliqueReference, + plan *MaterializationPlan, + agentPodCliqueName string, ) lpxv1alpha1.LPUPipelineRequestSpec { connectors := make([]lpxv1alpha1.PropSyncConnectorRequest, len(p.connectors)) for i := range p.connectors { @@ -114,22 +117,28 @@ func (p *ModelProjection) RequestSpec( PartitionID: partitionID, } } - ns := namespace spec := lpxv1alpha1.LPUPipelineRequestSpec{ AllocationMetadata: runtime.RawExtension{Raw: slices.Clone(p.allocationMetadata)}, - PodGangRef: lpxv1alpha1.NamespacedName{Name: podGangName, Namespace: &ns}, + MaterializationTarget: lpxv1alpha1.MaterializationTarget{ + PodCliqueSetReplicaIndex: 0, + PodCliqueScalingGroupRef: &lpxv1alpha1.PodCliqueScalingGroupReference{ + Name: plan.LPXScalingGroup, ReplicaIndex: int64(plan.ReplicaIndex), + }, + }, + RepairPolicy: &lpxv1alpha1.RepairPolicy{Mode: lpxv1alpha1.RepairPolicyModeSamePlacement}, PropSyncConnectors: connectors, TargetFamily: lpxv1alpha1.TargetFamily(p.configuredBuild.Family), WorkloadMode: p.schedulerWorkloadMode(), ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, NodeLocal: &lpxv1alpha1.NodeLocalRequest{ + AgentPodCliqueRef: lpxv1alpha1.PodCliqueReference{Name: agentPodCliqueName}, Model: p.model, PartitionMappings: mappings, }, Partitions: partitions, } - if cyborgRef != nil { - spec.CyborgPodCliqueRef = cyborgRef.DeepCopy() + if plan.CyborgClique != "" { + spec.CyborgPodCliqueRef = &lpxv1alpha1.PodCliqueReference{Name: plan.CyborgClique} } return spec } diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go index ad64b2118d58..0b72b79cc53f 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -31,16 +31,22 @@ const ( BuildSchemeGCS = "gs" ) -// ModelRegistry resolves build references and acquires their compiler metadata. -type ModelRegistry struct { +type defaultModelRegistry struct { registryURL *url.URL mxClient modelpb.ModelServiceClient } +// ModelRegistry resolves build references and acquires their compiler metadata. +type ModelRegistry interface { + BuildURL(id string) (*url.URL, error) + EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) + AcquireBuildSnapshot(ctx context.Context, id string) (*BuildSnapshot, error) +} + // NewModelRegistry constructs a registry rooted at modelRegistryURL. An empty // URL is supported for absolute build references. mxClient may be nil when GCS // downloads are not required. -func NewModelRegistry(modelRegistryURL string, mxClient modelpb.ModelServiceClient) (*ModelRegistry, error) { +func NewModelRegistry(modelRegistryURL string, mxClient modelpb.ModelServiceClient) (ModelRegistry, error) { var registryURL *url.URL if modelRegistryURL != "" { @@ -56,7 +62,7 @@ func NewModelRegistry(modelRegistryURL string, mxClient modelpb.ModelServiceClie } } - return &ModelRegistry{ + return &defaultModelRegistry{ registryURL: registryURL, mxClient: mxClient, }, nil @@ -65,7 +71,7 @@ func NewModelRegistry(modelRegistryURL string, mxClient modelpb.ModelServiceClie // EnsureDownloaded ensures buildURL is locally available and reports whether // the download is complete. For GCS builds it may initiate or advance a // ModelExpress download. The receiver must be non-nil and is not mutated. -func (r *ModelRegistry) EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) { +func (r *defaultModelRegistry) EnsureDownloaded(ctx context.Context, buildURL url.URL) (bool, error) { switch buildURL.Scheme { case BuildSchemeFile: return true, nil @@ -117,7 +123,7 @@ func (r *ModelRegistry) EnsureDownloaded(ctx context.Context, buildURL url.URL) // BuildURL resolves id against the configured registry URL. The receiver must // be non-nil and is not mutated. -func (r *ModelRegistry) BuildURL(id string) (*url.URL, error) { +func (r *defaultModelRegistry) BuildURL(id string) (*url.URL, error) { refURL, err := parseBuildRef(id) if err == nil { return refURL, nil @@ -184,7 +190,7 @@ func parseBuildRef(ref string) (*url.URL, error) { // readBuildFileBounded reads one relative file. The receiver and buildURL must // be non-nil, and maxBytes must be non-negative. -func (r *ModelRegistry) readBuildFileBounded( +func (r *defaultModelRegistry) readBuildFileBounded( ctx context.Context, buildURL *url.URL, relativePath string, @@ -309,7 +315,7 @@ func (r *ModelRegistry) readBuildFileBounded( } } -func (r *ModelRegistry) listBuildFiles(ctx context.Context, buildURL *url.URL) ([]string, error) { +func (r *defaultModelRegistry) listBuildFiles(ctx context.Context, buildURL *url.URL) ([]string, error) { var ( paths []string err error diff --git a/deploy/operator/internal/dynamo/lpx/model_registry_test.go b/deploy/operator/internal/dynamo/lpx/model_registry_test.go index 9dfb8ba8e30b..3b3967baf391 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry_test.go @@ -130,7 +130,7 @@ func TestAcquireLocalBuildSnapshotCancellation(t *testing.T) { } t.Log("Reject the acquisition without returning a snapshot") - registry := &ModelRegistry{} + registry := &defaultModelRegistry{} snapshot, err := registry.AcquireBuildSnapshot(ctx, buildDir) require.ErrorIs(t, err, tt.wantErr) require.Nil(t, snapshot) @@ -146,7 +146,7 @@ func TestAcquireLocalBuildSnapshotRejectsFIFOManifest(t *testing.T) { t.Log("Reject the FIFO manifest while the acquisition context is still active") ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() - snapshot, err := (&ModelRegistry{}).AcquireBuildSnapshot(ctx, buildDir) + snapshot, err := (&defaultModelRegistry{}).AcquireBuildSnapshot(ctx, buildDir) require.ErrorContains(t, err, "not a regular file") require.ErrorContains(t, err, gbuildManifestV2CapnpFile) require.Nil(t, snapshot) @@ -305,7 +305,7 @@ func TestReadBuildFileBoundsLocalFileAtAcquisition(t *testing.T) { const metadataFile = "metadata.bin" require.NoError(t, os.WriteFile(filepath.Join(buildDir, metadataFile), []byte("12345"), 0o600)) buildURL := &url.URL{Scheme: BuildSchemeFile, Path: buildDir} - registry := &ModelRegistry{} + registry := &defaultModelRegistry{} t.Log("Read the file at the inclusive acquisition limit") data, err := registry.readBuildFileBounded(t.Context(), buildURL, metadataFile, 5) @@ -334,7 +334,7 @@ func TestReadBuildFileBoundsGCSAccumulatedChunks(t *testing.T) { {chunks: modelFileChunks(metadataFile, "12", "345")}, }, } - registry := &ModelRegistry{mxClient: client} + registry := &defaultModelRegistry{mxClient: client} buildURL := &url.URL{Scheme: BuildSchemeGCS, Host: "bucket", Path: "/model/build"} t.Log("Reject the accumulated stream before accepting oversized metadata") @@ -509,7 +509,7 @@ func TestRegistryEnsureDownloadedReturnsFirstStatus(t *testing.T) { }, }, } - registry := &ModelRegistry{mxClient: client} + registry := &defaultModelRegistry{mxClient: client} buildURL := mustParseURL(t, "gs://bucket/models/build") t.Log("Classify the first Model Express download status") @@ -544,7 +544,7 @@ func TestRegistryEnsureDownloadedPropagatesTransportFailures(t *testing.T) { if scenario.rpc { client.err = scenario.err } - registry := &ModelRegistry{mxClient: client} + registry := &defaultModelRegistry{mxClient: client} t.Log("Reject the failed or empty stream before classifying readiness") ready, err := registry.EnsureDownloaded(t.Context(), mustParseURL(t, "gs://bucket/models/build")) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 29cea99c0813..ff2a5494b9f6 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -176,6 +176,7 @@ func RenderSelectedNodeLocal( annotations := roleAnnotations(template.Annotations, lpxv1alpha1.PodRoleAgent, projection.Digest().String()) annotations[commonconsts.AnnotationExtraResourcesHash] = configHash annotations[lpxv1alpha1.PodModelAnnotation] = projection.model + annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation] = projection.CompilerSnapshotDigest() annotations[WorkloadModeAnnotation] = string(projection.schedulerWorkloadMode()) agent := plan.Agents[index] replicas := int32(agent.Replicas) @@ -239,6 +240,11 @@ func RenderSelectedNodeLocal( MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), }} + // Select Grove's LPX backend for the complete engine, including its KAI fallback. + for _, clique := range pcs.Spec.Template.Cliques { + clique.Spec.PodSpec.SchedulerName = SchedulerName + } + explicit := grovev1alpha1.CliqueStartupTypeExplicit pcs.Spec.Template.StartupType = &explicit return extraResources, nil @@ -287,6 +293,7 @@ func roleAnnotations( for _, key := range []string{ ExecutionBackendAnnotation, WorkloadModeAnnotation, + lpxv1alpha1.CompilerSnapshotDigestAnnotation, lpxv1alpha1.PodModelAnnotation, lpxv1alpha1.PodPartitionIDAnnotation, lpxv1alpha1.PodRankInPartitionAnnotation, diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 2e85f6bc852a..9945b0be860b 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -84,18 +84,23 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Seed conflicting runtime annotations and a nondefault mount") projection := projectRenderFixture(t, test.family, test.pipeline, test.snapshot) + require.NotEqual(t, projection.Digest().String(), projection.CompilerSnapshotDigest()) pcs := renderTestPCS(test.pipeline == PipelineLPX) pcs.Annotations = map[string]string{ ExecutionBackendAnnotation: "stale", } for _, clique := range pcs.Spec.Template.Cliques { - clique.Annotations = map[string]string{ExecutionBackendAnnotation: "stale"} + clique.Annotations = map[string]string{ + ExecutionBackendAnnotation: "stale", + lpxv1alpha1.CompilerSnapshotDigestAnnotation: "stale", + } } template := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ "user": "kept", lpxv1alpha1.PodModelAnnotation: "spoofed", - ExecutionBackendAnnotation: "stale", + lpxv1alpha1.CompilerSnapshotDigestAnnotation: "stale", + ExecutionBackendAnnotation: "stale", }}, Spec: renderTestPodSpec(), } @@ -118,10 +123,16 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { require.NotContains(t, rendered.Annotations, ExecutionBackendAnnotation) for _, clique := range rendered.Spec.Template.Cliques { require.NotContains(t, clique.Annotations, ExecutionBackendAnnotation) + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { + require.Equal(t, projection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) + } else { + require.NotContains(t, clique.Annotations, lpxv1alpha1.CompilerSnapshotDigestAnnotation) + } } agent := namedClique(t, rendered, testAgentTemplateName) require.Equal(t, projection.Model(), agent.Annotations[lpxv1alpha1.PodModelAnnotation]) require.Equal(t, projection.Digest().String(), agent.Annotations[WorkloadDigestAnnotation]) + require.Equal(t, projection.CompilerSnapshotDigest(), agent.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) t.Log("Resolve runtime mounts using the authored storage path") if test.family == lpxv1alpha1.TargetFamilyXt8888 { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index ba2b4182d22a..5036e84f151a 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -37,7 +37,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.NoError(t, err) require.Equal(t, PipelineSingle, hx.Pipeline()) require.Equal(t, BuildFamilyHX, hx.BuildFamily()) - require.Equal(t, lpxv1alpha1.WorkloadModeV3HxLPUOnly, hx.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Equal(t, lpxv1alpha1.WorkloadModeV3HxLPUOnly, hx.modelProjections[0].RequestSpec(&MaterializationPlan{}, "agents").WorkloadMode) require.Equal(t, "LPX", hx.LPXComponentName()) plan, err := hx.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) @@ -79,7 +79,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.NoError(t, err) require.Equal(t, PipelineLPX, xt.Pipeline()) require.Equal(t, BuildFamilyXT, xt.BuildFamily()) - require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, xt.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, xt.modelProjections[0].RequestSpec(&MaterializationPlan{}, "agents").WorkloadMode) require.Len(t, xt.modelProjections[0].configuredBuild.Partitions, 2) plan, err = xt.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) @@ -158,7 +158,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { require.NoError(t, err) require.Equal(t, before, dgd, "canonical ordering must not rewrite the authored target-first list") require.Equal(t, test.family, selected.BuildFamily()) - require.Equal(t, test.wantMode, selected.modelProjections[0].RequestSpec("test", "agents", nil).WorkloadMode) + require.Equal(t, test.wantMode, selected.modelProjections[0].RequestSpec(&MaterializationPlan{}, "agents").WorkloadMode) require.Equal(t, PipelineSpecDecode, selected.Pipeline()) require.Equal(t, "lpx", selected.LPXComponentName()) diff --git a/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go index 0b3988e19a29..9ba60eda68b5 100644 --- a/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go +++ b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go @@ -21,14 +21,15 @@ const ( var GroupVersion = schema.GroupVersion{Group: APIGroup, Version: APIVersion} const ( - PodRoleAnnotation = "scheduling.lpu.nvidia.com/role" - PodModelAnnotation = "scheduling.lpu.nvidia.com/model" - PodPartitionIDAnnotation = "scheduling.lpu.nvidia.com/partition-id" - PodRankInPartitionAnnotation = "scheduling.lpu.nvidia.com/rank-in-partition" - PodRoleAgent = "agent" - PodRoleConductor = "conductor" - PodRoleCyborgWorker = "cyborg-worker" - MaxModelAnnotationBytes = 261054 + PodRoleAnnotation = "scheduling.lpu.nvidia.com/role" + PodModelAnnotation = "scheduling.lpu.nvidia.com/model" + CompilerSnapshotDigestAnnotation = "scheduling.lpu.nvidia.com/compiler-snapshot-digest" + PodPartitionIDAnnotation = "scheduling.lpu.nvidia.com/partition-id" + PodRankInPartitionAnnotation = "scheduling.lpu.nvidia.com/rank-in-partition" + PodRoleAgent = "agent" + PodRoleConductor = "conductor" + PodRoleCyborgWorker = "cyborg-worker" + MaxModelAnnotationBytes = 261054 ) func ParsePodRole(value string) (string, error) { @@ -367,9 +368,10 @@ const ( ) type ConsumerBinding struct { - ConsumerID string `json:"consumerId"` - Current *CurrentBinding `json:"current,omitempty"` - Target *BindingTarget `json:"target,omitempty"` + ConsumerID string `json:"consumerId"` + Current *CurrentBinding `json:"current,omitempty"` + PodClique PodCliqueIncarnation `json:"podClique"` + Target *BindingTarget `json:"target,omitempty"` } func (in *ConsumerBinding) DeepCopyInto(out *ConsumerBinding) { @@ -377,6 +379,7 @@ func (in *ConsumerBinding) DeepCopyInto(out *ConsumerBinding) { if in.Current != nil { out.Current = in.Current.DeepCopy() } + in.PodClique.DeepCopyInto(&out.PodClique) if in.Target != nil { out.Target = in.Target.DeepCopy() } @@ -392,10 +395,10 @@ func (in *ConsumerBinding) DeepCopy() *ConsumerBinding { } type ConsumerRequest struct { - ClaimNames []string `json:"claimNames"` - EndpointID *string `json:"endpointId,omitempty"` - ID string `json:"id"` - PodRef NamespacedName `json:"podRef"` + ClaimNames []string `json:"claimNames"` + EndpointID *string `json:"endpointId,omitempty"` + ID string `json:"id"` + PodCliqueRef PodCliqueReference `json:"podCliqueRef"` } func (in *ConsumerRequest) DeepCopyInto(out *ConsumerRequest) { @@ -407,7 +410,7 @@ func (in *ConsumerRequest) DeepCopyInto(out *ConsumerRequest) { value := *in.EndpointID out.EndpointID = &value } - in.PodRef.DeepCopyInto(&out.PodRef) + in.PodCliqueRef.DeepCopyInto(&out.PodCliqueRef) } func (in *ConsumerRequest) DeepCopy() *ConsumerRequest { @@ -420,13 +423,13 @@ func (in *ConsumerRequest) DeepCopy() *ConsumerRequest { } type ConsumerSlot struct { - Claims []ClaimReference `json:"claims"` - ConsumerID string `json:"consumerId"` - EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` - EndpointID string `json:"endpointId"` - GroveLineage *GroveLineage `json:"groveLineage,omitempty"` - NodeName string `json:"nodeName"` - SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` + Claims []ClaimReference `json:"claims"` + ConsumerID string `json:"consumerId"` + EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` + EndpointID string `json:"endpointId"` + NodeName string `json:"nodeName"` + PodCliqueRef PodCliqueReference `json:"podCliqueRef"` + SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` } func (in *ConsumerSlot) DeepCopyInto(out *ConsumerSlot) { @@ -443,9 +446,7 @@ func (in *ConsumerSlot) DeepCopyInto(out *ConsumerSlot) { out.EffectiveNodeRequests[key] = value } } - if in.GroveLineage != nil { - out.GroveLineage = in.GroveLineage.DeepCopy() - } + in.PodCliqueRef.DeepCopyInto(&out.PodCliqueRef) } func (in *ConsumerSlot) DeepCopy() *ConsumerSlot { @@ -476,24 +477,6 @@ func (in *CurrentBinding) DeepCopy() *CurrentBinding { return out } -type CyborgPodCliqueReference struct { - Name string `json:"name"` - UID string `json:"uid"` -} - -func (in *CyborgPodCliqueReference) DeepCopyInto(out *CyborgPodCliqueReference) { - *out = *in -} - -func (in *CyborgPodCliqueReference) DeepCopy() *CyborgPodCliqueReference { - if in == nil { - return nil - } - out := new(CyborgPodCliqueReference) - in.DeepCopyInto(out) - return out -} - type DerivedStageAssignment struct { FabricID string `json:"fabricId"` ID string `json:"id"` @@ -543,27 +526,6 @@ const ( ExecutionBackendNodeLocal ExecutionBackend = "nodeLocal" ) -type GroveLineage struct { - PodCliqueRef ObjectReference `json:"podCliqueRef"` - PodCliqueSetReplicaIndex int64 `json:"podCliqueSetReplicaIndex"` - PodIndex int64 `json:"podIndex"` - PodTemplateHash string `json:"podTemplateHash"` -} - -func (in *GroveLineage) DeepCopyInto(out *GroveLineage) { - *out = *in - in.PodCliqueRef.DeepCopyInto(&out.PodCliqueRef) -} - -func (in *GroveLineage) DeepCopy() *GroveLineage { - if in == nil { - return nil - } - out := new(GroveLineage) - in.DeepCopyInto(out) - return out -} - type GroveSlotDigest string type HxLogicalConnection struct { @@ -656,17 +618,17 @@ func (in *LPUPartitionPlacement) DeepCopy() *LPUPartitionPlacement { } type LPUPipelineRequestSpec struct { - AllocationMetadata runtime.RawExtension `json:"allocationMetadata"` - CyborgPodCliqueRef *CyborgPodCliqueReference `json:"cyborgPodCliqueRef,omitempty"` - ExecutionBackend ExecutionBackend `json:"executionBackend"` - NodeLocal *NodeLocalRequest `json:"nodeLocal,omitempty"` - Partitions []PartitionRequest `json:"partitions"` - PodGangRef NamespacedName `json:"podGangRef"` - PropSyncConnectors []PropSyncConnectorRequest `json:"propSyncConnectors"` - RemoteDra *RemoteDraRequest `json:"remoteDra,omitempty"` - RepairPolicy *RepairPolicy `json:"repairPolicy,omitempty"` - TargetFamily TargetFamily `json:"targetFamily"` - WorkloadMode WorkloadMode `json:"workloadMode"` + AllocationMetadata runtime.RawExtension `json:"allocationMetadata"` + CyborgPodCliqueRef *PodCliqueReference `json:"cyborgPodCliqueRef,omitempty"` + ExecutionBackend ExecutionBackend `json:"executionBackend"` + MaterializationTarget MaterializationTarget `json:"materializationTarget"` + NodeLocal *NodeLocalRequest `json:"nodeLocal,omitempty"` + Partitions []PartitionRequest `json:"partitions"` + PropSyncConnectors []PropSyncConnectorRequest `json:"propSyncConnectors"` + RemoteDra *RemoteDraRequest `json:"remoteDra,omitempty"` + RepairPolicy *RepairPolicy `json:"repairPolicy,omitempty"` + TargetFamily TargetFamily `json:"targetFamily"` + WorkloadMode WorkloadMode `json:"workloadMode"` } func (in *LPUPipelineRequestSpec) DeepCopyInto(out *LPUPipelineRequestSpec) { @@ -675,6 +637,7 @@ func (in *LPUPipelineRequestSpec) DeepCopyInto(out *LPUPipelineRequestSpec) { if in.CyborgPodCliqueRef != nil { out.CyborgPodCliqueRef = in.CyborgPodCliqueRef.DeepCopy() } + in.MaterializationTarget.DeepCopyInto(&out.MaterializationTarget) if in.NodeLocal != nil { out.NodeLocal = in.NodeLocal.DeepCopy() } @@ -684,7 +647,6 @@ func (in *LPUPipelineRequestSpec) DeepCopyInto(out *LPUPipelineRequestSpec) { in.Partitions[i].DeepCopyInto(&out.Partitions[i]) } } - in.PodGangRef.DeepCopyInto(&out.PodGangRef) if in.PropSyncConnectors != nil { out.PropSyncConnectors = make([]PropSyncConnectorRequest, len(in.PropSyncConnectors)) for i := range in.PropSyncConnectors { @@ -758,24 +720,53 @@ func (in *LPUPipelineRequestStatus) DeepCopy() *LPUPipelineRequestStatus { return out } -type NamespacedName struct { - Name string `json:"name"` - Namespace *string `json:"namespace,omitempty"` +type MaterializationScope struct { + CompilerSnapshotDigest string `json:"compilerSnapshotDigest"` + CyborgPodCliqueRef *PodCliqueReference `json:"cyborgPodCliqueRef,omitempty"` + DynamoGraphDeploymentRef ObjectReference `json:"dynamoGraphDeploymentRef"` + PodCliqueScalingGroupRef *PodCliqueScalingGroupScope `json:"podCliqueScalingGroupRef,omitempty"` + PodCliqueSetRef ObjectReference `json:"podCliqueSetRef"` + PodCliqueSetReplicaIndex int64 `json:"podCliqueSetReplicaIndex"` } -func (in *NamespacedName) DeepCopyInto(out *NamespacedName) { +func (in *MaterializationScope) DeepCopyInto(out *MaterializationScope) { *out = *in - if in.Namespace != nil { - value := *in.Namespace - out.Namespace = &value + if in.CyborgPodCliqueRef != nil { + out.CyborgPodCliqueRef = in.CyborgPodCliqueRef.DeepCopy() + } + in.DynamoGraphDeploymentRef.DeepCopyInto(&out.DynamoGraphDeploymentRef) + if in.PodCliqueScalingGroupRef != nil { + out.PodCliqueScalingGroupRef = in.PodCliqueScalingGroupRef.DeepCopy() + } + in.PodCliqueSetRef.DeepCopyInto(&out.PodCliqueSetRef) +} + +func (in *MaterializationScope) DeepCopy() *MaterializationScope { + if in == nil { + return nil + } + out := new(MaterializationScope) + in.DeepCopyInto(out) + return out +} + +type MaterializationTarget struct { + PodCliqueScalingGroupRef *PodCliqueScalingGroupReference `json:"podCliqueScalingGroupRef,omitempty"` + PodCliqueSetReplicaIndex int64 `json:"podCliqueSetReplicaIndex"` +} + +func (in *MaterializationTarget) DeepCopyInto(out *MaterializationTarget) { + *out = *in + if in.PodCliqueScalingGroupRef != nil { + out.PodCliqueScalingGroupRef = in.PodCliqueScalingGroupRef.DeepCopy() } } -func (in *NamespacedName) DeepCopy() *NamespacedName { +func (in *MaterializationTarget) DeepCopy() *MaterializationTarget { if in == nil { return nil } - out := new(NamespacedName) + out := new(MaterializationTarget) in.DeepCopyInto(out) return out } @@ -831,12 +822,14 @@ func (in *NodeLocalEndpoint) DeepCopy() *NodeLocalEndpoint { } type NodeLocalExecution struct { + Materialization NodeLocalMaterialization `json:"materialization"` PartitionSelections []NodeLocalPartitionExecution `json:"partitionSelections"` PlanReplacement *NodeLocalPlanReplacement `json:"planReplacement,omitempty"` } func (in *NodeLocalExecution) DeepCopyInto(out *NodeLocalExecution) { *out = *in + in.Materialization.DeepCopyInto(&out.Materialization) if in.PartitionSelections != nil { out.PartitionSelections = make([]NodeLocalPartitionExecution, len(in.PartitionSelections)) for i := range in.PartitionSelections { @@ -857,6 +850,50 @@ func (in *NodeLocalExecution) DeepCopy() *NodeLocalExecution { return out } +type NodeLocalMaterialization struct { + Current NodeLocalMaterializationIncarnation `json:"current"` + Target *NodeLocalMaterializationIncarnation `json:"target,omitempty"` +} + +func (in *NodeLocalMaterialization) DeepCopyInto(out *NodeLocalMaterialization) { + *out = *in + in.Current.DeepCopyInto(&out.Current) + if in.Target != nil { + out.Target = in.Target.DeepCopy() + } +} + +func (in *NodeLocalMaterialization) DeepCopy() *NodeLocalMaterialization { + if in == nil { + return nil + } + out := new(NodeLocalMaterialization) + in.DeepCopyInto(out) + return out +} + +type NodeLocalMaterializationIncarnation struct { + AgentPodClique PodCliqueIncarnation `json:"agentPodClique"` + CyborgPodClique *PodCliqueIncarnation `json:"cyborgPodClique,omitempty"` +} + +func (in *NodeLocalMaterializationIncarnation) DeepCopyInto(out *NodeLocalMaterializationIncarnation) { + *out = *in + in.AgentPodClique.DeepCopyInto(&out.AgentPodClique) + if in.CyborgPodClique != nil { + out.CyborgPodClique = in.CyborgPodClique.DeepCopy() + } +} + +func (in *NodeLocalMaterializationIncarnation) DeepCopy() *NodeLocalMaterializationIncarnation { + if in == nil { + return nil + } + out := new(NodeLocalMaterializationIncarnation) + in.DeepCopyInto(out) + return out +} + type NodeLocalPartitionExecution struct { PartitionID string `json:"partitionId"` SelectedRows []NodeLocalRowExecution `json:"selectedRows"` @@ -925,21 +962,23 @@ func (in *NodeLocalPartitionSelection) DeepCopy() *NodeLocalPartitionSelection { } type NodeLocalPlacement struct { - CompilerMetadataDigest string `json:"compilerMetadataDigest"` - ConnectorEvidence []NodeLocalConnectorEvidence `json:"connectorEvidence,omitempty"` - EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` - InventoryRevision string `json:"inventoryRevision"` - Model string `json:"model"` - NativeRepairBase *string `json:"nativeRepairBase,omitempty"` - Objective PlanObjective `json:"objective"` - PartitionSelections []NodeLocalPartitionSelection `json:"partitionSelections"` - PodGangRef ObjectReference `json:"podGangRef"` - SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` - WorkloadMode WorkloadMode `json:"workloadMode"` + AgentPodCliqueRef PodCliqueReference `json:"agentPodCliqueRef"` + CompilerMetadataDigest string `json:"compilerMetadataDigest"` + ConnectorEvidence []NodeLocalConnectorEvidence `json:"connectorEvidence,omitempty"` + EffectiveNodeRequests map[string]int64 `json:"effectiveNodeRequests"` + InventoryRevision string `json:"inventoryRevision"` + Model string `json:"model"` + NativeRepairBase *string `json:"nativeRepairBase,omitempty"` + Objective PlanObjective `json:"objective"` + PartitionSelections []NodeLocalPartitionSelection `json:"partitionSelections"` + PodSchedulingDeclarationDigest string `json:"podSchedulingDeclarationDigest"` + SchedulingConstraintsDigest string `json:"schedulingConstraintsDigest"` + WorkloadMode WorkloadMode `json:"workloadMode"` } func (in *NodeLocalPlacement) DeepCopyInto(out *NodeLocalPlacement) { *out = *in + in.AgentPodCliqueRef.DeepCopyInto(&out.AgentPodCliqueRef) if in.ConnectorEvidence != nil { out.ConnectorEvidence = make([]NodeLocalConnectorEvidence, len(in.ConnectorEvidence)) for i := range in.ConnectorEvidence { @@ -963,7 +1002,6 @@ func (in *NodeLocalPlacement) DeepCopyInto(out *NodeLocalPlacement) { in.PartitionSelections[i].DeepCopyInto(&out.PartitionSelections[i]) } } - in.PodGangRef.DeepCopyInto(&out.PodGangRef) } func (in *NodeLocalPlacement) DeepCopy() *NodeLocalPlacement { @@ -1018,12 +1056,14 @@ func (in *NodeLocalRepairAuthorization) DeepCopy() *NodeLocalRepairAuthorization } type NodeLocalRequest struct { + AgentPodCliqueRef PodCliqueReference `json:"agentPodCliqueRef"` Model string `json:"model"` PartitionMappings []NodeLocalPartitionMapping `json:"partitionMappings"` } func (in *NodeLocalRequest) DeepCopyInto(out *NodeLocalRequest) { *out = *in + in.AgentPodCliqueRef.DeepCopyInto(&out.AgentPodCliqueRef) if in.PartitionMappings != nil { out.PartitionMappings = make([]NodeLocalPartitionMapping, len(in.PartitionMappings)) for i := range in.PartitionMappings { @@ -1042,6 +1082,7 @@ func (in *NodeLocalRequest) DeepCopy() *NodeLocalRequest { } type NodeLocalRowExecution struct { + AdoptedTarget bool `json:"adoptedTarget,omitempty"` Current *CurrentBinding `json:"current,omitempty"` RankInPartition int64 `json:"rankInPartition"` Repair *NodeLocalRepairAuthorization `json:"repair,omitempty"` @@ -1072,7 +1113,6 @@ func (in *NodeLocalRowExecution) DeepCopy() *NodeLocalRowExecution { type NodeLocalSelectedRow struct { Endpoint NodeLocalEndpoint `json:"endpoint"` - GroveLineage *GroveLineage `json:"groveLineage,omitempty"` NodeRef NodeReference `json:"nodeRef"` RankInPartition int64 `json:"rankInPartition"` } @@ -1080,9 +1120,6 @@ type NodeLocalSelectedRow struct { func (in *NodeLocalSelectedRow) DeepCopyInto(out *NodeLocalSelectedRow) { *out = *in in.Endpoint.DeepCopyInto(&out.Endpoint) - if in.GroveLineage != nil { - out.GroveLineage = in.GroveLineage.DeepCopy() - } in.NodeRef.DeepCopyInto(&out.NodeRef) } @@ -1246,13 +1283,15 @@ func (in *PlanObjective) DeepCopy() *PlanObjective { } type PlanPlacement struct { - ExecutionBackend ExecutionBackend `json:"executionBackend"` - NodeLocal *NodeLocalPlacement `json:"nodeLocal,omitempty"` - RemoteDra *RemoteDraPlacement `json:"remoteDra,omitempty"` + ExecutionBackend ExecutionBackend `json:"executionBackend"` + MaterializationScope MaterializationScope `json:"materializationScope"` + NodeLocal *NodeLocalPlacement `json:"nodeLocal,omitempty"` + RemoteDra *RemoteDraPlacement `json:"remoteDra,omitempty"` } func (in *PlanPlacement) DeepCopyInto(out *PlanPlacement) { *out = *in + in.MaterializationScope.DeepCopyInto(&out.MaterializationScope) if in.NodeLocal != nil { out.NodeLocal = in.NodeLocal.DeepCopy() } @@ -1272,6 +1311,79 @@ func (in *PlanPlacement) DeepCopy() *PlanPlacement { type PlanReplacementAbortFence string +type PodCliqueIncarnation struct { + PodTemplateHash string `json:"podTemplateHash"` + UID string `json:"uid"` +} + +func (in *PodCliqueIncarnation) DeepCopyInto(out *PodCliqueIncarnation) { + *out = *in +} + +func (in *PodCliqueIncarnation) DeepCopy() *PodCliqueIncarnation { + if in == nil { + return nil + } + out := new(PodCliqueIncarnation) + in.DeepCopyInto(out) + return out +} + +type PodCliqueReference struct { + Name string `json:"name"` +} + +func (in *PodCliqueReference) DeepCopyInto(out *PodCliqueReference) { + *out = *in +} + +func (in *PodCliqueReference) DeepCopy() *PodCliqueReference { + if in == nil { + return nil + } + out := new(PodCliqueReference) + in.DeepCopyInto(out) + return out +} + +type PodCliqueScalingGroupReference struct { + Name string `json:"name"` + ReplicaIndex int64 `json:"replicaIndex"` +} + +func (in *PodCliqueScalingGroupReference) DeepCopyInto(out *PodCliqueScalingGroupReference) { + *out = *in +} + +func (in *PodCliqueScalingGroupReference) DeepCopy() *PodCliqueScalingGroupReference { + if in == nil { + return nil + } + out := new(PodCliqueScalingGroupReference) + in.DeepCopyInto(out) + return out +} + +type PodCliqueScalingGroupScope struct { + Name string `json:"name"` + ReplicaIndex int64 `json:"replicaIndex"` + SpecDigest string `json:"specDigest"` + UID string `json:"uid"` +} + +func (in *PodCliqueScalingGroupScope) DeepCopyInto(out *PodCliqueScalingGroupScope) { + *out = *in +} + +func (in *PodCliqueScalingGroupScope) DeepCopy() *PodCliqueScalingGroupScope { + if in == nil { + return nil + } + out := new(PodCliqueScalingGroupScope) + in.DeepCopyInto(out) + return out +} + type PropSyncConnectorKind string const ( @@ -1451,7 +1563,6 @@ type RemoteDraPlacement struct { Objective PlanObjective `json:"objective"` PartitionConductors []PartitionConductorAssignment `json:"partitionConductors"` PartitionPlacements []LPUPartitionPlacement `json:"partitionPlacements"` - PodGangRef ObjectReference `json:"podGangRef"` WorkloadMode WorkloadMode `json:"workloadMode"` } @@ -1493,7 +1604,6 @@ func (in *RemoteDraPlacement) DeepCopyInto(out *RemoteDraPlacement) { in.PartitionPlacements[i].DeepCopyInto(&out.PartitionPlacements[i]) } } - in.PodGangRef.DeepCopyInto(&out.PodGangRef) } func (in *RemoteDraPlacement) DeepCopy() *RemoteDraPlacement { diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml similarity index 98% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 46112c875d26..22b97afb70d1 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -27,6 +27,8 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a namespace: inference-engine --- @@ -185,6 +187,8 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 namespace: inference-engine --- @@ -195,6 +199,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm name: dgd-lpx-v2-vllm-94cd namespace: inference-engine @@ -207,6 +212,7 @@ spec: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: d73a69201a8e19d2013c560e1155903bacf71d3a2cbdaaf4f3aea4deafa8d033 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/model: default @@ -590,7 +596,7 @@ spec: - mountPath: /dev/shm name: shared-memory restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml similarity index 97% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 51e889ece842..6ccab754927b 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -10,6 +10,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 namespace: test --- @@ -49,6 +51,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 namespace: test --- @@ -59,6 +63,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid name: node-local-v2-hybrid-9d06 namespace: test @@ -71,6 +76,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 5a21b650237fd262f004a640a67c5ecac0781784fa057703ee95223ab6a30c59 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/model: default @@ -385,7 +391,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml similarity index 97% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 014544e1569b..f438eabcfb73 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -53,6 +53,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 namespace: test --- @@ -63,6 +65,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only name: node-local-v2-lpu-only-33b1 namespace: test @@ -75,6 +78,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6acbae87b5dfe4f0837a02ffddf18a9af54e8ce74b645d0267c059517b8c1e2b + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/model: default @@ -367,7 +371,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml similarity index 97% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 10ac016bdf09..7756a2f58da4 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -122,6 +122,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 namespace: test --- @@ -132,6 +134,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode name: node-local-v2-specdecode-63df namespace: test @@ -144,6 +147,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 scheduling.lpu.nvidia.com/model: draft0 @@ -290,6 +294,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b scheduling.lpu.nvidia.com/model: draft1 @@ -436,6 +441,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e scheduling.lpu.nvidia.com/model: target @@ -729,7 +735,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml similarity index 97% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 3fc16b8bef5e..a9210393749d 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -46,6 +46,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 namespace: test --- @@ -56,6 +58,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid name: node-local-v3-hx-hybrid-be18 namespace: test @@ -68,6 +71,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 7443e572ed6c9124e2bfdacd89de30a0515735f36cfb677a99e1be70668ca8d0 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 scheduling.lpu.nvidia.com/model: default @@ -363,7 +367,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml similarity index 98% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 5648761b2e72..d008b1d641c8 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -45,6 +45,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f namespace: test --- @@ -55,6 +57,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only name: node-local-v3-hx-lpu-only-d43c namespace: test @@ -67,6 +70,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b388a698f7dcc08f8fd26e78b06feab8d6992ea55a77555aacb9c70d9262f77a + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/model: default @@ -431,7 +435,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml similarity index 98% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 7250fd4ecfed..e0083f475ebf 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -101,6 +101,8 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b namespace: test --- @@ -111,6 +113,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode name: node-local-v3-hx-specdecode-1f7a namespace: test @@ -123,6 +126,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea scheduling.lpu.nvidia.com/model: draft0 @@ -302,6 +306,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed scheduling.lpu.nvidia.com/model: draft1 @@ -481,6 +486,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a scheduling.lpu.nvidia.com/model: target @@ -847,7 +853,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml similarity index 100% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.input.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml diff --git a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml similarity index 98% rename from deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml rename to deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index b7ff682b85f8..5c0c6649abe1 100644 --- a/deploy/operator/internal/dynamo/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -129,6 +129,8 @@ metadata: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa namespace: inference-engine --- @@ -139,6 +141,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 name: dgd-single-v2-3a98 namespace: inference-engine @@ -151,6 +154,7 @@ spec: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: cc95c2008c398eaa60b329a696fa0f2faeff6c094df93b3d88d73cde3e6966a4 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/model: default @@ -452,7 +456,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go index 7e7cfba9586c..49d01ba79991 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -12,8 +12,6 @@ import ( "encoding/json" "fmt" "hash" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" ) const ( @@ -22,10 +20,6 @@ const ( modelProjectionDigestVersion = "dynamo-node-local-compilation/v1" workloadSetDigestVersion = "dynamo-lpx-compilation-set/v1" maxLPXPartitions = 256 - // LPX reserves 192 KiB of the Kubernetes 1 MiB object ceiling for - // lifecycle status. Its pre-solve admission charges the status-free - // request plus a deterministic echo of every connector requirement. - maxLPXPlannedRequestObjectBytes = (1024 - 192) * 1024 ) // appendModelProjections appends one component's immutable model projections to @@ -81,36 +75,6 @@ func workloadSetDigest(projections []*ModelProjection) (WorkloadDigest, error) { return transcript.sum(), nil } -// ValidateRequestSize applies the pinned LPX scheduler's pre-solve size -// accounting to a non-nil compiler-produced or API-decoded request with valid -// raw JSON fields. The request is not mutated. -func ValidateRequestSize(request *lpxv1alpha1.LPUPipelineRequest) error { - projectedBytes := requestWithRequirementEchoBytes(request) - if projectedBytes > maxLPXPlannedRequestObjectBytes { - return fmt.Errorf( - "status-free request plus connector-requirement echo is %d bytes; maximum is %d", - projectedBytes, - maxLPXPlannedRequestObjectBytes, - ) - } - return nil -} - -func requestWithRequirementEchoBytes(request *lpxv1alpha1.LPUPipelineRequest) int { - statusFree := *request - statusFree.Status = nil - if statusFree.ResourceVersion != "" { - statusFree.ResourceVersion = "0" - } - requestBytes, _ := json.Marshal(&statusFree) - requirements := make([]lpxv1alpha1.PropSyncConnectorRequirement, 0, len(request.Spec.PropSyncConnectors)) - for _, connector := range request.Spec.PropSyncConnectors { - requirements = append(requirements, connector.Requirement) - } - echoBytes, _ := json.Marshal(requirements) - return len(requestBytes) + len(echoBytes) -} - // canonicalModelSettings canonicalizes an API-validated optional JSON object. func canonicalModelSettings(raw json.RawMessage) (json.RawMessage, map[string]any) { trimmed := bytes.TrimSpace(raw) diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go index a3dae2d102f4..8bdf61f71129 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -7,79 +7,12 @@ package lpx import ( "encoding/json" - "strings" "testing" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" ) -func TestValidateRequestSizeMatchesLPXBoundary(t *testing.T) { - t.Parallel() - - t.Log("Build a request close enough to the limit that valid annotation padding can reach the exact boundary") - request := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: "request", - Namespace: "namespace", - Annotations: map[string]string{"scheduling.lpu.nvidia.com/size-proof": ""}, - }, - Spec: lpxv1alpha1.LPUPipelineRequestSpec{ - AllocationMetadata: runtime.RawExtension{Raw: []byte(`{}`)}, - ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, - NodeLocal: &lpxv1alpha1.NodeLocalRequest{Model: "default"}, - Partitions: []lpxv1alpha1.PartitionRequest{{ - ID: "partition-000", Ordinal: 0, CompilerPartitionID: 0, - Extent: &[]int64{16, 1, 1, 1}, - }}, - PodGangRef: lpxv1alpha1.NamespacedName{Name: "agents"}, - TargetFamily: lpxv1alpha1.TargetFamilyHx16x8x2x3, - WorkloadMode: lpxv1alpha1.WorkloadModeV3HxLPUOnly, - }, - } - connections := make([]lpxv1alpha1.HxLogicalConnection, 256) - for index := range connections { - connections[index] = lpxv1alpha1.HxLogicalConnection{ - FromLogicalDevice: int64(index / 16), - ToLogicalDevice: int64(index % 16), - } - } - lanes := []int64{4, 2, 1} - baseBytes := 0 - for len(request.Spec.PropSyncConnectors) < maxLPXPartitions-1 { - baseBytes = requestWithRequirementEchoBytes(request) - if baseBytes >= maxLPXPlannedRequestObjectBytes-200*1024 { - break - } - request.Spec.PropSyncConnectors = append( - request.Spec.PropSyncConnectors, - lpxv1alpha1.PropSyncConnectorRequest{ - FromPartitionID: "partition-000", - ToPartitionID: "partition-000", - Requirement: lpxv1alpha1.PropSyncConnectorRequirement{ - Kind: lpxv1alpha1.PropSyncConnectorKindHxPropSyncV1, - Connections: &connections, - AcceptableLaneMultiplicities: &lanes, - }, - }, - ) - } - - t.Log("Pad the request annotation to the exact inclusive scheduler budget") - baseBytes = requestWithRequirementEchoBytes(request) - require.Less(t, baseBytes, maxLPXPlannedRequestObjectBytes) - padding := maxLPXPlannedRequestObjectBytes - baseBytes - require.LessOrEqual(t, padding, 200*1024) - request.Annotations["scheduling.lpu.nvidia.com/size-proof"] = strings.Repeat("x", padding) - - t.Log("Accept equality and reject the first byte beyond LPX's inclusive limit") - require.NoError(t, ValidateRequestSize(request)) - request.Annotations["scheduling.lpu.nvidia.com/size-proof"] += "x" - require.ErrorContains(t, ValidateRequestSize(request), "maximum") -} - func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { t.Parallel() @@ -93,8 +26,18 @@ func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { firstProjection := projectTestBuild(t, first, PipelineSingle, `{"prop_sync":true}`) secondProjection := projectTestBuild(t, second, PipelineSingle, `{"prop_sync":true}`) - t.Log("Produce the same projection digest independent of build locator") + t.Log("Publish the canonical compiler snapshot identity independently of build locator") + require.Equal(t, "scheduling.lpu.nvidia.com/compiler-snapshot-digest", lpxv1alpha1.CompilerSnapshotDigestAnnotation) + require.Equal(t, first.contentID, firstProjection.CompilerSnapshotDigest()) + require.Equal(t, second.contentID, secondProjection.CompilerSnapshotDigest()) + + t.Log("Produce the same workload projection digest independent of build locator") require.Equal(t, firstProjection.Digest(), secondProjection.Digest()) + + t.Log("Keep compiler identity separate from downstream workload projection identity") + specDecodeProjection := projectTestBuild(t, first, PipelineSpecDecode, `{"prop_sync":true}`) + require.Equal(t, firstProjection.CompilerSnapshotDigest(), specDecodeProjection.CompilerSnapshotDigest()) + require.NotEqual(t, firstProjection.Digest(), specDecodeProjection.Digest()) } func TestCanonicalModelSettingsRetainRawAndNormalizeRuntimeNumbers(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index c63f9e229c50..a59cc0e43f99 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -143,15 +143,16 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu transcript.field("allocation-metadata", allocationMetadata) dst = append(dst, &ModelProjection{ - digest: transcript.sum(), - runtimeBuildRef: intent.RuntimeBuildRef, - model: intent.Models[index], - pipeline: intent.Pipeline, - configuredBuild: configured, - allocationMetadata: allocationMetadata, - partitions: partitions, - connectors: connectors, - agentReplicas: agentReplicas, + digest: transcript.sum(), + compilerSnapshotDigest: intent.BuildSnapshot.contentID, + runtimeBuildRef: intent.RuntimeBuildRef, + model: intent.Models[index], + pipeline: intent.Pipeline, + configuredBuild: configured, + allocationMetadata: allocationMetadata, + partitions: partitions, + connectors: connectors, + agentReplicas: agentReplicas, }) } return dst, nil diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index 8622c45d1a44..b9dbc48deff5 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -38,7 +38,7 @@ func TestProjectModelV2SubHostPartitionUsesWholeHostShape(t *testing.T) { projection := projectionBatch[0] t.Log("Verify the two-chip compiler partition reserves one C8 host and produces one Agent replica") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.Partitions, 1) require.Equal(t, lpxv1alpha1.Xt8888PartitionShapeC8, *spec.Partitions[0].XtShape) require.NotNil(t, spec.NodeLocal) @@ -109,14 +109,14 @@ func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { }) require.NoError(t, err) defaulted := defaultedBatch[0] - defaultSpec := defaulted.RequestSpec("test", "agents", nil) + defaultSpec := defaulted.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, defaultSpec.Partitions, 2) require.Len(t, defaultSpec.PropSyncConnectors, 1) require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) t.Log("Enable prop sync and project its effective connector offset") projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":true,"prop_sync_multiple_load_sets":true}`) - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.PropSyncConnectors, 1) require.Equal(t, int64(3), *spec.PropSyncConnectors[0].Requirement.MaxInterPartitionOffset) @@ -130,7 +130,7 @@ func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { disabledBatch, err := appendModelProjections(nil, intent) require.NoError(t, err) disabled := disabledBatch[0] - require.Len(t, disabled.RequestSpec("test", "agents", nil).PropSyncConnectors, 1) + require.Len(t, disabled.RequestSpec(&MaterializationPlan{}, "agents").PropSyncConnectors, 1) t.Log("Require manifest-selected evidence when revision 2 enables prop sync") build.SelectedPropSyncChains = nil @@ -154,12 +154,12 @@ func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { t.Log("Omit source partition zero when CPU embeddings are enabled") offLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":true}`) - require.Equal(t, int64(1), offLPU.RequestSpec("test", "agents", nil).Partitions[0].CompilerPartitionID) + require.Equal(t, int64(1), offLPU.RequestSpec(&MaterializationPlan{}, "agents").Partitions[0].CompilerPartitionID) t.Log("Retain source partition zero when CPU embeddings are disabled") onLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":false}`) require.Equal(t, 4, onLPU.agentReplicas) - require.Equal(t, int64(0), onLPU.RequestSpec("test", "agents", nil).Partitions[0].CompilerPartitionID) + require.Equal(t, int64(0), onLPU.RequestSpec(&MaterializationPlan{}, "agents").Partitions[0].CompilerPartitionID) } func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { @@ -176,7 +176,7 @@ func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { t.Log("Keep Cyborg's partition zero regardless of the Nova CPU-embedding setting") for _, settings := range []string{"", `{"cpu_embeddings":false}`, `{"cpu_embeddings":true}`} { projection := projectTestBuild(t, normalized, PipelineLPX, settings) - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) require.Len(t, spec.Partitions, 2) require.Equal(t, int64(0), spec.Partitions[0].CompilerPartitionID) @@ -212,7 +212,7 @@ func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":false}`) t.Log("Preserve only the source-selected connector and partition identities") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.Partitions, 2) require.Len(t, spec.PropSyncConnectors, 1) require.Equal(t, int64(7), spec.Partitions[0].CompilerPartitionID) @@ -254,7 +254,7 @@ func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing. projection := projectTestBuild(t, normalized, PipelineLPX, "") t.Log("Project all physical scheduler partitions and only the selected connector") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.Partitions, 3) require.Len(t, spec.PropSyncConnectors, 1) require.Equal(t, spec.Partitions[0].ID, spec.PropSyncConnectors[0].FromPartitionID) @@ -312,7 +312,7 @@ func TestProjectModelV2CollapsesSelectedChainIncludingPartitionZero(t *testing.T projection := projectTestBuild(t, normalized, PipelineLPX, "") t.Log("Keep all physical scheduler partitions while collapsing their Agent runtime projection") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.Partitions, 3) for index, partition := range spec.Partitions { require.Equal(t, int64(index), partition.CompilerPartitionID) @@ -347,7 +347,7 @@ func TestProjectModelV2PreservesAgentReplicasWhenCollapsingSubHostPartitions(t * projection := projectTestBuild(t, normalized, PipelineLPX, "") t.Log("Preserve both physical scheduler partitions and Agent replicas") - require.Len(t, projection.RequestSpec("test", "agents", nil).Partitions, 2) + require.Len(t, projection.RequestSpec(&MaterializationPlan{}, "agents").Partitions, 2) require.Equal(t, 2, projection.agentReplicas) t.Log("Keep both physical Agent endpoints in the collapsed C4 runtime partition") diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 6f7fbfb19264..2f1a25739377 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -77,15 +77,16 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu bindHybridRuntimeIO(transcript, intent.Pipeline, ioFPGACount, ioFanoutFactor) dst = append(dst, &ModelProjection{ - digest: transcript.sum(), - runtimeBuildRef: intent.RuntimeBuildRef, - model: intent.Models[index], - pipeline: intent.Pipeline, - configuredBuild: runtimeBuild, - allocationMetadata: allocationMetadata, - partitions: manifestPartitions, - connectors: connectors, - agentReplicas: agentReplicas, + digest: transcript.sum(), + compilerSnapshotDigest: intent.BuildSnapshot.contentID, + runtimeBuildRef: intent.RuntimeBuildRef, + model: intent.Models[index], + pipeline: intent.Pipeline, + configuredBuild: runtimeBuild, + allocationMetadata: allocationMetadata, + partitions: manifestPartitions, + connectors: connectors, + agentReplicas: agentReplicas, }) } return dst, nil diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index 20435d1ae7af..f80271f0d31c 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -44,8 +44,8 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.Equal(t, projection.Digest(), digest, "one model is the aggregate digest base case") t.Log("Project only the two LPU artifacts and their selected chain into the hybrid scheduler request") - cyborgRef := &lpxv1alpha1.CyborgPodCliqueReference{Name: "decode", UID: "decode-uid"} - spec := projection.RequestSpec("test", "agents", cyborgRef) + cyborgRef := &lpxv1alpha1.PodCliqueReference{Name: "decode"} + spec := projection.RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents") require.Equal(t, lpxv1alpha1.WorkloadModeV3HxStrictHybrid, spec.WorkloadMode) require.Len(t, spec.Partitions, 2) require.Equal(t, int64(1), spec.Partitions[0].CompilerPartitionID) @@ -55,8 +55,7 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.Equal(t, spec.Partitions[1].ID, spec.PropSyncConnectors[0].ToPartitionID) t.Log("Bind workload references while mapping compiler IDs to model ordinals") - require.Equal(t, "agents", spec.PodGangRef.Name) - require.Equal(t, "test", *spec.PodGangRef.Namespace) + require.Equal(t, "agents", spec.NodeLocal.AgentPodCliqueRef.Name) require.Equal(t, "default", spec.NodeLocal.Model) require.Equal(t, cyborgRef, spec.CyborgPodCliqueRef) require.Len(t, spec.NodeLocal.PartitionMappings, 2) @@ -74,8 +73,8 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( spec.AllocationMetadata.Raw[0] = ' ' spec.CyborgPodCliqueRef.Name = "changed" require.Equal(t, before.CyborgPodCliqueRef, cyborgRef) - require.Equal(t, *before, projection.RequestSpec("test", "agents", cyborgRef)) - require.Nil(t, projection.RequestSpec("test", "agents", nil).CyborgPodCliqueRef) + require.Equal(t, *before, projection.RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) + require.Nil(t, projection.RequestSpec(&MaterializationPlan{}, "agents").CyborgPodCliqueRef) t.Log("Remove the manifest's selected chain without synthesizing hybrid connectors") fixture.selectedPropSyncChains = nil @@ -84,7 +83,7 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( projectionBatch, err = appendModelProjections(nil, intent) require.NoError(t, err) projection = projectionBatch[0] - require.Empty(t, projection.RequestSpec("test", "agents", nil).PropSyncConnectors) + require.Empty(t, projection.RequestSpec(&MaterializationPlan{}, "agents").PropSyncConnectors) } func TestProjectModelV3RejectsInvalidPropSyncChains(t *testing.T) { @@ -175,7 +174,7 @@ func TestProjectModelV3ProjectsSelectedPropSyncChain(t *testing.T) { projection := projectionBatch[0] t.Log("Project both partitions and one HX prop-sync connector") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.Partitions, 2) require.Len(t, spec.PropSyncConnectors, 1) connector := spec.PropSyncConnectors[0] @@ -235,7 +234,7 @@ func TestProjectModelV3ProjectsSelectedPropSyncChain(t *testing.T) { projection = projectionBatch[0] t.Log("Advance each native scheduler connector to the next physical partition") - spec = projection.RequestSpec("test", "agents", nil) + spec = projection.RequestSpec(&MaterializationPlan{}, "agents") require.Len(t, spec.PropSyncConnectors, 2) require.Equal(t, "partition-000", spec.PropSyncConnectors[0].FromPartitionID) require.Equal(t, "partition-001", spec.PropSyncConnectors[0].ToPartitionID) @@ -262,7 +261,7 @@ func TestProjectModelV3UsesMultiNodePropSyncBoundary(t *testing.T) { projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineSingle, "") t.Log("Project logical connections from the source partition's final node") - spec := projection.RequestSpec("test", "agents", nil) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") connections := *spec.PropSyncConnectors[0].Requirement.Connections require.Equal(t, lpxv1alpha1.HxLogicalConnection{FromLogicalDevice: 16, ToLogicalDevice: 0}, connections[0]) require.Equal(t, lpxv1alpha1.HxLogicalConnection{FromLogicalDevice: 31, ToLogicalDevice: 15}, connections[15]) @@ -288,7 +287,7 @@ func TestProjectModelV3UsesTopologyMetadataAndTracksManifestDigest(t *testing.T) t.Log("Retain the acquired locator and exact single-partition allocation metadata") require.Equal(t, "file://"+buildDir, intent.BuildSnapshot.build.Path) require.Equal(t, intent.BuildSnapshot.build.Path, first.configuredBuild.Path) - firstSpec := first.RequestSpec("test", "agents", nil) + firstSpec := first.RequestSpec(&MaterializationPlan{}, "agents") require.JSONEq(t, `{ "arch":"lp30", "topology":"lyra", @@ -314,7 +313,7 @@ func TestProjectModelV3UsesTopologyMetadataAndTracksManifestDigest(t *testing.T) t.Log("Track the immutable manifest change in snapshot and projection digests") require.NotEqual(t, firstSnapshot.contentID, secondSnapshot.contentID) require.NotEqual(t, first.Digest(), second.Digest()) - secondSpec := second.RequestSpec("test", "agents", nil) + secondSpec := second.RequestSpec(&MaterializationPlan{}, "agents") t.Log("Project each manifest extent and the updated runtime partition data") require.Equal(t, []int64{16, 1, 1, 1}, *firstSpec.Partitions[0].Extent) diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go index 337747c9e1b1..48ba0bb0cd4e 100644 --- a/deploy/operator/internal/features/lpx.go +++ b/deploy/operator/internal/features/lpx.go @@ -7,7 +7,6 @@ import ( "context" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - groveschedulerv1alpha1 "github.com/ai-dynamo/grove/scheduler/api/core/v1alpha1" "k8s.io/client-go/rest" "k8s.io/utils/ptr" ) @@ -21,7 +20,7 @@ import ( // Configuration: lpx.enabled // Auto-detection: N/A; API availability is verified when explicitly enabled // Requires: LPX Scheduler serving scheduling.lpu.nvidia.com/v1alpha1 -// LpuPipelineRequest resources and scheduler.grove.io/v1alpha1 PodGang resources +// LpuPipelineRequest resources // Default: false const LPX Name = "lpx" @@ -35,18 +34,6 @@ func resolveLPX(ctx context.Context, config *rest.Config) (bool, error) { return false, err } - if _, err := resolve(ptr.To(true), available, - "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster"); err != nil { - return false, err - } - - // Discover the PodGang resource before starting its watch. - resource = groveschedulerv1alpha1.SchemeGroupVersion.WithResource("podgangs") - available, err = detectAPIAvailability(ctx, config, resource.Group, resource.Version, resource.Resource) - if err != nil { - return false, err - } - return resolve(ptr.To(true), available, - "LPX is explicitly enabled in config but the scheduler.grove.io/v1alpha1 PodGang API was not detected in the cluster") + "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster") } diff --git a/deploy/operator/internal/features/lpx_test.go b/deploy/operator/internal/features/lpx_test.go index f4b4572d20f7..88bf0743c794 100644 --- a/deploy/operator/internal/features/lpx_test.go +++ b/deploy/operator/internal/features/lpx_test.go @@ -16,40 +16,33 @@ import ( func TestResolveLPX(t *testing.T) { const ( - lprPath = "/apis/scheduling.lpu.nvidia.com/v1alpha1" - podGangPath = "/apis/scheduler.grove.io/v1alpha1" - missingLPR = "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster" - missingPodGang = "LPX is explicitly enabled in config but the scheduler.grove.io/v1alpha1 PodGang API was not detected in the cluster" + lprPath = "/apis/scheduling.lpu.nvidia.com/v1alpha1" + missingLPR = "LPX is explicitly enabled in config but the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API was not detected in the cluster" ) tests := []struct { name, path string status int resource, wantErr string }{ - {name: "both APIs present"}, + {name: "LPR API present"}, + {name: "LPR group version absent", path: lprPath, status: http.StatusNotFound, wantErr: missingLPR}, {name: "LPR resource absent", path: lprPath, status: http.StatusOK, wantErr: missingLPR}, - {name: "PodGang group version absent", path: podGangPath, status: http.StatusNotFound, wantErr: missingPodGang}, - {name: "PodGang resource absent", path: podGangPath, status: http.StatusOK, wantErr: missingPodGang}, - {name: "PodGang subresource alone is insufficient", path: podGangPath, status: http.StatusOK, resource: "podgangs/status", wantErr: missingPodGang}, - {name: "PodGang discovery forbidden", path: podGangPath, status: http.StatusForbidden, wantErr: "discover scheduler.grove.io/v1alpha1 API resources: Forbidden"}, - {name: "PodGang discovery fails", path: podGangPath, status: http.StatusInternalServerError, wantErr: "discover scheduler.grove.io/v1alpha1 API resources: Internal Server Error"}, + {name: "LPR subresource alone is insufficient", path: lprPath, status: http.StatusOK, resource: "lpupipelinerequests/status", wantErr: missingLPR}, + {name: "LPR discovery forbidden", path: lprPath, status: http.StatusForbidden, wantErr: "discover scheduling.lpu.nvidia.com/v1alpha1 API resources: Forbidden"}, + {name: "LPR discovery fails", path: lprPath, status: http.StatusInternalServerError, wantErr: "discover scheduling.lpu.nvidia.com/v1alpha1 API resources: Internal Server Error"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Log("Serve the exact LPX and PodGang API discovery paths") + t.Log("Serve the exact LPX API discovery path") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - resource := "lpupipelinerequests" - switch r.URL.Path { - case lprPath: - case podGangPath: - resource = "podgangs" - default: + if r.URL.Path != lprPath { t.Errorf("unexpected discovery path %q", r.URL.Path) http.NotFound(w, r) return } + resource := "lpupipelinerequests" status := http.StatusOK - if r.URL.Path == tt.path { + if tt.path != "" { status, resource = tt.status, tt.resource } w.Header().Set("Content-Type", "application/json") @@ -68,7 +61,7 @@ func TestResolveLPX(t *testing.T) { })) defer server.Close() - t.Log("Require both APIs and preserve discovery failures") + t.Log("Require the LPR API and preserve discovery failures") enabled, err := resolveLPX(t.Context(), &rest.Config{Host: server.URL}) if enabled != (tt.wantErr == "") || (err == nil) != (tt.wantErr == "") { t.Fatalf("resolveLPX() = %v, %v; want enabled=%v, error=%q", enabled, err, tt.wantErr == "", tt.wantErr) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 1051ca3f1bbb..c7ef0c17ad33 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -457,7 +457,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { consts.KubeAnnotationLPXSchedulerBackend: "ignored-scheduler", consts.KubeAnnotationLPXExecutionBackend: "ignored-execution", } - dgd.Spec.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(30))} component := &dgd.Spec.Components[0] component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ @@ -471,7 +470,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { { name: "selected v1alpha1 LPX derives SpecDecode from two models", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { - dgd.Spec.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(30))} setAlphaLPXSpecDec(dgd, nil) }), }, From bc4933173e9206273b59d7d36c41a8984f042179 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:19:25 -0700 Subject: [PATCH 084/196] refactor(lpx)!: simplify LPR deadline handling (#506) (cherry picked from commit f615904da8403852ebde3c4211978fcf9b78d1ea) Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 16 - .../dynamographdeployment_conversion.go | 58 -- .../v1alpha1/dynamographdeployment_types.go | 23 - .../api/v1alpha1/lpx_conversion_test.go | 14 +- .../api/v1alpha1/lpxgraphdeployment_types.go | 3 +- .../api/v1alpha1/zz_generated.deepcopy.go | 52 -- deploy/operator/api/v1beta1/common.go | 4 - deploy/operator/api/v1beta1/lpx_types.go | 22 +- .../api/v1beta1/zz_generated.deepcopy.go | 52 -- .../nvidia.com_dynamographdeployments.yaml | 192 +---- .../bases/nvidia.com_lpxgraphdeployments.yaml | 51 +- .../controller/dgd_grove_lpx_status.go | 3 - .../dgd_lpx_handoff_envtest_test.go | 8 +- .../controller/dgd_lpx_handoff_test.go | 32 +- .../controller/lpx/attempt_deadline.go | 486 +++-------- .../controller/lpx/attempt_deadline_test.go | 811 ++++++------------ .../controller/lpx/attempt_retirement.go | 153 ---- .../controller/lpx/attempt_retirement_test.go | 159 ---- .../internal/controller/lpx/controller.go | 298 ++++--- .../controller/lpx/controller_test.go | 581 ++++++++++--- .../controller/lpx/deadline_retirement.go | 173 ++++ .../lpx/deadline_retirement_test.go | 278 ++++++ .../internal/controller/lpx/lifecycle.go | 298 +++---- .../internal/controller/lpx/lifecycle_test.go | 281 ++++-- .../controller/lpx/modeldownload_test.go | 8 +- .../internal/controller/lpx/preflight.go | 17 +- .../internal/controller/lpx/render.go | 3 +- .../internal/controller/lpx/render_test.go | 100 +-- .../internal/controller/lpx/watch_test.go | 8 + deploy/operator/internal/dynamo/graph_lpx.go | 50 +- .../internal/dynamo/graph_lpx_test.go | 8 +- deploy/operator/internal/dynamo/grove_lpx.go | 38 - .../internal/dynamo/grove_lpx_test.go | 85 -- .../internal/dynamo/lpx/cyborg_render.go | 2 + .../internal/dynamo/lpx/cyborg_render_test.go | 7 +- .../internal/dynamo/lpx/materialization.go | 17 +- .../dynamo/lpx/materialization_test.go | 54 +- deploy/operator/internal/dynamo/lpx/render.go | 13 +- .../dynamo/lpx/scheduler/v1alpha1/types.go | 4 + .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 4 +- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 4 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 4 +- .../node-local-v2-specdecode.yaml | 6 +- .../node-local-v3-hx-hybrid.yaml | 4 +- .../node-local-v3-hx-lpu-only.yaml | 4 +- .../node-local-v3-hx-specdecode.yaml | 6 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 4 +- .../validation/dynamographdeployment.go | 9 + ...hdeployment_lpx_validation_envtest_test.go | 12 + .../additional-resources/api-reference-k8s.md | 84 +- .../dynamo-graph-deployment.mdx | 4 +- .../kubernetes-api/full-api-reference.mdx | 120 +-- .../scripts/tests/test_gen_kubernetes_api.py | 4 +- 53 files changed, 1943 insertions(+), 2788 deletions(-) delete mode 100644 deploy/operator/internal/controller/lpx/attempt_retirement.go delete mode 100644 deploy/operator/internal/controller/lpx/attempt_retirement_test.go create mode 100644 deploy/operator/internal/controller/lpx/deadline_retirement.go create mode 100644 deploy/operator/internal/controller/lpx/deadline_retirement_test.go diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 50da73799674..13d6c673c577 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -193,25 +193,9 @@ DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions DynamoGraphDeploymentStatus.lpx.modelDownload.builds DynamoGraphDeploymentStatus.lpx.modelDownload.lastCheckedAt -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.deadlineAt -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.disarmedAt -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.exceededAt -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.observedGeneration -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.podCliqueSetUID -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.attemptDigest -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.name -DynamoGraphDeploymentStatus.lpx.placement.lpxAttempt.requests.uid DynamoGraphDeploymentStatus.lpx.placement.score DynamoGraphDeploymentStatus.lpx.placement.state DynamoGraphDeploymentStatus.observedGeneration -DynamoGraphDeploymentStatus.placement.lpxAttempt.deadlineAt -DynamoGraphDeploymentStatus.placement.lpxAttempt.disarmedAt -DynamoGraphDeploymentStatus.placement.lpxAttempt.exceededAt -DynamoGraphDeploymentStatus.placement.lpxAttempt.observedGeneration -DynamoGraphDeploymentStatus.placement.lpxAttempt.podCliqueSetUID -DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.attemptDigest -DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.name -DynamoGraphDeploymentStatus.placement.lpxAttempt.requests.uid DynamoGraphDeploymentStatus.placement.score DynamoGraphDeploymentStatus.placement.state DynamoGraphDeploymentStatus.restart.inProgress diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 2452284a9606..e8e98e8714c0 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -743,10 +743,6 @@ func ConvertFromPlacementStatus(src *PlacementStatus, dst *v1beta1.PlacementStat if src.Score != nil { dst.Score = ptr.To(*src.Score) } - if src.LPXAttempt != nil { - dst.LPXAttempt = &v1beta1.LPXAttemptStatus{} - ConvertFromLPXAttemptStatus(src.LPXAttempt, dst.LPXAttempt) - } } // ConvertToPlacementStatus converts placement status from v1beta1 to v1alpha1. @@ -755,60 +751,6 @@ func ConvertToPlacementStatus(src *v1beta1.PlacementStatus, dst *PlacementStatus if src.Score != nil { dst.Score = ptr.To(*src.Score) } - if src.LPXAttempt != nil { - dst.LPXAttempt = &LPXAttemptStatus{} - ConvertToLPXAttemptStatus(src.LPXAttempt, dst.LPXAttempt) - } -} - -// ConvertFromLPXAttemptStatus converts LPX attempt status from v1alpha1 to v1beta1. -func ConvertFromLPXAttemptStatus(src *LPXAttemptStatus, dst *v1beta1.LPXAttemptStatus) { - *dst = v1beta1.LPXAttemptStatus{ - ObservedGeneration: src.ObservedGeneration, - PodCliqueSetUID: src.PodCliqueSetUID, - Requests: make([]v1beta1.LPXAttemptRequestStatus, len(src.Requests)), - } - // Copy timestamp pointers before converting each request structurally. - dst.DeadlineAt = src.DeadlineAt.DeepCopy() - dst.ExceededAt = src.ExceededAt.DeepCopy() - dst.DisarmedAt = src.DisarmedAt.DeepCopy() - for i := range src.Requests { - ConvertFromLPXAttemptRequestStatus(&src.Requests[i], &dst.Requests[i]) - } -} - -// ConvertToLPXAttemptStatus converts LPX attempt status from v1beta1 to v1alpha1. -func ConvertToLPXAttemptStatus(src *v1beta1.LPXAttemptStatus, dst *LPXAttemptStatus) { - *dst = LPXAttemptStatus{ - ObservedGeneration: src.ObservedGeneration, - PodCliqueSetUID: src.PodCliqueSetUID, - Requests: make([]LPXAttemptRequestStatus, len(src.Requests)), - } - // Copy timestamp pointers before converting each request structurally. - dst.DeadlineAt = src.DeadlineAt.DeepCopy() - dst.ExceededAt = src.ExceededAt.DeepCopy() - dst.DisarmedAt = src.DisarmedAt.DeepCopy() - for i := range src.Requests { - ConvertToLPXAttemptRequestStatus(&src.Requests[i], &dst.Requests[i]) - } -} - -// ConvertFromLPXAttemptRequestStatus converts one request from v1alpha1 to v1beta1. -func ConvertFromLPXAttemptRequestStatus(src *LPXAttemptRequestStatus, dst *v1beta1.LPXAttemptRequestStatus) { - *dst = v1beta1.LPXAttemptRequestStatus{ - Name: src.Name, - AttemptDigest: src.AttemptDigest, - UID: src.UID, - } -} - -// ConvertToLPXAttemptRequestStatus converts one request from v1beta1 to v1alpha1. -func ConvertToLPXAttemptRequestStatus(src *v1beta1.LPXAttemptRequestStatus, dst *LPXAttemptRequestStatus) { - *dst = LPXAttemptRequestStatus{ - Name: src.Name, - AttemptDigest: src.AttemptDigest, - UID: src.UID, - } } // ConvertFromRestartStatus converts restart status from v1alpha1 to v1beta1. diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 4f228b00aafb..20fd2450aabe 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -22,7 +22,6 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -82,28 +81,6 @@ type PlacementStatus struct { // State indicates placement score reporting state. // +optional State PlacementScoreState `json:"state,omitempty"` - // LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - // +optional - LPXAttempt *LPXAttemptStatus `json:"lpxAttempt,omitempty"` -} - -// LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. -type LPXAttemptStatus struct { - ObservedGeneration int64 `json:"observedGeneration"` - PodCliqueSetUID types.UID `json:"podCliqueSetUID,omitempty"` - DeadlineAt *metav1.Time `json:"deadlineAt,omitempty"` - ExceededAt *metav1.Time `json:"exceededAt,omitempty"` - DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` - // +listType=map - // +listMapKey=name - Requests []LPXAttemptRequestStatus `json:"requests"` -} - -// LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. -type LPXAttemptRequestStatus struct { - Name string `json:"name"` - AttemptDigest string `json:"attemptDigest"` - UID types.UID `json:"uid,omitempty"` } // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index 5645edfccad1..5a4beaf83296 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -96,7 +96,7 @@ func TestDynamoGraphDeploymentRoleShapesRoundTrip(t *testing.T) { func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { checkedAt := metav1.NewTime(time.Unix(1_800_000_000, 0)) - deadlineAt := metav1.NewTime(time.Unix(1_800_000_900, 0)) + placementScore := 0.75 src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "lpx", Namespace: "ns"}, Spec: v1beta1.DynamoGraphDeploymentSpec{ @@ -119,14 +119,10 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { Builds: []string{"gs://models/draft", "gs://models/target"}, LastCheckedAt: &checkedAt, }, - Placement: &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ - ObservedGeneration: 2, - PodCliqueSetUID: "pcs-uid", - DeadlineAt: &deadlineAt, - Requests: []v1beta1.LPXAttemptRequestStatus{{ - Name: "request", AttemptDigest: "sha256:attempt", UID: "request-uid", - }}, - }}, + Placement: &v1beta1.PlacementStatus{ + Score: &placementScore, + State: v1beta1.PlacementScoreStateReported, + }, }, }, } diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index 04c6ee255294..e89e89ac4fb4 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -36,8 +36,7 @@ type LPXGraphDeploymentStatus struct { // modelDownload retains the existing remote-build download progress. // +optional ModelDownload *v1beta1.ModelDownloadStatus `json:"modelDownload,omitempty"` - // placement retains the scheduling attempt fence, progress, and deadline. - // Its observed generation belongs to this child, not the source DGD. + // placement retains scheduler placement progress. // +optional Placement *v1beta1.PlacementStatus `json:"placement,omitempty"` } diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 51f864cbeed6..22a4e1faa4cc 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1368,53 +1368,6 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LPXAttemptRequestStatus) DeepCopyInto(out *LPXAttemptRequestStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptRequestStatus. -func (in *LPXAttemptRequestStatus) DeepCopy() *LPXAttemptRequestStatus { - if in == nil { - return nil - } - out := new(LPXAttemptRequestStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LPXAttemptStatus) DeepCopyInto(out *LPXAttemptStatus) { - *out = *in - if in.DeadlineAt != nil { - in, out := &in.DeadlineAt, &out.DeadlineAt - *out = (*in).DeepCopy() - } - if in.ExceededAt != nil { - in, out := &in.ExceededAt, &out.ExceededAt - *out = (*in).DeepCopy() - } - if in.DisarmedAt != nil { - in, out := &in.DisarmedAt, &out.DisarmedAt - *out = (*in).DeepCopy() - } - if in.Requests != nil { - in, out := &in.Requests, &out.Requests - *out = make([]LPXAttemptRequestStatus, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptStatus. -func (in *LPXAttemptStatus) DeepCopy() *LPXAttemptStatus { - if in == nil { - return nil - } - out := new(LPXAttemptStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXGraphDeployment) DeepCopyInto(out *LPXGraphDeployment) { *out = *in @@ -1622,11 +1575,6 @@ func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { *out = new(float64) **out = **in } - if in.LPXAttempt != nil { - in, out := &in.LPXAttempt, &out.LPXAttempt - *out = new(LPXAttemptStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index a26d03cc80ca..e3fea1966413 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -731,10 +731,6 @@ type PlacementStatus struct { // for the semantics of each value. // +optional State PlacementScoreState `json:"state,omitempty"` - - // lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - // +optional - LPXAttempt *LPXAttemptStatus `json:"lpxAttempt,omitempty"` } // RestartPhase enumerates phases of a graph-level restart. diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index c45ee4349fa7..4ec912d37609 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -6,7 +6,6 @@ package v1beta1 import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) // LPXConfig identifies the component's compiled model and runtime settings. @@ -23,7 +22,7 @@ type LPXConfig struct { // SchedulingSpec configures LPX scheduling attempts. type SchedulingSpec struct { - // attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + // attemptDeadlineSeconds limits how long each LPR may remain pending. // Omission means unlimited; the value is not a solver budget. // +optional // +kubebuilder:validation:Minimum=1 @@ -31,25 +30,6 @@ type SchedulingSpec struct { AttemptDeadlineSeconds *int64 `json:"attemptDeadlineSeconds,omitempty"` } -// LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. -type LPXAttemptStatus struct { - ObservedGeneration int64 `json:"observedGeneration"` - PodCliqueSetUID types.UID `json:"podCliqueSetUID,omitempty"` - DeadlineAt *metav1.Time `json:"deadlineAt,omitempty"` - ExceededAt *metav1.Time `json:"exceededAt,omitempty"` - DisarmedAt *metav1.Time `json:"disarmedAt,omitempty"` - // +listType=map - // +listMapKey=name - Requests []LPXAttemptRequestStatus `json:"requests"` -} - -// LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. -type LPXAttemptRequestStatus struct { - Name string `json:"name"` - AttemptDigest string `json:"attemptDigest"` - UID types.UID `json:"uid,omitempty"` -} - // DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. // Both served DGD versions use this type, so conversion preserves its live fields natively. type DynamoGraphDeploymentLPXStatus struct { diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index d2fc2c869b67..0e47ef9d84cd 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -1163,53 +1163,6 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LPXAttemptRequestStatus) DeepCopyInto(out *LPXAttemptRequestStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptRequestStatus. -func (in *LPXAttemptRequestStatus) DeepCopy() *LPXAttemptRequestStatus { - if in == nil { - return nil - } - out := new(LPXAttemptRequestStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LPXAttemptStatus) DeepCopyInto(out *LPXAttemptStatus) { - *out = *in - if in.DeadlineAt != nil { - in, out := &in.DeadlineAt, &out.DeadlineAt - *out = (*in).DeepCopy() - } - if in.ExceededAt != nil { - in, out := &in.ExceededAt, &out.ExceededAt - *out = (*in).DeepCopy() - } - if in.DisarmedAt != nil { - in, out := &in.DisarmedAt, &out.DisarmedAt - *out = (*in).DeepCopy() - } - if in.Requests != nil { - in, out := &in.Requests, &out.Requests - *out = make([]LPXAttemptRequestStatus, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXAttemptStatus. -func (in *LPXAttemptStatus) DeepCopy() *LPXAttemptStatus { - if in == nil { - return nil - } - out := new(LPXAttemptStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXConfig) DeepCopyInto(out *LPXConfig) { *out = *in @@ -1363,11 +1316,6 @@ func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { *out = new(float64) **out = **in } - if in.LPXAttempt != nil { - in, out := &in.LPXAttempt, &out.LPXAttempt - *out = new(LPXAttemptStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index b738bf03cda8..d27244bb3a50 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -403,7 +403,7 @@ spec: properties: attemptDeadlineSeconds: description: |- - attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + attemptDeadlineSeconds limits how long each LPR may remain pending. Omission means unlimited; the value is not a solver budget. format: int64 maximum: 9223372036 @@ -20281,53 +20281,6 @@ spec: placement: description: placement contains LPX scheduling progress and placement signals. properties: - lpxAttempt: - description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - properties: - deadlineAt: - format: date-time - type: string - disarmedAt: - format: date-time - type: string - exceededAt: - format: date-time - type: string - observedGeneration: - format: int64 - type: integer - podCliqueSetUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - requests: - items: - description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - properties: - attemptDigest: - type: string - name: - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - attemptDigest - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - observedGeneration - - requests - type: object score: description: |- score is the DGD-level scheduler placement score aggregated from @@ -20360,53 +20313,6 @@ spec: Placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). properties: - lpxAttempt: - description: LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - properties: - deadlineAt: - format: date-time - type: string - disarmedAt: - format: date-time - type: string - exceededAt: - format: date-time - type: string - observedGeneration: - format: int64 - type: integer - podCliqueSetUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - requests: - items: - description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - properties: - attemptDigest: - type: string - name: - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - attemptDigest - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - observedGeneration - - requests - type: object score: description: |- Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] @@ -37995,7 +37901,7 @@ spec: properties: attemptDeadlineSeconds: description: |- - attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. + attemptDeadlineSeconds limits how long each LPR may remain pending. Omission means unlimited; the value is not a solver budget. format: int64 maximum: 9223372036 @@ -38241,53 +38147,6 @@ spec: placement: description: placement contains LPX scheduling progress and placement signals. properties: - lpxAttempt: - description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - properties: - deadlineAt: - format: date-time - type: string - disarmedAt: - format: date-time - type: string - exceededAt: - format: date-time - type: string - observedGeneration: - format: int64 - type: integer - podCliqueSetUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - requests: - items: - description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - properties: - attemptDigest: - type: string - name: - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - attemptDigest - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - observedGeneration - - requests - type: object score: description: |- score is the DGD-level scheduler placement score aggregated from @@ -38320,53 +38179,6 @@ spec: placement groups DGD-level scheduler placement signals (score, reporting state, and any future placement fields). properties: - lpxAttempt: - description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - properties: - deadlineAt: - format: date-time - type: string - disarmedAt: - format: date-time - type: string - exceededAt: - format: date-time - type: string - observedGeneration: - format: int64 - type: integer - podCliqueSetUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - requests: - items: - description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - properties: - attemptDigest: - type: string - name: - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - attemptDigest - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - observedGeneration - - requests - type: object score: description: |- score is the DGD-level scheduler placement score aggregated from diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index 976e87cf15e5..d2133af7a132 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -256,57 +256,8 @@ spec: format: int64 type: integer placement: - description: |- - placement retains the scheduling attempt fence, progress, and deadline. - Its observed generation belongs to this child, not the source DGD. + description: placement retains scheduler placement progress. properties: - lpxAttempt: - description: lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. - properties: - deadlineAt: - format: date-time - type: string - disarmedAt: - format: date-time - type: string - exceededAt: - format: date-time - type: string - observedGeneration: - format: int64 - type: integer - podCliqueSetUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - requests: - items: - description: LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - properties: - attemptDigest: - type: string - name: - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - attemptDigest - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - observedGeneration - - requests - type: object score: description: |- score is the DGD-level scheduler placement score aggregated from diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index beefd7c9da21..63318fa8c0df 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -43,9 +43,6 @@ func mergeLPXChildStatus( ModelDownload: child.Status.ModelDownload.DeepCopy(), Placement: child.Status.Placement.DeepCopy(), } - if status.Placement != nil && status.Placement.LPXAttempt != nil { - status.Placement.LPXAttempt.ObservedGeneration = source.Generation - } for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { ordinary.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 423ccd67e30f..97b4f4f2dfd8 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -389,11 +389,9 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) require.Equal(t, source.Status, storedSource.Status) - t.Log("An empty scheduling batch must persist as an array, not null") - stored.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: stored.Generation}} - require.True(t, apierrors.IsInvalid(env.Client().Status().Update(t.Context(), stored))) - stored.Status.Placement.LPXAttempt.Requests = []v1beta1.LPXAttemptRequestStatus{} + t.Log("Placement status persists independently of request deadlines") + stored.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} require.NoError(t, env.Client().Status().Update(t.Context(), stored)) require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(stored), stored)) - require.Equal(t, []v1beta1.LPXAttemptRequestStatus{}, stored.Status.Placement.LPXAttempt.Requests) + require.Equal(t, 0.75, *stored.Status.Placement.Score) } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 07d125b6c4ea..c41c4304454b 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "testing" - "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -221,10 +220,8 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), }} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"build"}} - child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ - ObservedGeneration: child.Generation, - Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "request", UID: "request-uid"}}, - }} + placementScore := 0.75 + child.Status.Placement = &v1beta1.PlacementStatus{Score: &placementScore, State: v1beta1.PlacementScoreStateReported} meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4)), Ready: true}}} status := v1beta1.DynamoGraphDeploymentStatus{} @@ -234,9 +231,9 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) require.Equal(t, status.LPX.Placement, status.Placement) status.LPX.ModelDownload.Builds[0] = "parent-copy" - status.LPX.Placement.LPXAttempt.Requests[0].UID = "parent-copy" + *status.LPX.Placement.Score = 0.25 require.Equal(t, "build", child.Status.ModelDownload.Builds[0]) - require.Equal(t, "request-uid", string(child.Status.Placement.LPXAttempt.Requests[0].UID)) + require.Equal(t, placementScore, *child.Status.Placement.Score) t.Log("Reject missing or stale children and stale Ready conditions") for _, failure := range []string{"missing-child", "generation", "condition"} { @@ -316,7 +313,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"stale-build"}} - child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: child.Generation}} + child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "PublicationDenied", Message: "Check the namespace quota"}} result := ReconcileResult{State: v1beta1.DGDStateSuccessful} status := v1beta1.DynamoGraphDeploymentStatus{LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ @@ -433,8 +430,8 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { } } -func TestLPXHandoffOrdinaryScalingPreservesTheEntireAttempt(t *testing.T) { - t.Log("Seed the shared child's current readiness, published request identity and deadline") +func TestLPXHandoffOrdinaryScalingPreservesPlacement(t *testing.T) { + t.Log("Seed the shared child's current readiness and placement") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-specdecode") source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1)), @@ -442,11 +439,7 @@ func TestLPXHandoffOrdinaryScalingPreservesTheEntireAttempt(t *testing.T) { ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:0"}}}}, }) - deadline := metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(time.Minute)) - child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ - ObservedGeneration: child.Generation, DeadlineAt: &deadline, - Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "published-request", UID: "request-uid"}}, - }} + child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Status.ObservedGeneration = child.Generation child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ "lpx": {Replicas: 1, AvailableReplicas: ptr.To(int32(1)), Ready: true}, @@ -473,16 +466,14 @@ func TestLPXHandoffOrdinaryScalingPreservesTheEntireAttempt(t *testing.T) { require.Equal(t, v1beta1.DGDStateSuccessful, projected.Status.State) require.True(t, meta.IsStatusConditionTrue(projected.Status.Conditions, "Ready")) require.Equal(t, source.Generation, projected.Status.ObservedGeneration) - require.Equal(t, source.Generation, projected.Status.LPX.Placement.LPXAttempt.ObservedGeneration) require.Equal(t, beforeChild.Status.Placement, current.Status.Placement) - require.True(t, deadline.Equal(projected.Status.LPX.Placement.LPXAttempt.DeadlineAt)) + require.Equal(t, beforeChild.Status.Placement, projected.Status.LPX.Placement) require.Equal(t, beforeSource, source) source.Status = projected.Status } require.Equal(t, int64(8), source.Generation) - require.Equal(t, int64(3), beforeChild.Status.Placement.LPXAttempt.ObservedGeneration) - t.Log("Scale prefill without changing the child or its durable attempt") + t.Log("Scale prefill without changing the child or its placement") source.GetComponentByName("prefill").Replicas = ptr.To(int32(4)) source.Generation++ current, err := handoff.Reconcile(t.Context(), source) @@ -882,7 +873,7 @@ func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}, LastCheckedAt: &checkedAt}, - Placement: &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: 1}}, + Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported}, }, }, } @@ -911,7 +902,6 @@ func TestOrdinaryDGDClearsMirroredLPXPlacement(t *testing.T) { t.Log("Seed an ordinary DGD with the same compatibility projection at both placement paths") placement := &v1beta1.PlacementStatus{ Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported, - LPXAttempt: &v1beta1.LPXAttemptStatus{ObservedGeneration: 1}, } source := &v1beta1.DynamoGraphDeployment{ Status: v1beta1.DynamoGraphDeploymentStatus{ diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index 751f9018c983..2139939c106c 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -7,15 +7,12 @@ package lpx import ( "context" - "fmt" - "sort" "time" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -24,42 +21,28 @@ import ( ) const ( - lpxAttemptPreparedReason string = "LPXAttemptPrepared" - lpxSchedulingAttemptActiveReason string = "LPXSchedulingAttemptActive" - lpxSchedulingDispositionObservedReason string = "LPXSchedulingDispositionObserved" - lpxSchedulingDeadlineExceededReason string = "LPXSchedulingDeadlineExceeded" - lpxAttemptAuthorityLostReason string = "LPXAttemptAuthorityLost" + lpxSchedulingFailedCondition = "SchedulingFailed" + lpxSchedulingDeadlineExceededReason = "LPXSchedulingDeadlineExceeded" ) -type lpxDeadlineTransition struct { - result reconcileOutcome - attempt *nvidiacomv1beta1.LPXAttemptStatus - requeueAfter time.Duration +type lpxDeadlineExceeded struct { + requeueAfter time.Duration + recordFailure bool } -func (*lpxDeadlineTransition) lpxClassification() {} +func (*lpxDeadlineExceeded) lpxClassification() {} -func newLPXDeadlineTransition(attempt *nvidiacomv1beta1.LPXAttemptStatus, state nvidiacomv1beta1.DGDState, reason string, message string, requeueAfter time.Duration) *lpxDeadlineTransition { - return &lpxDeadlineTransition{ - result: reconcileOutcome{State: state, Reason: reason, Message: message}, - attempt: attempt, - requeueAfter: requeueAfter, - } -} - -// completeLPXDeadline preserves successful deadline wakes and converts active-deadline errors into bounded retries. +// completeLPXDeadline preserves deadline wakes and converts errors before an +// active deadline into bounded retries. func completeLPXDeadline( ctx context.Context, deadlineAt time.Time, result ctrl.Result, reconcileErr error, ) (ctrl.Result, error) { - // Preserve ordinary results when no scheduling deadline is active. if deadlineAt.IsZero() { return result, reconcileErr } - - // Record failures and bound completion by the remaining scheduling deadline. if reconcileErr != nil { log.FromContext(ctx).Error(reconcileErr, "retrying reconciliation under the LPX scheduling deadline") } @@ -79,373 +62,174 @@ func completeLPXDeadline( return result, nil } -func (r *graphReconciler) reconcileLPXAttemptDeadline( +// reconcileLPXRequestDeadlines derives every scheduling clock from the start +// of the corresponding desired LPR's current scheduler-owned scheduling cycle. +// pcs is nil when no current PodCliqueSet was observed, which leaves no active +// clock. +func (r *graphReconciler) reconcileLPXRequestDeadlines( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, -) (lpxClassification, time.Time, []lpxv1alpha1.LPUPipelineRequest, error) { - // Source edits do not reset a scheduling batch's absolute deadline. - attempt := currentLPXAttemptStatus(deployment) - deadlineSeconds := lpxAttemptDeadlineSeconds(source) - if attempt == nil { - return nil, time.Time{}, nil, nil - } - deadlineAt := time.Time{} - if attempt.DeadlineAt != nil { - deadlineAt = attempt.DeadlineAt.Time - } - requests, err := r.listOwnedLPXRequests(ctx, deployment) + pcs *grovev1alpha1.PodCliqueSet, + desired []lpxModelMaterializing, +) (lpxClassification, time.Time, error) { + deadlineSeconds := lpxRequestDeadlineSeconds(source) + if deadlineSeconds == nil || pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) { + return nil, time.Time{}, nil + } + requests, err := r.listOwnedLPXRequests(ctx, deployment, pcs) if err != nil { - return nil, deadlineAt, nil, err + return nil, time.Time{}, err } - observed := lpxRequestsByName(requests) - if updateLPXAttemptFromLive(deadlineSeconds, attempt, observed) { - if attempt.DeadlineAt != nil { - deadlineAt = attempt.DeadlineAt.Time - } - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, - "Persisted the exact LPX request identity and immutable scheduling deadline", time.Nanosecond, - ), deadlineAt, nil, nil - } - if attempt.ExceededAt != nil { - state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, requests, lpxSchedulingDeadlineExceededReason, - "LPX scheduling deadline exceeded; no retry is authorized", - ) - return state, time.Time{}, nil, err - } - active, disposition, missing := exactLPXAttemptState(attempt, observed) - if attempt.DisarmedAt != nil && !missing && !active { - return nil, time.Time{}, requests, nil - } - now := time.Now() - // Completion is authoritative even when the status write lags a Bound observation. - if disposition { - observedAt := metav1.NewTime(now) - attempt.DisarmedAt = &observedAt - return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStatePending, - lpxSchedulingDispositionObservedReason, "Recorded the scheduling batch disposition", time.Nanosecond), time.Time{}, requests, nil + desiredNames := make(map[string]struct{}, len(desired)) + for index := range desired { + desiredNames[desired[index].requestName] = struct{}{} } - if attempt.DeadlineAt != nil && (attempt.DisarmedAt == nil || !missing) && !now.Before(attempt.DeadlineAt.Time) { - decision := metav1.NewTime(now) - attempt.ExceededAt = &decision - attempt.DisarmedAt = nil - attempt.ObservedGeneration = deployment.Generation - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStateFailed, lpxSchedulingDeadlineExceededReason, - "LPX scheduling deadline exceeded; persisting expiry before exact cleanup", time.Nanosecond, - ), attempt.DeadlineAt.Time, nil, nil - } - if missing { - state, err := r.continueTerminalLPXAttempt( - ctx, deployment, attempt, requests, lpxAttemptAuthorityLostReason, - "An exact LPX request disappeared; retiring its scheduling batch without replacement", - ) - return state, time.Time{}, nil, err - } - if attempt.DeadlineAt == nil { - return nil, time.Time{}, requests, nil - } - if attempt.DisarmedAt != nil { - attempt.DisarmedAt = nil - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStatePending, lpxSchedulingAttemptActiveReason, - "LPX scheduling resumed under the original deadline", time.Nanosecond, - ), attempt.DeadlineAt.Time, nil, nil - } - return nil, deadlineAt, requests, nil -} -// retireLPXEngineRequest records intentional removal before deleting the -// request's owning PCS. -func (r *graphReconciler) retireLPXEngineRequest(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest, reason string, intentional bool) (lpxClassification, error) { - // Persist batch membership before deletion can be mistaken for lost authority. - if current := currentLPXAttemptStatus(deployment); current != nil && intentional && current.ExceededAt == nil { - for i, row := range current.Requests { - if row.Name == request.Name && row.AttemptDigest != "" && (row.UID == request.UID || row.UID == "" && row.AttemptDigest == request.Annotations[lpxAttemptDigestAnnotation]) { - next := current.DeepCopy() - // Deleting the shared PCS replaces every request it owns. - next.Requests[i].UID = request.UID - for j := range next.Requests { - next.Requests[j].AttemptDigest = "" - } - return newLPXDeadlineTransition(next, nvidiacomv1beta1.DGDStatePending, - lpxRetiringReason, reason, time.Nanosecond), nil - } + // Expire each desired nonterminal request independently and retain the earliest future wake. + now := time.Now() + duration := time.Duration(*deadlineSeconds) * time.Second + expired := make([]*lpxv1alpha1.LPUPipelineRequest, 0) + var nextDeadline time.Time + for index := range requests { + request := &requests[index] + if _, current := desiredNames[request.Name]; !current { + continue } - } - owner := metav1.GetControllerOf(request) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != "PodCliqueSet" || owner.UID == "" { - return nil, fmt.Errorf("LPX request %q does not have an exact PodCliqueSet controller owner", request.Name) - } - if _, err := r.deleteLPXPodCliqueSet(ctx, deployment, owner.Name, owner.UID); err != nil { - return nil, err - } - return &lpxRetiring{retirementReason: reason}, nil -} - -func (r *graphReconciler) reconcileLPXAttemptPreparation( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - deadlineSeconds *int64, - want *nvidiacomv1beta1.LPXAttemptStatus, - currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) *lpxDeadlineTransition { - if want == nil { - return nil - } - // New engines join an active batch, but never a completed batch awaiting disarm. - current := currentLPXAttemptStatus(deployment) - if current != nil && current.ExceededAt != nil { - return newLPXDeadlineTransition(current, nvidiacomv1beta1.DGDStateFailed, - lpxSchedulingDeadlineExceededReason, "LPX scheduling deadline exceeded; waiting for cleanup and a later edit", lpxRetirementRequeueAfter) - } - next := want.DeepCopy() - previous := make(map[string]nvidiacomv1beta1.LPXAttemptRequestStatus) - completed := false - if current != nil { - var missing bool - _, completed, missing = exactLPXAttemptState(current, currents) - active := !completed && (current.DisarmedAt == nil || missing) - if active { - next = current.DeepCopy() + if !request.DeletionTimestamp.IsZero() || lpxRequestHasTerminalSchedulingDisposition(request) { + continue } - - for _, row := range current.Requests { - if active || row.UID != "" && row.AttemptDigest == "" { - previous[row.Name] = row - } + startedAt, known := lpxSchedulingStartedAt(request) + if !known { + continue } - // Move the preserved attempt clock to its replacement PCS once every old request is gone. - if active && lpxAttemptRequestsRetired(current.Requests) { - next.ObservedGeneration = want.ObservedGeneration - next.PodCliqueSetUID = want.PodCliqueSetUID + deadline := startedAt.Add(duration) + if !now.Before(deadline) { + expired = append(expired, request) + continue } - } - next.Requests = make([]nvidiacomv1beta1.LPXAttemptRequestStatus, 0, len(want.Requests)) - // Desired rows are already sorted; only newly joining engines need disposition checks. - for _, row := range want.Requests { - if old, tracked := previous[row.Name]; tracked { - if old.UID != "" || row.AttemptDigest == "" { - row = old - } - } else if request := currents[row.Name]; request != nil && request.DeletionTimestamp.IsZero() { - if row.AttemptDigest == "" { - row.AttemptDigest = request.Annotations[lpxAttemptDigestAnnotation] - } - switch state := classifyPublishedLPX(request).(type) { - case *lpxBound: - continue - case *lpxSchedulerObserved: - if state.Phase == lpxv1alpha1.RequestPhaseNoFit || state.Phase == lpxv1alpha1.RequestPhaseUnsupported { - continue - } - } + if nextDeadline.IsZero() || deadline.Before(nextDeadline) { + nextDeadline = deadline } - next.Requests = append(next.Requests, row) } - // Cancel the last pending engine durably; replacement placeholders keep their clock. - if len(next.Requests) == 0 { - if current == nil || completed { - return nil - } - next = current.DeepCopy() - next.Requests = []nvidiacomv1beta1.LPXAttemptRequestStatus{} - if next.DisarmedAt == nil { - disarmedAt := metav1.Now() - next.DisarmedAt = &disarmedAt - } + if len(expired) == 0 { + return nil, nextDeadline, nil } - updateLPXAttemptFromLive(deadlineSeconds, next, currents) - if apiequality.Semantic.DeepEqual(current, next) { - return nil + + // Persist retry authorization before cleanup removes the request that proves expiry. + if !lpxDeadlineFailureCovers(deployment, expired) { + return &lpxDeadlineExceeded{requeueAfter: time.Nanosecond, recordFailure: true}, time.Time{}, nil } - return newLPXDeadlineTransition( - next, nvidiacomv1beta1.DGDStatePending, lpxAttemptPreparedReason, - "Prepared the scheduling batch before publication", time.Nanosecond, - ) + + // Lower Grove before deleting the affected engine suffix; convergence is asynchronous. + retirementErr := r.retireExpiredLPXRequests(ctx, deployment, source, pcs, requests, expired) + return &lpxDeadlineExceeded{requeueAfter: lpxRetirementRequeueAfter}, time.Time{}, retirementErr } -func lpxAttemptRequestsRetired(requests []nvidiacomv1beta1.LPXAttemptRequestStatus) bool { - if len(requests) == 0 { +func lpxRequestHasTerminalSchedulingDisposition(request *lpxv1alpha1.LPUPipelineRequest) bool { + if request.Status == nil || request.Status.ObservedGeneration == nil || + *request.Status.ObservedGeneration != request.Generation { return false } - for _, request := range requests { - if request.UID != "" || request.AttemptDigest != "" { - return false - } + switch request.Status.Phase { + case lpxv1alpha1.RequestPhaseBound, lpxv1alpha1.RequestPhaseNoFit, lpxv1alpha1.RequestPhaseUnsupported: + return true + case lpxv1alpha1.RequestPhaseDegraded, lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: + committed := request.Status.Committed + return committed != nil && committed.Execution.AcceptedGeneration != nil && + *committed.Execution.AcceptedGeneration == request.Generation + default: + return false } - return true } -// revalidateLPXAttemptPublication fences live authority after preparation has -// established the current nonnil attempt and an unpublished desired request. -func (r *graphReconciler) revalidateLPXAttemptPublication(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, pcsName string) error { - attempt := currentLPXAttemptStatus(deployment) - if attempt.ExceededAt != nil || attempt.DisarmedAt != nil { - return fmt.Errorf("LPX publication is forbidden after a durable attempt decision") +// lpxSchedulingStartedAt requires a nonnil request and reports whether the +// scheduler published a usable start for its current scheduling cycle. +func lpxSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) (time.Time, bool) { + if request.Status == nil || request.Status.SchedulingStartedAt == nil || request.Status.SchedulingStartedAt.IsZero() { + return time.Time{}, false } - requests, err := r.listOwnedLPXRequests(ctx, deployment) - if err != nil { - return err - } - if _, _, missing := exactLPXAttemptState(attempt, lpxRequestsByName(requests)); missing { - return fmt.Errorf("recorded LPX request authority changed before sibling publication") - } - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { - return err - } - if pcs.UID != attempt.PodCliqueSetUID || !pcs.DeletionTimestamp.IsZero() { - return fmt.Errorf("PodCliqueSet authority changed before LPX publication") - } - if attempt.DeadlineAt != nil && !time.Now().Before(attempt.DeadlineAt.Time) { - return fmt.Errorf("LPX scheduling deadline elapsed before sibling publication") - } - return nil + return request.Status.SchedulingStartedAt.Time, true } -func lpxAttemptDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) *int64 { +func lpxRequestDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) *int64 { if source == nil || source.Spec.Scheduling == nil { return nil } return source.Spec.Scheduling.AttemptDeadlineSeconds } -// desiredLPXAttemptStatus projects desired for its reconcile-local LPXGraphDeployment generation; desired must be nonnil. -func desiredLPXAttemptStatus(desired *lpxMaterializing, observedGeneration int64, pcsUID types.UID) *nvidiacomv1beta1.LPXAttemptStatus { - requests := make([]nvidiacomv1beta1.LPXAttemptRequestStatus, len(desired.requests)) - for i := range desired.requests { - requests[i] = nvidiacomv1beta1.LPXAttemptRequestStatus{ - Name: desired.requests[i].requestName, AttemptDigest: desired.requests[i].attemptDigest, - } - } - sort.Slice(requests, func(i, j int) bool { return requests[i].Name < requests[j].Name }) - return &nvidiacomv1beta1.LPXAttemptStatus{ - ObservedGeneration: observedGeneration, PodCliqueSetUID: pcsUID, Requests: requests, - } +// lpxDeadlineFailureCurrent reports whether the current generation must remain +// failed until a later input revision explicitly authorizes another publication. +func lpxDeadlineFailureCurrent(deployment *nvidiacomv1alpha1.LPXGraphDeployment) bool { + failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) + return lpxDeadlineFailureRecorded(deployment) && + failed.ObservedGeneration >= deployment.Generation } -func currentLPXAttemptStatus(deployment *nvidiacomv1alpha1.LPXGraphDeployment) *nvidiacomv1beta1.LPXAttemptStatus { - if deployment == nil || deployment.Status.Placement == nil { - return nil - } - return deployment.Status.Placement.LPXAttempt +func lpxDeadlineFailureRecorded(deployment *nvidiacomv1alpha1.LPXGraphDeployment) bool { + failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) + return failed != nil && failed.Status == metav1.ConditionTrue && failed.Reason == lpxSchedulingDeadlineExceededReason } -// updateLPXAttemptFromLive incorporates server-persisted requests into the non-nil, -// reconcile-owned attempt and reports whether a status transition is needed. -func updateLPXAttemptFromLive( - deadlineSeconds *int64, - attempt *nvidiacomv1beta1.LPXAttemptStatus, - observed map[string]*lpxv1alpha1.LPUPipelineRequest, +// lpxDeadlineFailureCovers reports whether every expired scheduling cycle +// predates the recorded failure. A later cycle needs its own durable failure +// before cleanup. +func lpxDeadlineFailureCovers( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + requests []*lpxv1alpha1.LPUPipelineRequest, ) bool { - changed := false - var firstPublishedAt time.Time - for i := range attempt.Requests { - row := &attempt.Requests[i] - if row.AttemptDigest == "" { - continue - } - request := observed[row.Name] - if request == nil { - continue - } - if row.UID == "" { - if request.Annotations[lpxAttemptDigestAnnotation] != row.AttemptDigest { - continue - } - changed = true - row.UID = request.UID - } else if request.UID != row.UID { - continue - } - if attempt.PodCliqueSetUID == "" { - changed = true - attempt.PodCliqueSetUID = types.UID(request.Annotations[lpxPCSUIDAnnotation]) - } - if attempt.DeadlineAt == nil && - (firstPublishedAt.IsZero() || request.CreationTimestamp.Time.Before(firstPublishedAt)) { - firstPublishedAt = request.CreationTimestamp.Time - } + failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) + if failed == nil || failed.Status != metav1.ConditionTrue || failed.Reason != lpxSchedulingDeadlineExceededReason { + return false } - if attempt.DeadlineAt == nil && deadlineSeconds != nil && !firstPublishedAt.IsZero() { - deadline := metav1.NewTime(firstPublishedAt.Add( - time.Duration(*deadlineSeconds) * time.Second, - )) - changed = true - attempt.DeadlineAt = &deadline + if failed.ObservedGeneration < deployment.Generation { + return false } - return changed -} - -func lpxRequestsByName(requests []lpxv1alpha1.LPUPipelineRequest) map[string]*lpxv1alpha1.LPUPipelineRequest { - byName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(requests)) - for i := range requests { - byName[requests[i].Name] = &requests[i] + if failed.LastTransitionTime.IsZero() { + return false } - return byName -} - -// exactLPXAttemptState reports aggregate request state; attempt must be nonnil. -func exactLPXAttemptState( - attempt *nvidiacomv1beta1.LPXAttemptStatus, - observed map[string]*lpxv1alpha1.LPUPipelineRequest, -) (active bool, disposition bool, missing bool) { - disposition = len(attempt.Requests) > 0 - for i := range attempt.Requests { - // Distinguish unpublished rows from recorded request authority loss. - row := &attempt.Requests[i] - request := observed[row.Name] - if row.AttemptDigest == "" || row.UID == "" { - disposition = false - continue - } - if request == nil || request.UID != row.UID || !request.DeletionTimestamp.IsZero() { - disposition = false - missing = true - continue - } - - // Reuse the canonical publication state after exact row ownership is proven. - switch state := classifyPublishedLPX(request).(type) { - case *lpxBound: - continue - case *lpxSchedulerObserved: - switch state.Phase { - case lpxv1alpha1.RequestPhasePending, lpxv1alpha1.RequestPhasePlanned, - lpxv1alpha1.RequestPhaseReserving, lpxv1alpha1.RequestPhaseBinding: - active = true - case lpxv1alpha1.RequestPhaseNoFit, lpxv1alpha1.RequestPhaseUnsupported: - continue - } + for _, request := range requests { + startedAt, known := lpxSchedulingStartedAt(request) + if !known || !startedAt.Before(failed.LastTransitionTime.Time) { + return false } - disposition = false } - return active, disposition, missing + return true } -func (r *graphReconciler) continueTerminalLPXAttempt( - ctx context.Context, +// projectLPXSchedulingFailureCondition keeps retry authorization independent +// from the aggregate Failed condition, which may report another controller error. +func projectLPXSchedulingFailureCondition( deployment *nvidiacomv1alpha1.LPXGraphDeployment, - attempt *nvidiacomv1beta1.LPXAttemptStatus, - requests []lpxv1alpha1.LPUPipelineRequest, - reason string, - message string, -) (lpxClassification, error) { - pending, err := r.retireLPXAttempt(ctx, deployment, attempt, requests) - if err != nil { - return nil, err - } - if !pending && attempt.ExceededAt != nil && deployment.Generation > attempt.ObservedGeneration { - return newLPXDeadlineTransition(nil, nvidiacomv1beta1.DGDStatePending, - lpxAttemptPreparedReason, "The failed batch is gone; a later edit permits scheduling", time.Nanosecond), nil - } - requeueAfter := time.Duration(0) - if pending { - requeueAfter = lpxRetirementRequeueAfter - } - return newLPXDeadlineTransition( - attempt, nvidiacomv1beta1.DGDStateFailed, reason, message, requeueAfter, - ), nil + previous *nvidiacomv1alpha1.LPXGraphDeploymentStatus, + state reconcileOutcome, + recordFailure bool, +) { + prior := meta.FindStatusCondition(previous.Conditions, lpxSchedulingFailedCondition) + if state.Reason == lpxSchedulingDeadlineExceededReason { + observedGeneration := deployment.Generation + if prior != nil && prior.Status == metav1.ConditionTrue && !recordFailure { + observedGeneration = prior.ObservedGeneration + } + if recordFailure { + meta.RemoveStatusCondition(&deployment.Status.Conditions, lpxSchedulingFailedCondition) + } + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, + ObservedGeneration: observedGeneration, Reason: state.Reason, Message: state.Message, + }) + return + } + if prior == nil { + return + } + if deployment.Generation <= prior.ObservedGeneration { + return + } + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionFalse, + ObservedGeneration: deployment.Generation, Reason: "LPXSchedulingRetryAuthorized", + Message: "The expired request was removed and current input may be published", + }) } diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 1c97e05ef19d..6bcf1fd12cda 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -1,643 +1,309 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 package lpx import ( "context" "errors" - "fmt" "testing" "time" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" ) -func TestLPXAttemptIsPreparedBeforePublicationAndCannotBeRecreated(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) +func TestLPXDeadlineUsesEachSchedulingCycleStart(t *testing.T) { + t.Log("Publish two requests created together whose scheduling cycles start at different times") + dgd, source, _ := newLPXTestDGD(t, "single") source.Spec.Scheduling = deadlineTestScheduling() - reconciler := newLPXTestReconciler(t, registry, dgd, source) - desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) - require.Nil(t, rejected) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - - classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) - require.NoError(t, err) - require.Equal(t, lpxAttemptPreparedReason, lpxResult(classification).Reason) - prepared := applyLPXAttemptTransition(t, dgd, classification) - require.NotEmpty(t, prepared.PodCliqueSetUID) - require.Empty(t, ownedLPXRequests(t, ctx, reconciler, dgd)) - - _, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) - require.NoError(t, err) - requests := ownedLPXRequests(t, ctx, reconciler, dgd) - require.Len(t, requests, 1) - require.Empty(t, requests[0].Finalizers) - dgd.Status.Placement = nil - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + pcs := deadlineTestPCS(dgd, "pcs-uid") + firstStarted := time.Now().UTC().Truncate(time.Second) + created := firstStarted.Add(-time.Hour) + first := deadlineTestRequest(dgd, pcs, "first", created, lpxv1alpha1.RequestPhasePending) + first.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(firstStarted)) + second := deadlineTestRequest(dgd, pcs, "second", created, lpxv1alpha1.RequestPhasePending) + second.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(firstStarted.Add(10 * time.Second))) + r := newLPXTestReconciler(t, nil, dgd, source, pcs, first, second) + + t.Log("Wake at the first scheduling cycle's deadline without using object creation time") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(first, second)) require.NoError(t, err) - recovered := applyLPXAttemptTransition(t, dgd, classification) - require.Equal(t, requests[0].UID, recovered.Requests[0].UID, "publication reconciliation recovers the existing request") - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: recovered} + require.Nil(t, classification) + require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) +} - require.NoError(t, reconciler.Delete(ctx, &requests[0])) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) +func TestLPXDeadlineIgnoresRequestsOutsideCurrentPublication(t *testing.T) { + t.Log("Observe one current request and one expired request retired by a source edit") + dgd, source, _ := newLPXTestDGD(t, "single") + source.Spec.Scheduling = deadlineTestScheduling() + pcs := deadlineTestPCS(dgd, "pcs-uid") + started := time.Now().UTC().Truncate(time.Second) + current := deadlineTestRequest(dgd, pcs, "current", started, lpxv1alpha1.RequestPhasePending) + stale := deadlineTestRequest(dgd, pcs, "stale", started.Add(-time.Minute), lpxv1alpha1.RequestPhasePending) + r := newLPXTestReconciler(t, nil, dgd, source, pcs, current, stale) + + t.Log("Keep the current wake without converting stale cleanup into deadline failure") + classification, wake, err := r.reconcileLPXRequestDeadlines( + t.Context(), dgd, source, pcs, deadlineTestDesired(current), + ) require.NoError(t, err) - require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) - - for _, test := range []struct { - name string - deadline, terminating bool - }{ - {name: "absent with deadline", deadline: true}, - {name: "terminating with deadline", deadline: true, terminating: true}, - {name: "absent without deadline"}, - {name: "terminating without deadline", terminating: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Keep ready Grove identities after losing an already-recorded request") - deployment, source := dgd.DeepCopy(), source.DeepCopy() - attempt := currentLPXAttemptStatus(deployment) - attempt.DeadlineAt = nil - if test.deadline { - attempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Hour))) - } else { - source.Spec.Scheduling = nil - attempt.DisarmedAt = ptr.To(metav1.Now()) - } - objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - removed := deadlineTestRequest(deployment, pcs, "removed-engine", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) - objects = append(objects, removed) - attempt.Requests = append(attempt.Requests, deadlineTestAttempt(deployment, removed, time.Now()).Requests...) - if test.terminating { - request := requests[0].DeepCopy() - request.DeletionTimestamp = ptr.To(metav1.Now()) - request.Finalizers = []string{"scheduler.example/cleanup"} - objects = append(objects, request) - } - r := newLPXTestReconciler(t, registry, deployment, source, objects...) - t.Log("Terminal authority loss deletes the PCS before its requests") - var transition *lpxDeadlineTransition - for range 2 { - classification, _, _, err := r.reconcileLPXAttemptDeadline(ctx, deployment, source) - require.NoError(t, err) - require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) - transition = classification.(*lpxDeadlineTransition) - deployment.Status.Placement.LPXAttempt = transition.attempt - } - pcs = &grovev1alpha1.PodCliqueSet{} - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs))) - removed = getLPXRequest(t, ctx, r.Client, deployment.Namespace, "removed-engine") - require.NoError(t, r.Delete(ctx, removed), "emulate garbage collection in the fake client") - live := ownedLPXRequests(t, ctx, r, deployment) - if test.terminating { - require.Len(t, live, 1) - require.Equal(t, requests[0].UID, live[0].UID) - require.False(t, live[0].DeletionTimestamp.IsZero()) - require.Equal(t, lpxRetirementRequeueAfter, transition.requeueAfter) - } else { - require.Empty(t, live) - require.Equal(t, lpxRetirementRequeueAfter, transition.requeueAfter) - } - }) - } + require.Nil(t, classification) + require.True(t, started.Add(30*time.Second).Equal(wake)) } -func TestLPXPreflightReusesActiveDeadlineRequestObservation(t *testing.T) { - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - dgd := newLPXTestDeployment(t, source) +func TestLPXDeadlineWaitsForSchedulingCycleStart(t *testing.T) { + t.Log("Observe an old pending request before the scheduler initializes its cycle timestamp") + dgd, source, _ := newLPXTestDGD(t, "single") source.Spec.Scheduling = deadlineTestScheduling() - deadline := metav1.NewTime(time.Now().Add(time.Minute)) - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: &nvidiacomv1beta1.LPXAttemptStatus{ - ObservedGeneration: dgd.Generation, - DeadlineAt: &deadline, - Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ - Name: "unpublished", AttemptDigest: "sha256:attempt", - }}, - }} - reconciler := newLPXTestReconciler(t, nil, dgd, source) - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - requestLists := 0 - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - List: func( - ctx context.Context, - delegated client.WithWatch, - list client.ObjectList, - opts ...client.ListOption, - ) error { - if _, ok := list.(*lpxv1alpha1.LPUPipelineRequestList); ok { - requestLists++ - } - return delegated.List(ctx, list, opts...) - }, - }) + pcs := deadlineTestPCS(dgd, "pcs-uid") + request := deadlineTestRequest(dgd, pcs, "pending", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending) + request.Status.SchedulingStartedAt = nil + r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - t.Log("Reuse the active deadline request observation during safety preflight") - classification, wake, requests, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - require.Nil(t, classification) - selected, classification, err := reconciler.reconcileLPXSafetyPreflight(t.Context(), dgd, source, requests) + t.Log("Do not infer a deadline from creation time or the current wall clock") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) require.NoError(t, err) - require.Nil(t, selected) require.Nil(t, classification) - require.True(t, wake.Equal(deadline.Time)) - require.Equal(t, 1, requestLists) + require.True(t, wake.IsZero()) } -func TestLPXAttemptPartialPublicationLossFencesEverySibling(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXSpecDecodeTestDGD(t) +func TestLPXDeadlineRestartsWhenReleasedRequestReturnsToPending(t *testing.T) { + t.Log("Observe a previously planned request in a new pending scheduling cycle") + dgd, source, _ := newLPXTestDGD(t, "single") source.Spec.Scheduling = deadlineTestScheduling() - reconciler := newLPXTestReconciler(t, registry, dgd, source) - desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) - require.Nil(t, rejected) - require.Len(t, desired.requests, 3) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - - for range 3 { - classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) - require.NoError(t, err) - applyLPXAttemptTransition(t, dgd, classification) - } - requests := ownedLPXRequests(t, ctx, reconciler, dgd) - require.Len(t, requests, 2, "preparation plus one persisted transition per successful create") - - dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Second))) - require.ErrorContains(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd, desired.plan.PodCliqueSetName), "deadline") - dgd.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Minute))) - require.NoError(t, reconciler.Delete(ctx, &requests[0])) - require.Error(t, reconciler.revalidateLPXAttemptPublication(ctx, dgd, desired.plan.PodCliqueSetName)) - classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) - require.NoError(t, err) - require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) - require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) - applyLPXAttemptTransition(t, dgd, classification) - - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(ctx, dgd, source) + pcs := deadlineTestPCS(dgd, "pcs-uid") + restarted := time.Now().UTC().Truncate(time.Second) + created := restarted.Add(-time.Hour) + request := deadlineTestRequest(dgd, pcs, "pending-again", created, lpxv1alpha1.RequestPhasePending) + request.Status.LastPlanRevision = 1 + request.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(restarted)) + r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) + + t.Log("Deadline the new cycle from the scheduler's reset timestamp") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) require.NoError(t, err) - require.Equal(t, lpxAttemptAuthorityLostReason, lpxResult(classification).Reason) - _, _ = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) - require.LessOrEqual(t, len(ownedLPXRequests(t, ctx, reconciler, dgd)), 1) + require.Nil(t, classification) + require.True(t, restarted.Add(30*time.Second).Equal(wake)) } -func TestLPXAttemptDeadlineRaceUsesOnlyDurableDisposition(t *testing.T) { - t.Log("Seed an exact Bound request whose deadline has not yet been recovered") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - dgd := newLPXTestDeployment(t, source) +func TestLPXTerminalRequestHasNoDeadline(t *testing.T) { + t.Log("Observe a bound request") + dgd, source, _ := newLPXTestDGD(t, "single") source.Spec.Scheduling = deadlineTestScheduling() pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "attempt", time.Now().UTC().Truncate(time.Second), lpxv1alpha1.RequestPhaseBound) - running := deadlineTestRequest(dgd, pcs, "older-engine", time.Now().UTC().Truncate(time.Second).Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) - group := &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "engine-group", Namespace: dgd.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 2}, - } - request.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: 1} - running.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: 0} - request.Finalizers = []string{"scheduling.lpu.nvidia.com/lpx-cleanup"} - attempt := deadlineTestAttempt(dgd, request, time.Now().Add(time.Minute)) - attempt.DeadlineAt, attempt.PodCliqueSetUID = nil, "" - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - reconciler := newLPXTestReconciler(t, nil, dgd, source, request, running, pcs, group) - running = getLPXRequest(t, t.Context(), reconciler.Client, running.Namespace, running.Name) - - t.Log("Recover the native deadline and record the Bound scheduling disposition") - classification, _, _, err := reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - recovered := applyLPXAttemptTransition(t, dgd, classification) - require.True(t, recovered.DeadlineAt.Time.Equal(request.CreationTimestamp.Add(30*time.Second))) - require.Equal(t, types.UID(request.Annotations[lpxPCSUIDAnnotation]), recovered.PodCliqueSetUID) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - disarmed := applyLPXAttemptTransition(t, dgd, classification) - require.NotNil(t, disarmed.DisarmedAt) + request := deadlineTestRequest(dgd, pcs, "bound", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) + r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - t.Log("A durably recorded disposition survives the elapsed deadline") - disarmed.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + t.Log("Terminal scheduler disposition has neither expiry nor a wake") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) require.NoError(t, err) require.Nil(t, classification) + require.True(t, wake.IsZero()) +} - t.Log("Resumed scheduling expires under the original elapsed deadline") - request.Status.Phase = lpxv1alpha1.RequestPhasePending - reconciler = newLPXTestReconciler(t, nil, dgd, source, request, running, pcs, group) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - result := lpxResult(classification) - require.Equal(t, lpxSchedulingDeadlineExceededReason, result.Reason) - transition := classification.(*lpxDeadlineTransition) - require.NotNil(t, transition.attempt.ExceededAt) - require.Nil(t, transition.attempt.DisarmedAt) - - t.Log("Persist expiry before cleaning up the exact scheduling request") - applyLPXAttemptTransition(t, dgd, classification) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) - - t.Log("Scale down the failed engine before deleting its request, preserving the older engine") - _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - storedPCS := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKeyFromObject(pcs) - require.NoError(t, reconciler.Get(t.Context(), pcsKey, storedPCS)) - require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name).DeletionTimestamp.IsZero()) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).DeletionTimestamp.IsZero()) - - t.Log("Retire only the failed request after Grove has accepted the lower scale") - _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - deleting := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, request.Name) - require.False(t, deleting.DeletionTimestamp.IsZero()) - require.Equal(t, running, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name)) - _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) +func TestLPXCompletedSchedulingDoesNotRestartAfterDegradation(t *testing.T) { + t.Log("Observe a request whose initial scheduling completed before runtime degradation") + dgd, source, _ := newLPXTestDGD(t, "single") + source.Spec.Scheduling = deadlineTestScheduling() + pcs := deadlineTestPCS(dgd, "pcs-uid") + request := deadlineTestRequest(dgd, pcs, "degraded", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) + request.Status.Phase = lpxv1alpha1.RequestPhaseDegraded + r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - t.Log("The request disappears when scheduler cleanup releases its own finalizer") - deleting.Finalizers = nil - require.NoError(t, reconciler.Update(t.Context(), deleting)) - _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(reconciler.Get(t.Context(), client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) - require.Zero(t, classification.(*lpxDeadlineTransition).requeueAfter) - - t.Log("A completed old attempt cannot touch same-name replacement identities") - replacement := request.DeepCopy() - replacement.UID = "replacement-uid" - replacement.ResourceVersion = "2" - replacement.Annotations[lpxPCSUIDAnnotation] = "replacement-pcs-uid" - pcs = deadlineTestPCS(dgd, "replacement-pcs-uid") - reconciler = newLPXTestReconciler(t, nil, dgd, source, replacement, pcs) - - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) require.NoError(t, err) - require.Equal(t, lpxSchedulingDeadlineExceededReason, lpxResult(classification).Reason) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, replacement.Name).DeletionTimestamp.IsZero()) - require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(pcs), storedPCS)) - require.Equal(t, int32(1), storedPCS.Spec.Replicas) - - t.Log("A stale cached attempt has no exact objects to clean up after replacement") - key := client.ObjectKeyFromObject(dgd) - require.NoError(t, reconciler.Get(t.Context(), key, dgd)) - cached := dgd.DeepCopy() - current := dgd.DeepCopy() - current.Status.Placement.LPXAttempt = deadlineTestAttempt(current, replacement, time.Now().Add(time.Minute)) - require.NoError(t, reconciler.Status().Update(t.Context(), current)) - require.NotEqual(t, cached.ResourceVersion, current.ResourceVersion) - running.ResourceVersion = "" - require.NoError(t, reconciler.Create(t.Context(), running)) - cached.Status.Placement.LPXAttempt.Requests = append(cached.Status.Placement.LPXAttempt.Requests, - deadlineTestAttempt(cached, running, time.Now()).Requests[0]) - _, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), cached, source) - require.NoError(t, err) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, running.Name).DeletionTimestamp.IsZero()) - require.True(t, getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, replacement.Name).DeletionTimestamp.IsZero()) - - t.Log("An exact Bound observation is authoritative even before its disposition is persisted") - late := deadlineTestAttempt(dgd, request, time.Now().Add(-time.Minute)) - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: late} - request.Status.Phase = lpxv1alpha1.RequestPhaseBound - reconciler = newLPXTestReconciler(t, nil, dgd, source, request) - classification, _, _, err = reconciler.reconcileLPXAttemptDeadline(t.Context(), dgd, source) - require.NoError(t, err) - require.Equal(t, lpxSchedulingDispositionObservedReason, lpxResult(classification).Reason) - require.NotNil(t, classification.(*lpxDeadlineTransition).attempt.DisarmedAt) + require.Nil(t, classification) + require.True(t, wake.IsZero()) } -func TestLPXAttemptBatchEditsPreserveUnchangedEngines(t *testing.T) { - for _, tc := range []struct { - name string - phase lpxv1alpha1.RequestPhase - add, remove, untimed bool - unrecorded, deleting bool +func TestLPXDeadlineIgnoresRetiringRequests(t *testing.T) { + for _, test := range []struct { + name string + deleteRequest bool + deletePodClique bool }{ - {name: "bound-before-disarm", phase: lpxv1alpha1.RequestPhaseBound, add: true}, - {name: "no-fit", phase: lpxv1alpha1.RequestPhaseNoFit, add: true}, - {name: "unsupported", phase: lpxv1alpha1.RequestPhaseUnsupported, add: true}, - {name: "active-addition", phase: lpxv1alpha1.RequestPhasePending, add: true}, - {name: "timeout-option-removed", phase: lpxv1alpha1.RequestPhasePending, add: true, untimed: true}, - {name: "metadata", phase: lpxv1alpha1.RequestPhasePending}, - {name: "intentional-removal", phase: lpxv1alpha1.RequestPhasePending, remove: true}, - {name: "remove-and-add", phase: lpxv1alpha1.RequestPhasePending, remove: true, add: true}, - {name: "remove-and-replace", phase: lpxv1alpha1.RequestPhasePending, remove: true, add: true, deleting: true}, - {name: "unrecorded-pending", phase: lpxv1alpha1.RequestPhasePending, unrecorded: true}, + {name: "request is terminating", deleteRequest: true}, + {name: "owner PCS is terminating", deletePodClique: true}, } { - t.Run(tc.name, func(t *testing.T) { - t.Log("Keep the existing engine and its deadline across a new source generation") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - deployment := newLPXTestDeployment(t, source) - pcs := deadlineTestPCS(deployment, "pcs-uid") - now := time.Now().UTC().Truncate(time.Second) - old := deadlineTestRequest(deployment, pcs, "old", now.Add(-time.Hour), tc.phase) - deadline := now.Add(time.Minute) - if tc.phase != lpxv1alpha1.RequestPhasePending { - deadline = now.Add(-time.Minute) - } - current := deadlineTestAttempt(deployment, old, deadline) - if tc.unrecorded { - current.Requests[0].UID = "" - } - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: current} - deployment.Generation++ - want := &nvidiacomv1beta1.LPXAttemptStatus{ObservedGeneration: deployment.Generation, PodCliqueSetUID: current.PodCliqueSetUID} - currents := map[string]*lpxv1alpha1.LPUPipelineRequest{old.Name: old} - if !tc.remove { - want.Requests = append(want.Requests, current.Requests[0]) - want.Requests[0].AttemptDigest = "" + t.Run(test.name, func(t *testing.T) { + t.Log("Observe an expired request that intentional cleanup already owns") + dgd, source, _ := newLPXTestDGD(t, "single") + source.Spec.Scheduling = deadlineTestScheduling() + pcs := deadlineTestPCS(dgd, "pcs-uid") + request := deadlineTestRequest(dgd, pcs, "expired", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending) + deletingAt := metav1.Now() + if test.deleteRequest { + request.Finalizers = []string{"scheduler.example/cleanup"} + request.DeletionTimestamp = &deletingAt } - if tc.add { - added := deadlineTestRequest(deployment, pcs, "new", now, lpxv1alpha1.RequestPhasePending) - if tc.deleting { - added.DeletionTimestamp = ptr.To(metav1.Now()) - } - currents[added.Name] = added - want.Requests = append(want.Requests, nvidiacomv1beta1.LPXAttemptRequestStatus{Name: added.Name}) + if test.deletePodClique { + pcs.Finalizers = []string{"grove.example/cleanup"} + pcs.DeletionTimestamp = &deletingAt } + r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - t.Log("New engines get a fresh batch only after the old scheduling batch completes") - r := newLPXTestReconciler(t, nil, deployment, source) - timeout := ptr.To[int64](30) - if tc.untimed { - timeout = nil - } - if transition := r.reconcileLPXAttemptPreparation(deployment, timeout, want, currents); transition != nil { - current = applyLPXAttemptTransition(t, deployment, transition) - } - if tc.phase != lpxv1alpha1.RequestPhasePending { - require.Len(t, current.Requests, 1) - require.Equal(t, "new", current.Requests[0].Name) - require.Equal(t, now.Add(30*time.Second), current.DeadlineAt.Time) - } else { - require.Equal(t, deadline, current.DeadlineAt.Time) - require.Equal(t, len(want.Requests), len(current.Requests)) - } - if tc.add || tc.unrecorded { - require.Nil(t, current.DisarmedAt) - if !tc.deleting { - require.NotEmpty(t, current.Requests[0].UID) - } - } - require.True(t, old.DeletionTimestamp.IsZero()) - require.Equal(t, types.UID("old-uid"), old.UID) + t.Log("Do not turn an intentional removal into a scheduling failure") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) + require.NoError(t, err) + require.Nil(t, classification) + require.True(t, wake.IsZero()) }) } } -func TestLPXAttemptReplacementAdoptsNewPodCliqueSetIdentity(t *testing.T) { - for _, timed := range []bool{false, true} { - t.Run(fmt.Sprintf("timed=%t", timed), func(t *testing.T) { - t.Log("Record an active attempt after every old request has finished garbage collection") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - deployment := newLPXTestDeployment(t, source) - deployment.Generation = 2 - current := &nvidiacomv1beta1.LPXAttemptStatus{ - ObservedGeneration: 1, - PodCliqueSetUID: "previous-pcs-uid", - Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ - Name: "engine", - }}, - } - var deadlineSeconds *int64 - if timed { - current.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(time.Minute))) - deadlineSeconds = ptr.To[int64](30) - } - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: current} - replacement := deadlineTestPCS(deployment, "replacement-pcs-uid") - reconciler := newLPXTestReconciler(t, nil, deployment, source, replacement) - want := &nvidiacomv1beta1.LPXAttemptStatus{ - ObservedGeneration: deployment.Generation, - PodCliqueSetUID: replacement.UID, - Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ - Name: "engine", AttemptDigest: "sha256:replacement", - }}, - } +func TestLPXDeadlineListFailurePropagatesWithoutSyntheticWake(t *testing.T) { + t.Log("Fail the authoritative LPR list before any deadline can be derived") + dgd, source, _ := newLPXTestDGD(t, "single") + source.Spec.Scheduling = deadlineTestScheduling() + pcs := deadlineTestPCS(dgd, "pcs-uid") + r := newLPXTestReconciler(t, nil, dgd, source, pcs) + listErr := errors.New("request list unavailable") + r.apiReader = interceptor.NewClient(r.apiReader.(client.WithWatch), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return listErr + }, + }) - t.Log("Move the preserved attempt to the replacement PCS before publishing its request") - transition := reconciler.reconcileLPXAttemptPreparation( - deployment, - deadlineSeconds, - want, - map[string]*lpxv1alpha1.LPUPipelineRequest{}, - ) - require.NotNil(t, transition) - next := applyLPXAttemptTransition(t, deployment, transition) - require.Equal(t, deployment.Generation, next.ObservedGeneration) - require.Equal(t, replacement.UID, next.PodCliqueSetUID) - require.Equal(t, current.DeadlineAt, next.DeadlineAt) - require.Equal(t, want.Requests, next.Requests) - require.NoError(t, reconciler.revalidateLPXAttemptPublication(t.Context(), deployment, replacement.Name)) - }) - } + t.Log("Return the real error so controller-runtime applies error backoff") + classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, nil) + require.ErrorIs(t, err, listErr) + require.Nil(t, classification) + require.True(t, wake.IsZero()) + result, err := completeLPXDeadline(t.Context(), wake, ctrl.Result{}, err) + require.ErrorIs(t, err, listErr) + require.Zero(t, result) } -func TestLPXAttemptScaleInCancellationPrecedesDeadline(t *testing.T) { - for _, tc := range []struct { - name string - phase lpxv1alpha1.RequestPhase - }{ - {name: "pending", phase: lpxv1alpha1.RequestPhasePending}, - {name: "completed", phase: lpxv1alpha1.RequestPhaseBound}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Log("Publish two engines, with only the second engine recorded in the scheduling batch") - ctx := t.Context() - deployment, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Components[0].Replicas = ptr.To[int32](2) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) - objects := lpxMaterializedObjects(t, r, deployment, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, deployment, desired) - running := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[0].requestName) - running.Status = newLPXNodeLocalBoundStatus(running.Generation, 1, "sha256:bound") - pending := getLPXRequest(t, ctx, r.Client, deployment.Namespace, desired.requests[1].requestName) - pending.Finalizers = []string{"scheduler.example/cleanup"} - pending.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(pending.Generation), - } - if tc.phase == lpxv1alpha1.RequestPhaseBound { - pending.Status = newLPXNodeLocalBoundStatus(pending.Generation, 1, "sha256:bound") - } - deadline := time.Now().UTC().Truncate(time.Second).Add(-time.Minute) - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: deadlineTestAttempt(deployment, pending, deadline)} - source.Spec.Scheduling = deadlineTestScheduling() - source.Spec.Components[0].Replicas = ptr.To[int32](1) - objects = append(objects, running, pending) - r = newLPXTestReconciler(t, registry, deployment, source, objects...) - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - return delegated.Delete(ctx, object.DeepCopyObject().(client.Object), opts...) - }, - }) - key := client.ObjectKeyFromObject(deployment) - - t.Log("Persist cancellation before evaluating the expired deadline") - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, key, deployment)) - attempt := currentLPXAttemptStatus(deployment) - require.Equal(t, []nvidiacomv1beta1.LPXAttemptRequestStatus{}, attempt.Requests) - require.Nil(t, attempt.ExceededAt) - require.NotNil(t, attempt.DisarmedAt) - require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) - - t.Log("A source read failure on the next reconcile cannot expire the canceled batch") - readError := errors.New("temporary source observation failure") - original := r.Client - r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, source := object.(*nvidiacomv1beta1.DynamoGraphDeployment); source { - return readError - } - return delegated.Get(ctx, key, object, opts...) - }, - }) - _, readErr := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - r.Client = original - require.NoError(t, r.Get(ctx, key, deployment)) - require.Nil(t, currentLPXAttemptStatus(deployment).ExceededAt) - require.ErrorIs(t, readErr, readError) - - t.Log("Cancellation remains disarmed while only the removed request retires") - for range 4 { - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, key, deployment)) - attempt = currentLPXAttemptStatus(deployment) - require.True(t, deadline.Equal(attempt.DeadlineAt.Time)) - } - require.Nil(t, attempt.ExceededAt) - require.NotNil(t, attempt.DisarmedAt) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) - require.Equal(t, types.UID("pcs-uid"), pcs.UID) - require.Equal(t, running, getLPXRequest(t, ctx, r.Client, deployment.Namespace, running.Name)) - stored := getLPXRequest(t, ctx, r.Client, deployment.Namespace, pending.Name) - require.False(t, stored.DeletionTimestamp.IsZero()) - }) - } +func TestLPXDeadlineFailureRequiresPostFailureGeneration(t *testing.T) { + t.Log("Record a deadline failure at the generation current when the failure occurred") + dgd, _, _ := newLPXTestDGD(t, "single") + dgd.Generation++ + dgd.Status.Conditions = []metav1.Condition{{ + Type: lpxSchedulingFailedCondition, + Status: metav1.ConditionTrue, + Reason: lpxSchedulingDeadlineExceededReason, + ObservedGeneration: dgd.Generation, + }} + require.True(t, lpxDeadlineFailureCurrent(dgd)) + + t.Log("Only a generation after the failure authorizes another publication") + dgd.Generation++ + require.False(t, lpxDeadlineFailureCurrent(dgd)) } -func TestLPXAttemptRetryRequiresLaterEditAndCompletedCleanup(t *testing.T) { - for _, tc := range []struct { - name string - laterEdit, oldGone bool - }{{name: "same-generation", oldGone: true}, {name: "cleanup-pending", laterEdit: true}, {name: "later-edit", laterEdit: true, oldGone: true}} { - t.Run(tc.name, func(t *testing.T) { - t.Log("Keep the failed batch fenced until both a later edit and exact-request cleanup") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - deployment := newLPXTestDeployment(t, source) - pcs := deadlineTestPCS(deployment, "pcs-uid") - old := deadlineTestRequest(deployment, pcs, "old", time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending) - old.Finalizers = []string{"scheduler.example/cleanup"} - attempt := deadlineTestAttempt(deployment, old, time.Now().Add(-time.Second)) - attempt.ExceededAt = ptr.To(metav1.Now()) - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - if tc.laterEdit { - deployment.Generation++ - } - var objects []client.Object - if !tc.oldGone { - objects = append(objects, old) - } - r := newLPXTestReconciler(t, nil, deployment, source, objects...) - var requests []lpxv1alpha1.LPUPipelineRequest - if !tc.oldGone { - requests = []lpxv1alpha1.LPUPipelineRequest{*old} - } - classification, err := r.continueTerminalLPXAttempt(t.Context(), deployment, attempt, requests, lpxSchedulingDeadlineExceededReason, "expired") - require.NoError(t, err) - transition := classification.(*lpxDeadlineTransition) - if tc.laterEdit && tc.oldGone { - require.Nil(t, transition.attempt) - } else { - require.Equal(t, lpxSchedulingDeadlineExceededReason, transition.result.Reason) - require.Equal(t, attempt, transition.attempt) - } - }) - } + +func TestLPXDeadlineFailureOnlyCoversPreviouslyPublishedRequests(t *testing.T) { + t.Log("Record a deadline failure for the current deployment generation") + dgd, _, _ := newLPXTestDGD(t, "single") + failureTime := time.Now().Add(-time.Minute) + dgd.Status.Conditions = []metav1.Condition{{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, + Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, + LastTransitionTime: metav1.NewTime(failureTime), + }} + + t.Log("The failure covers cleanup of a scheduling cycle that started before it") + pcs := deadlineTestPCS(dgd, "pcs-uid") + oldRequest := deadlineTestRequest(dgd, pcs, "old", failureTime.Add(-time.Minute), lpxv1alpha1.RequestPhasePending) + require.True(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{oldRequest})) + + t.Log("A failure from an earlier deployment generation cannot cover current cleanup") + dgd.Generation++ + require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{oldRequest})) + dgd.Generation-- + + t.Log("A later scheduling cycle on the same request requires a new durable failure") + restartedRequest := deadlineTestRequest(dgd, pcs, "restarted", failureTime.Add(-time.Hour), lpxv1alpha1.RequestPhasePending) + restartedRequest.Status.LastPlanRevision = 1 + restartedRequest.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(failureTime.Add(time.Second))) + require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{restartedRequest})) + + t.Log("An unknown cycle start is never covered by an older failure") + restartedRequest.Status.SchedulingStartedAt = nil + require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{restartedRequest})) } -func TestLPXAttemptRecoversUIDWithoutTimeoutOrSource(t *testing.T) { - for _, sourceAvailable := range []bool{false, true} { - t.Log("Recover an already-published UID even when its timeout option or source is unavailable", sourceAvailable) - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - deployment := newLPXTestDeployment(t, source) - pcs := deadlineTestPCS(deployment, "pcs-uid") - request := deadlineTestRequest(deployment, pcs, "unrecorded", time.Now(), lpxv1alpha1.RequestPhasePending) - attempt := deadlineTestAttempt(deployment, request, time.Now()) - attempt.DeadlineAt, attempt.Requests[0].UID = nil, "" - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: attempt} - r := newLPXTestReconciler(t, nil, deployment, source, request, pcs) - if !sourceAvailable { - source = nil - } - classification, wake, _, err := r.reconcileLPXAttemptDeadline(t.Context(), deployment, source) - require.NoError(t, err) - recovered := applyLPXAttemptTransition(t, deployment, classification) - require.Equal(t, request.UID, recovered.Requests[0].UID) - require.Nil(t, recovered.DeadlineAt) - require.True(t, wake.IsZero()) - } +func TestProjectLPXDeadlineFailureAdvancesAStaleFailureFence(t *testing.T) { + dgd, _, _ := newLPXTestDGD(t, "single") + previous := dgd.Status.DeepCopy() + previous.Conditions = []metav1.Condition{{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, + Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, + LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)), + }} + dgd.Status = *previous.DeepCopy() + dgd.Generation++ + + t.Log("A newly expired request replaces the stale failure record") + projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{ + State: nvidiacomv1beta1.DGDStateFailed, Reason: lpxSchedulingDeadlineExceededReason, + Message: "expired", + }, true) + failed := meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition) + require.NotNil(t, failed) + require.Equal(t, dgd.Generation, failed.ObservedGeneration) + require.True(t, failed.LastTransitionTime.After(previous.Conditions[0].LastTransitionTime.Time)) } -func applyLPXAttemptTransition(t *testing.T, dgd *nvidiacomv1alpha1.LPXGraphDeployment, classification lpxClassification) *nvidiacomv1beta1.LPXAttemptStatus { - t.Helper() - transition := classification.(*lpxDeadlineTransition) - require.NotNil(t, transition.attempt) - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{LPXAttempt: transition.attempt.DeepCopy()} - return dgd.Status.Placement.LPXAttempt +func TestProjectLPXDeadlineFailureRemainsStickyAtRecordedGeneration(t *testing.T) { + t.Log("Record a deadline failure for the current generation") + dgd, _, _ := newLPXTestDGD(t, "single") + previous := dgd.Status.DeepCopy() + previous.Conditions = []metav1.Condition{{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, + Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, + }} + dgd.Status = *previous.DeepCopy() + + t.Log("Do not let a late successful observation authorize the same generation") + projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{ + State: nvidiacomv1beta1.DGDStateSuccessful, Reason: "Ready", Message: "ready", + }, false) + failed := meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition) + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionTrue, failed.Status) + require.Equal(t, lpxSchedulingDeadlineExceededReason, failed.Reason) } -func deadlineTestAttempt(dgd *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest, deadline time.Time) *nvidiacomv1beta1.LPXAttemptStatus { - return &nvidiacomv1beta1.LPXAttemptStatus{ - ObservedGeneration: dgd.Generation, - PodCliqueSetUID: types.UID(request.Annotations[lpxPCSUIDAnnotation]), - DeadlineAt: ptr.To(metav1.NewTime(deadline)), - Requests: []nvidiacomv1beta1.LPXAttemptRequestStatus{{ - Name: request.Name, AttemptDigest: request.Annotations[lpxAttemptDigestAnnotation], UID: request.UID, - }}, - } +func TestCompleteLPXDeadlineBoundsTransientErrors(t *testing.T) { + reconcileErr := errors.New("temporary failure") + result, err := completeLPXDeadline(context.Background(), time.Now().Add(time.Minute), ctrl.Result{}, reconcileErr) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.LessOrEqual(t, result.RequeueAfter, lpxRetirementRequeueAfter) + + result, err = completeLPXDeadline(context.Background(), time.Time{}, ctrl.Result{}, reconcileErr) + require.ErrorIs(t, err, reconcileErr) + require.Zero(t, result) } func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { generation := int64(1) + schedulingStartedAt := metav1.NewTime(created.UTC().Truncate(time.Second)) request := &lpxv1alpha1.LPUPipelineRequest{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: dgd.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", Generation: generation, CreationTimestamp: metav1.NewTime(created), - Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, - Annotations: map[string]string{ - lpxAttemptDigestAnnotation: "sha256:attempt", lpxDeploymentUIDAnnotation: string(dgd.UID), - lpxPCSUIDAnnotation: string(pcs.UID), - }, + Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, + Annotations: map[string]string{lpxDeploymentUIDAnnotation: string(dgd.UID)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, - Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, - Status: &lpxv1alpha1.LPUPipelineRequestStatus{Phase: phase, ObservedGeneration: &generation}, + Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: phase, ObservedGeneration: &generation, + SchedulingStartedAt: &schedulingStartedAt, + }, } if phase == lpxv1alpha1.RequestPhaseBound { request.Status.LastPlanRevision = 1 @@ -652,6 +318,14 @@ func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, pcs *grovev1 return request } +func deadlineTestDesired(requests ...*lpxv1alpha1.LPUPipelineRequest) []lpxModelMaterializing { + desired := make([]lpxModelMaterializing, 0, len(requests)) + for _, request := range requests { + desired = append(desired, lpxModelMaterializing{requestName: request.Name}) + } + return desired +} + func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { return &grovev1alpha1.PodCliqueSet{ ObjectMeta: metav1.ObjectMeta{ @@ -665,10 +339,3 @@ func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) * func deadlineTestScheduling() *nvidiacomv1beta1.SchedulingSpec { return &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} } - -func ownedLPXRequests(t *testing.T, ctx context.Context, reconciler *graphReconciler, dgd *nvidiacomv1alpha1.LPXGraphDeployment) []lpxv1alpha1.LPUPipelineRequest { - t.Helper() - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) - require.NoError(t, err) - return requests -} diff --git a/deploy/operator/internal/controller/lpx/attempt_retirement.go b/deploy/operator/internal/controller/lpx/attempt_retirement.go deleted file mode 100644 index 6c2e4e21ccaf..000000000000 --- a/deploy/operator/internal/controller/lpx/attempt_retirement.go +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - "fmt" - "math" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - autoscalingv1 "k8s.io/api/autoscaling/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// retireLPXAttempt retires only the recorded batch, preserving other engines in -// its shared PCS. deployment and attempt must be nonnil; requests are owned by deployment. -func (r *graphReconciler) retireLPXAttempt( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - attempt *nvidiacomv1beta1.LPXAttemptStatus, - requests []lpxv1alpha1.LPUPipelineRequest, -) (bool, error) { - // Partition the shared workload using exact request identities, including replacements. - identities := make(map[string]types.UID, len(attempt.Requests)) - for _, row := range attempt.Requests { - identities[row.Name] = row.UID - } - pcsName := dynamo.PCSNameForLPX(deployment) - var batch, retained []*lpxv1alpha1.LPUPipelineRequest - for index := range requests { - request := &requests[index] - owner := metav1.GetControllerOf(request) - if owner == nil || owner.UID != attempt.PodCliqueSetUID { - continue - } - pcsName = owner.Name - if uid := identities[request.Name]; uid != "" && uid == request.UID { - batch = append(batch, request) - } else { - retained = append(retained, request) - } - } - - // Preserve the shared PCS when a later scheduling batch fails during scale-out. - if len(retained) > 0 { - return r.retireLPXScaleOut(ctx, deployment, pcsName, attempt.PodCliqueSetUID, retained, batch) - } - pending, err := r.deleteLPXPodCliqueSet(ctx, deployment, pcsName, attempt.PodCliqueSetUID) - return pending || len(batch) > 0, err -} - -// retireLPXScaleOut removes trailing engines through Grove before deleting their -// requests, whose scheduler finalizers may wait for pods to disappear. deployment -// must be nonnil and retained must contain at least one request outside the batch. -func (r *graphReconciler) retireLPXScaleOut( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, - pcsUID types.UID, - retained, batch []*lpxv1alpha1.LPUPipelineRequest, -) (bool, error) { - // Derive the surviving prefix from immutable request targets, independent of source edits. - groupName, replicas, err := lpxRetainedEngineScale(retained, batch) - if err != nil { - return false, err - } - - // An authoritative owner read fences a replaced PCS before changing its group's scale. - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { - if apierrors.IsNotFound(err) { - return true, nil - } - return false, err - } - if pcs.UID != pcsUID { - return true, nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return false, fmt.Errorf("refusing to scale PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) - } - if !pcs.DeletionTimestamp.IsZero() { - return true, nil - } - - // Wait for Grove to create the group, then lower its scale without ever growing it. - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: groupName}, group); err != nil { - return true, client.IgnoreNotFound(err) - } - if !metav1.IsControlledBy(group, pcs) { - return false, fmt.Errorf("LPX scaling group lacks the current PCS owner") - } - if !group.DeletionTimestamp.IsZero() { - return true, nil - } - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return false, err - } - if group.Spec.Replicas > replicas { - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, - } - if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { - return false, err - } - return true, nil - } - - // Keep request cleanup fenced by exact UIDs until the scheduler releases each finalizer. - if err := r.deleteStaleLPXRequests(ctx, deployment, batch); err != nil { - return false, err - } - return len(batch) > 0, nil -} - -// lpxRetainedEngineScale requires nonnil requests and a nonempty retained slice. -// Grove removes a suffix when scaling down; reject targets that would retire any -// engine outside the failed batch, including a model sharing the same replica. -func lpxRetainedEngineScale(retained, batch []*lpxv1alpha1.LPUPipelineRequest) (string, int32, error) { - // Retain every replica with scheduler intent outside the failed batch. - groupName := "" - var replicas int64 - for _, request := range retained { - scope := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef - if scope == nil || scope.Name == "" || scope.ReplicaIndex < 0 || scope.ReplicaIndex >= math.MaxInt32 { - return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) - } - if groupName != "" && scope.Name != groupName { - return "", 0, fmt.Errorf("LPX requests target different scaling groups") - } - groupName = scope.Name - replicas = max(replicas, scope.ReplicaIndex+1) - } - - // A failed scale-out must lie entirely beyond the surviving engines. - for _, request := range batch { - scope := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef - if scope == nil || scope.Name != groupName || scope.ReplicaIndex < replicas { - return "", 0, fmt.Errorf("cannot retire LPX request %q without disturbing an engine outside its scheduling batch", request.Name) - } - } - return groupName, int32(replicas), nil -} diff --git a/deploy/operator/internal/controller/lpx/attempt_retirement_test.go b/deploy/operator/internal/controller/lpx/attempt_retirement_test.go deleted file mode 100644 index 64fb1d9ff25b..000000000000 --- a/deploy/operator/internal/controller/lpx/attempt_retirement_test.go +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/require" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" -) - -func TestLPXScaleOutRetirementProtectsUnrelatedAuthority(t *testing.T) { - const missingGroup = "missing group" - for _, scenario := range []string{"foreign group", "updated child", "scale conflict", missingGroup, "replacement request", "shared replica"} { - t.Run(scenario, func(t *testing.T) { - t.Log("Publish two engines and record only the second engine in the failed batch") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - failed := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) - attempt := deadlineTestAttempt(child, failed, time.Now().Add(-time.Second)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - - t.Log("Change the observed authority before terminal cleanup attempts a destructive write") - wantError := true - switch scenario { - case "foreign group": - group.OwnerReferences[0].UID = "foreign-pcs" - require.NoError(t, r.Update(ctx, group)) - case "updated child": - newer := child.DeepCopy() - newer.Generation++ - require.NoError(t, r.Update(ctx, newer)) - case "scale conflict": - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { - return apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), group.Name, errors.New("stale scale")) - }, - }) - case missingGroup: - require.NoError(t, r.Delete(ctx, group)) - wantError = false - case "replacement request": - require.NoError(t, r.Delete(ctx, failed)) - failed.UID, failed.ResourceVersion = "replacement-uid", "" - require.NoError(t, r.Create(ctx, failed)) - wantError = false - case "shared replica": - failed.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex = 0 - require.NoError(t, r.Update(ctx, failed)) - } - - t.Log("Preserve the PCS and request identities when scale-down is unsafe or not yet possible") - requests := ownedLPXRequests(t, ctx, r, child) - pending, err := r.retireLPXAttempt(ctx, child, attempt, requests) - if wantError { - require.Error(t, err) - } else { - require.NoError(t, err) - require.Equal(t, scenario == missingGroup, pending) - } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) - for _, before := range requests { - require.Equal(t, &before, getLPXRequest(t, ctx, r.Client, before.Namespace, before.Name)) - } - if scenario != missingGroup { - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - } - }) - } -} - -func TestLPXFailedSpecDecodeScaleOutRetiresAllModels(t *testing.T) { - t.Log("Publish one target and two draft requests for each of two engines") - ctx := t.Context() - child, source, registry := newLPXSpecDecodeTestDGD(t) - lpx.ServingComponent(source).Replicas = nil - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - group.Spec.Replicas = 2 - createLPXTestObjects(t, ctx, r.Client, objects...) - desired, rejected := requirePreparedLPX(t, r, ctx, child, source) - require.Nil(t, rejected) - publishSelectedLPXForTest(t, ctx, r, child, desired) - requests := ownedLPXRequests(t, ctx, r, child) - require.Len(t, requests, 6) - attempt := currentLPXAttemptStatus(child).DeepCopy() - attempt.Requests = nil - for _, request := range requests { - if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 1 { - attempt.Requests = append(attempt.Requests, deadlineTestAttempt(child, &request, time.Now()).Requests...) - } - } - attempt.ExceededAt = ptr.To(metav1.Now()) - require.Len(t, attempt.Requests, 3) - - t.Log("Retire all requests in the failed engine while preserving the serving engine's models") - for range 2 { - _, err := r.retireLPXAttempt(ctx, child, attempt, requests) - require.NoError(t, err) - } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - for _, request := range requests { - if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 0 { - require.Equal(t, &request, getLPXRequest(t, ctx, r.Client, request.Namespace, request.Name)) - } else { - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(&request), &lpxv1alpha1.LPUPipelineRequest{}))) - } - } -} - -func TestLPXScaleOutRetirementWaitsForOwnerGarbageCollection(t *testing.T) { - t.Log("Keep old requests observable after their shared PCS is already absent") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - failed := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) - attempt := deadlineTestAttempt(child, failed, time.Now().Add(-time.Second)) - pcs := findLPXTestPodCliqueSet(t, objects) - requests := ownedLPXRequests(t, ctx, r, child) - require.Len(t, requests, 2) - require.NoError(t, r.Delete(ctx, pcs)) - - t.Log("Keep terminal cleanup pending until owner GC removes every old request") - pending, err := r.retireLPXAttempt(ctx, child, attempt, requests) - require.NoError(t, err) - require.True(t, pending) - for index := range requests { - require.NoError(t, r.Delete(ctx, &requests[index])) - } - pending, err = r.retireLPXAttempt(ctx, child, attempt, nil) - require.NoError(t, err) - require.False(t, pending) -} diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 5456ab76b631..8f98036f27b9 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -146,46 +145,19 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu deployment.Status.ModelDownload = nil } state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} - var deadlineAt time.Time var source *v1beta1.DynamoGraphDeployment + var pcs *grovev1alpha1.PodCliqueSet + var selected *lpxMaterializing defer func() { - // Reconcile intended removals before deadline cleanup, including on failed renders. - if unavailableReason == "" { - classification, wake, _, deadlineErr := r.reconcileLPXAttemptDeadline(ctx, deployment, source) - deadlineAt, err = wake, errors.Join(err, deadlineErr) - if classification != nil { - state, result = lpxResult(classification), projectLPXLifecycleStatus(deployment, classification) - } - } - if err != nil { - state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXReconciliationFailed", truncateLPXMessage(err.Error()) - } else if unavailableReason == "" { - deployment.Status.ObservedGeneration = deployment.Generation - } - ready, failed := metav1.ConditionFalse, metav1.ConditionFalse - if state.State == v1beta1.DGDStateSuccessful { - ready = metav1.ConditionTrue - } - if state.State == v1beta1.DGDStateFailed { - failed = metav1.ConditionTrue - } - // Every authored component observes the complete shared runtime gate. - for name, component := range deployment.Status.Components { - component.Ready = ready == metav1.ConditionTrue - deployment.Status.Components[name] = component - } - for condition, value := range map[string]metav1.ConditionStatus{"Ready": ready, "Failed": failed} { - meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{Type: condition, Status: value, ObservedGeneration: deployment.Generation, Reason: state.Reason, Message: state.Message}) - } - if !apiequality.Semantic.DeepEqual(previous, &deployment.Status) { - if statusErr := r.Status().Update(ctx, deployment); statusErr != nil { - err = errors.Join(err, fmt.Errorf("persist LPX lifecycle status: %w", statusErr)) - } - } - if ready == metav1.ConditionTrue && deployment.Status.ModelDownload != nil && deployment.Status.ModelDownload.LastCheckedAt != nil && result.RequeueAfter == 0 { - result.RequeueAfter = max(modelDownloadRequeueAfter, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) - } - result, err = completeLPXDeadline(ctx, deadlineAt, result, err) + result, err = r.completeReconcile(ctx, deployment, previous, lpxReconcileCompletion{ + unavailableReason: unavailableReason, + state: state, + source: source, + pcs: pcs, + selected: selected, + result: result, + reconcileErr: err, + }) }() // Like Grove, disabling integration reports unavailability without touching workloads. @@ -200,17 +172,31 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu if sourceErr != nil { return ctrl.Result{}, sourceErr } + pcs, err = r.observeCurrentLPXPodCliqueSet(ctx, deployment) + if err != nil { + return ctrl.Result{}, err + } + // Invalid source authority retires existing LPX state before reporting rejection. if rejected != nil { + retiring, retireErr := r.retireInvalidLPXWorkload(ctx, deployment, pcs, rejected.reason) + if retireErr != nil { + return ctrl.Result{}, retireErr + } + if retiring != nil { + state = lpxResult(retiring) + return projectLPXLifecycleStatus(retiring), nil + } state = lpxResult(rejected) - return r.retireInvalidWorkload(ctx, deployment, rejected.reason) + return ctrl.Result{}, nil } - selected, classification, err := r.reconcileLPXSafetyPreflight(ctx, deployment, source, nil) + var classification lpxClassification + selected, classification, err = r.reconcileLPXSafetyPreflight(ctx, deployment, source, pcs, nil) if err != nil { return ctrl.Result{}, err } if classification != nil { state = lpxResult(classification) - return projectLPXLifecycleStatus(deployment, classification), nil + return projectLPXLifecycleStatus(classification), nil } ready, err := r.reconcileModelDownloads(ctx, deployment, source, selected == nil) if err != nil { @@ -221,69 +207,136 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{RequeueAfter: modelDownloadRequeueAfter}, nil } if selected == nil { - selected, classification, err = r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source) + selected, classification, err = r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source, pcs) if err != nil { return ctrl.Result{}, err } if classification != nil { state = lpxResult(classification) - return projectLPXLifecycleStatus(deployment, classification), nil + return projectLPXLifecycleStatus(classification), nil } } - state, result, err = r.reconcileWorkload(ctx, deployment, source, selected) + state, result, err = r.reconcileWorkload(ctx, deployment, source, selected, pcs) return result, err } -// reconcileWorkload publishes and observes the selected engine after source, -// deadline, placement, and model-download gates have passed. deployment, source -// and selected are non-nil and supplied by that preflight. -func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment, selected *lpxMaterializing) (reconcileOutcome, ctrl.Result, error) { - state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} +type lpxReconcileCompletion struct { + unavailableReason string + state reconcileOutcome + source *v1beta1.DynamoGraphDeployment + pcs *grovev1alpha1.PodCliqueSet + selected *lpxMaterializing + result ctrl.Result + reconcileErr error +} - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: selected.plan.PodCliqueSetName}, pcs); err != nil { - if !apierrors.IsNotFound(err) { - return state, ctrl.Result{}, err +// completeReconcile projects deadlines and the final lifecycle outcome after +// every reconcile path has finished mutating its in-memory status. +func (r *graphReconciler) completeReconcile( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + previous *v1alpha1.LPXGraphDeploymentStatus, + completion lpxReconcileCompletion, +) (ctrl.Result, error) { + state, result, err := completion.state, completion.result, completion.reconcileErr + deadlineAt := time.Time{} + + // Keep active-request deadlines running while unrelated request cleanup converges. + deadlineExceeded := false + recordDeadlineFailure := false + requestScopedRetirement := state.Reason == lpxRetiringReason && state.retirementScope == lpxRequestRetirement + if completion.unavailableReason == "" && completion.selected != nil && + (state.Reason != lpxRetiringReason || requestScopedRetirement) { + classification, wake, deadlineErr := r.reconcileLPXRequestDeadlines( + ctx, deployment, completion.source, completion.pcs, completion.selected.requests, + ) + deadlineAt, err = wake, errors.Join(err, deadlineErr) + if classification != nil { + state, result = lpxResult(classification), projectLPXLifecycleStatus(classification) + deadlineExceeded, recordDeadlineFailure = lpxDeadlineFlags(classification) + } + } + if err != nil { + if !deadlineExceeded { + state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXReconciliationFailed", truncateLPXMessage(err.Error()) + } + } else if completion.unavailableReason == "" { + deployment.Status.ObservedGeneration = deployment.Generation + } + ready, failed := metav1.ConditionFalse, metav1.ConditionFalse + if state.State == v1beta1.DGDStateSuccessful { + ready = metav1.ConditionTrue + } + if state.State == v1beta1.DGDStateFailed { + failed = metav1.ConditionTrue + } + // Every authored component observes the complete shared runtime gate. + for name, component := range deployment.Status.Components { + component.Ready = ready == metav1.ConditionTrue + deployment.Status.Components[name] = component + } + for condition, value := range map[string]metav1.ConditionStatus{"Ready": ready, "Failed": failed} { + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{Type: condition, Status: value, ObservedGeneration: deployment.Generation, Reason: state.Reason, Message: state.Message}) + } + projectLPXSchedulingFailureCondition(deployment, previous, state, recordDeadlineFailure) + if !apiequality.Semantic.DeepEqual(previous, &deployment.Status) { + if statusErr := r.Status().Update(ctx, deployment); statusErr != nil { + err = errors.Join(err, fmt.Errorf("persist LPX lifecycle status: %w", statusErr)) } - pcs = nil } - desired, resources, err := renderPodCliqueSet(ctx, source, r.Config, r.runtimeConfig, r.Client, r.DockerSecretRetriever, selected.workload, selected.plan, deployment) + if ready == metav1.ConditionTrue && deployment.Status.ModelDownload != nil && deployment.Status.ModelDownload.LastCheckedAt != nil && result.RequeueAfter == 0 { + result.RequeueAfter = max(modelDownloadRequeueAfter, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) + } + return completeLPXDeadline(ctx, deadlineAt, result, err) +} + +// reconcileWorkload publishes and observes the selected engine after source, +// deadline, placement, and model-download gates have passed. deployment, source +// and selected are non-nil and supplied by that preflight. pcs is nil when its +// stable name was not found before preflight. +func (r *graphReconciler) reconcileWorkload( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + source *v1beta1.DynamoGraphDeployment, + selected *lpxMaterializing, + pcs *grovev1alpha1.PodCliqueSet, +) (reconcileOutcome, ctrl.Result, error) { + state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} + desired, resources, err := renderPodCliqueSet(ctx, source, r.Config, r.runtimeConfig, r.DockerSecretRetriever, selected.workload, selected.plan, deployment) if err != nil { return state, ctrl.Result{}, err } if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { return state, ctrl.Result{}, err } - // Scale the live Grove group before stale request retirement. LPX scheduler - // finalizers may wait for Grove to remove the corresponding pods, so the - // publication fence must not block this write. - if err := r.reconcileLPXScalingGroupScale(ctx, source, pcs, selected.plan.LPXScalingGroup, true); err != nil { - return state, ctrl.Result{}, err + // Replica scale-down keeps the immutable PCS shape, so lower the live Grove + // group before deleting stale requests. A workload-shape change replaces the + // PCS instead and must not try to mutate its old group. + scaled := false + if pcs != nil && pcs.DeletionTimestamp.IsZero() && pcs.Annotations[lpx.WorkloadDigestAnnotation] == selected.workloadDigest { + scaled, err = r.reconcileLPXScalingGroupScale(ctx, source, pcs, selected.plan.LPXScalingGroup, true) + if err != nil { + return state, ctrl.Result{}, err + } } - currents, classification, err := r.reconcileLPXPublicationFence(ctx, deployment, selected, pcs == nil) + currents, classification, err := r.reconcileLPXPublicationFence(ctx, deployment, selected, pcs) if err != nil { return state, ctrl.Result{}, err } if classification != nil { state = lpxResult(classification) - return state, projectLPXLifecycleStatus(deployment, classification), nil + return state, projectLPXLifecycleStatus(classification), nil + } + // The existing PCS template only seeds its PCSG; preserve that seed while + // the live group scales independently, including to zero. + if err := preserveLPXScalingGroupReplicaSeed(desired, pcs); err != nil { + return state, ctrl.Result{}, err } for _, resource := range resources { if err := r.syncLPXResource(ctx, deployment, resource); err != nil { return state, ctrl.Result{}, err } } - // Grove seeds native scale only at creation; preserve that seed when replicas are omitted. - if pcs != nil && lpx.ServingComponent(source).Replicas == nil { - group := &desired.Spec.Template.PodCliqueScalingGroupConfigs[0] - for _, existing := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { - if existing.Name == group.Name { - group.Replicas = existing.Replicas - break - } - } - } - changed, synced, err := commoncontroller.SyncObservedResource( ctx, r, @@ -298,23 +351,17 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { return state, ctrl.Result{}, err } - if err := r.reconcileLPXScalingGroupScale(ctx, source, synced, selected.plan.LPXScalingGroup, false); err != nil { - return state, ctrl.Result{}, err + // Apply scale-out, or scale-down not already persisted before request cleanup. + if !scaled { + if _, err := r.reconcileLPXScalingGroupScale(ctx, source, synced, selected.plan.LPXScalingGroup, false); err != nil { + return state, ctrl.Result{}, err + } } readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, source, deployment, synced) if err != nil { return state, ctrl.Result{}, err } deployment.Status.Components = readiness.ComponentStatuses - shape, err := dynamo.ResolveLPXGPUShape(ctx, r.Client, synced) - if err != nil { - return state, ctrl.Result{}, err - } - // The serving component owns shared GPU costs exactly once. - name := selected.workload.LPXComponentName() - component := deployment.Status.Components[name] - component.GPUsPerEngine, component.GPUsPerReplica = ptr.To(shape.GPUsPerEngine), ptr.To(shape.GPUsPerReplica) - deployment.Status.Components[name] = component state.Reason, state.Message = readiness.Classification, readiness.Message if readiness.Ready && !changed { if err := r.deleteStaleLPXConfigMaps(ctx, deployment, resources); err != nil { @@ -327,51 +374,85 @@ func (r *graphReconciler) reconcileWorkload(ctx context.Context, deployment *v1a return state, ctrl.Result{}, err } state = overlayLPXResult(state, classification) - return state, projectLPXLifecycleStatus(deployment, classification), nil + return state, projectLPXLifecycleStatus(classification), nil +} + +// preserveLPXScalingGroupReplicaSeed mutates nonnil desired to retain the +// observed PCS template's scale-owned replica seed. observed is nil only when +// the first PCS has not been created yet. +func preserveLPXScalingGroupReplicaSeed( + desired *grovev1alpha1.PodCliqueSet, + observed *grovev1alpha1.PodCliqueSet, +) error { + if observed == nil { + return nil + } + + // LPX renders and owns exactly one complete-engine scaling group. + desiredConfigs := desired.Spec.Template.PodCliqueScalingGroupConfigs + observedConfigs := observed.Spec.Template.PodCliqueScalingGroupConfigs + if len(desiredConfigs) != 1 || len(observedConfigs) != 1 { + return fmt.Errorf("LPX PodCliqueSet %q requires exactly one scaling-group template", desired.Name) + } + if desiredConfigs[0].Name != observedConfigs[0].Name { + return fmt.Errorf("LPX PodCliqueSet %q scaling-group template changed from %q to %q", + desired.Name, observedConfigs[0].Name, desiredConfigs[0].Name) + } + + // Copy the pointer value so desired never aliases the observed object. + desiredConfigs[0].Replicas = nil + if observedConfigs[0].Replicas != nil { + replicas := *observedConfigs[0].Replicas + desiredConfigs[0].Replicas = &replicas + } + return nil } // reconcileLPXScalingGroupScale applies explicit component replicas to the // Grove group. Grove seeds this group once, so later replica changes use its -// scale subresource without rewriting the PCS template. +// scale subresource without rewriting the PCS template. pcs is nil when the +// desired PodCliqueSet was not found, in which case there is no group to scale. func (r *graphReconciler) reconcileLPXScalingGroupScale( ctx context.Context, source *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, groupName string, scaleDownOnly bool, -) error { +) (bool, error) { replicas := lpx.ServingComponent(source).Replicas if replicas == nil || pcs == nil { - return nil + return false, nil } group := &grovev1alpha1.PodCliqueScalingGroup{} if err := r.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: groupName}, group); err != nil { if apierrors.IsNotFound(err) { - return nil + return false, nil } - return err + return false, err } - if !metav1.IsControlledBy(group, pcs) || !group.DeletionTimestamp.IsZero() { - return fmt.Errorf("LPX scaling group lacks the current PCS owner") + if !metav1.IsControlledBy(group, pcs) { + return false, fmt.Errorf("LPX scaling group lacks the current PCS owner") + } + if !group.DeletionTimestamp.IsZero() { + return false, nil } if group.Spec.Replicas == *replicas || scaleDownOnly && group.Spec.Replicas < *replicas { - return nil + return false, nil } scale := &autoscalingv1.Scale{ ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, } - return r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)) + if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { + return false, err + } + return true, nil } // projectLPXLifecycleStatus transfers reconcile-owned results to the same child before returning. -func projectLPXLifecycleStatus(deployment *v1alpha1.LPXGraphDeployment, classification lpxClassification) ctrl.Result { - if transition, ok := classification.(*lpxDeadlineTransition); ok { - if deployment.Status.Placement == nil { - deployment.Status.Placement = &v1beta1.PlacementStatus{} - } - deployment.Status.Placement.LPXAttempt = transition.attempt - return ctrl.Result{RequeueAfter: transition.requeueAfter} +func projectLPXLifecycleStatus(classification lpxClassification) ctrl.Result { + if deadline, ok := classification.(*lpxDeadlineExceeded); ok { + return ctrl.Result{RequeueAfter: deadline.requeueAfter} } if _, retiring := classification.(*lpxRetiring); retiring { return ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter} @@ -379,15 +460,12 @@ func projectLPXLifecycleStatus(deployment *v1alpha1.LPXGraphDeployment, classifi return ctrl.Result{} } -func (r *graphReconciler) retireInvalidWorkload(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, reason string) (ctrl.Result, error) { - retiring, err := r.retireInvalidLPXWorkload(ctx, deployment, reason) - if err != nil { - return ctrl.Result{}, err - } - if retiring != nil { - return ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter}, nil +func lpxDeadlineFlags(classification lpxClassification) (bool, bool) { + deadline, ok := classification.(*lpxDeadlineExceeded) + if !ok { + return false, false } - return ctrl.Result{}, nil + return true, deadline.recordFailure } // Publication is an identity-sensitive write boundary: an uncached read fences diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index d7172c47e424..0a2c1782002a 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -24,9 +24,9 @@ import ( "github.com/stretchr/testify/require" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" @@ -69,13 +69,15 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) group.Spec.Replicas = 9 createLPXTestObjects(t, t.Context(), r.Client, objects...) + pcs = observedLPXTestPodCliqueSet(t, t.Context(), r, child, selected) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) beforePCS := pcs.DeepCopy() t.Log("Rebuild scheduler intent from explicit replicas or the current native scale") - selected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) - require.Nil(t, rejected) + selected, classification, err := r.prepareLPXMaterializing(t.Context(), child, source, pcs) + require.NoError(t, err) + require.Nil(t, classification) require.Equal(t, ptr.Deref(tc.replicas, int32(9)), selected.plan.Replicas) require.Len(t, selected.requests, int(selected.plan.Replicas)) updates := 0 @@ -96,7 +98,7 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { }) t.Log("Only explicit changes write scale, preserving the observed resource-version precondition") - _, _, err = r.reconcileWorkload(t.Context(), child, source, selected) + _, _, err = r.reconcileWorkload(t.Context(), child, source, selected, pcs) require.ErrorIs(t, err, tc.err) require.Equal(t, tc.wantUpdates, updates) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) @@ -113,6 +115,72 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { } } +func TestLPXTerminatingGroveResourcesRemainRetiring(t *testing.T) { + t.Log("Publish a workload whose PCS and scaling group are both terminating") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + for _, object := range []client.Object{pcs, group} { + object.SetFinalizers([]string{"example.com/cleanup"}) + require.NoError(t, r.Update(ctx, object)) + require.NoError(t, r.Delete(ctx, object)) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.False(t, pcs.DeletionTimestamp.IsZero()) + + t.Log("Let the publication fence report retirement before attempting scale") + state, result, err := r.reconcileWorkload(ctx, child, source, selected, pcs) + require.NoError(t, err) + require.Equal(t, lpxRetiringReason, state.Reason) + require.Equal(t, lpxRetirementRequeueAfter, result.RequeueAfter) +} + +func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { + t.Log("Observe an externally inflated group through a cache that stays stale after scale") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) + group.Spec.Replicas = 9 + createLPXTestObjects(t, ctx, r.Client, objects...) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + stale := group.DeepCopy() + scaled, updates := false, 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if scaled && key == client.ObjectKeyFromObject(stale) { + if observed, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { + stale.DeepCopyInto(observed) + return nil + } + } + return delegated.Get(ctx, key, object, opts...) + }, + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + updates++ + if err := delegated.SubResource(subresource).Update(ctx, object, opts...); err != nil { + return err + } + scaled = true + return nil + }, + }) + + t.Log("Continue reconciliation without issuing a second stale-resource-version write") + _, _, err := r.reconcileWorkload(ctx, child, source, selected, pcs) + require.NoError(t, err) + require.Equal(t, 1, updates) +} + func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { ctx := t.Context() child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -146,12 +214,7 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { group := &grovev1alpha1.PodCliqueScalingGroup{} require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) require.Equal(t, int32(1), group.Spec.Replicas) - stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) - require.True(t, stale.DeletionTimestamp.IsZero()) - - t.Log("Delete the stale request while its scheduler finalizer remains pending") - _, err = r.Reconcile(ctx, request) - require.NoError(t, err) + t.Log("Let request finalization proceed asynchronously after the lower scale is persisted") stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) require.False(t, stale.DeletionTimestamp.IsZero()) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) @@ -191,83 +254,349 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { require.True(t, replacement.DeletionTimestamp.IsZero()) } -func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { - for _, tc := range []struct { - name string - missing bool - }{{name: "deadline"}, {name: "missing request", missing: true}} { - t.Run(tc.name, func(t *testing.T) { - t.Log("Observe a completed engine and a newer scheduling batch in their shared PCS") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - pcs := findLPXTestPodCliqueSet(t, objects) - serving := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status - require.NoError(t, r.Update(ctx, serving)) - pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) - pending.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} - require.NoError(t, r.Update(ctx, pending)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - attempt := deadlineTestAttempt(child, pending, time.Now().Add(-time.Second)) - if tc.missing { - attempt.DeadlineAt = nil - pending.Finalizers = nil - require.NoError(t, r.Update(ctx, pending)) - require.NoError(t, r.Delete(ctx, pending)) - } else { - attempt.ExceededAt = ptr.To(metav1.Now()) - } - child.Status.Placement.LPXAttempt = attempt - require.NoError(t, r.Status().Update(ctx, child)) +func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { + t.Log("Publish one engine whose capacity is owned by the native Grove scaling group") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = nil + r, initial := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, initial) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, initial) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, initial) + pcsUID := pcs.UID + templateReplicas := ptr.Deref(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0) + require.Positive(t, templateReplicas) + request := getLPXRequest(t, ctx, r.Client, child.Namespace, initial.requests[0].requestName) + requestUID := request.UID - t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) - for range 3 { - _, err := r.Reconcile(ctx, request) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) - require.Equal(t, serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - } - if tc.missing { - requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) - return + t.Log("Scale the native Grove group to zero and derive empty scheduler intent") + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: initial.plan.LPXScalingGroup}, group)) + group.Spec.Replicas = 0 + require.NoError(t, r.Update(ctx, group)) + zero, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) + require.NoError(t, err) + require.Nil(t, classification) + require.Zero(t, zero.plan.Replicas) + require.Empty(t, zero.requests) + + t.Log("Delete the stale LPR before reconciling the stable PCS template") + _, _, err = r.reconcileWorkload(ctx, child, source, zero, pcs) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) + _, _, err = r.reconcileWorkload(ctx, child, source, zero, pcs) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Zero(t, group.Spec.Replicas) + storedPCS := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, pcsUID, storedPCS.UID) + require.Equal(t, templateReplicas, ptr.Deref(storedPCS.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0)) + + t.Log("Scale the native group back up and publish a fresh request through the same PCS") + group.Spec.Replicas = 1 + require.NoError(t, r.Update(ctx, group)) + scaledUp, classification, err := r.prepareLPXMaterializing(ctx, child, source, storedPCS) + require.NoError(t, err) + require.Nil(t, classification) + require.Equal(t, int32(1), scaledUp.plan.Replicas) + require.Len(t, scaledUp.requests, 1) + _, _, err = r.reconcileWorkload(ctx, child, source, scaledUp, storedPCS) + require.NoError(t, err) + replacement := getLPXRequest(t, ctx, r.Client, child.Namespace, scaledUp.requests[0].requestName) + require.NotEqual(t, requestUID, replacement.UID) + require.Equal(t, pcsUID, metav1.GetControllerOf(replacement).UID) +} + +func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { + ctx := t.Context() + child, source, registry := newLPXSpecDecodeTestDGD(t) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + + scaledSource := source.DeepCopy() + require.Len(t, scaledSource.Spec.Components, 2) + scaledSource.Spec.Components[1].Replicas = ptr.To(int32(1)) + scaled, rejected := requirePreparedLPX(t, r, ctx, child, scaledSource) + require.Nil(t, rejected) + require.Len(t, scaled.requests, 2) + + desiredNames := make(map[string]struct{}, len(scaled.requests)) + for _, request := range scaled.requests { + desiredNames[request.requestName] = struct{}{} + } + var stale *lpxv1alpha1.LPUPipelineRequest + for _, request := range selected.requests { + if _, retained := desiredNames[request.requestName]; retained { + continue + } + stale = getLPXRequest(t, ctx, r.Client, child.Namespace, request.requestName) + break + } + require.NotNil(t, stale) + stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, stale)) + + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), pcs)) + pcs.Finalizers = []string{"test.example/observe-retirement"} + require.NoError(t, r.Update(ctx, pcs)) + retiredTemplate := selected.plan.Agents[1].TemplateName + originalCliqueCount := len(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + + t.Log("Retire the immutable PCS without mutating Grove membership or directly deleting its LPRs") + pcsDeleteObserved := false + requestDeleteObserved := false + wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + switch object.(type) { + case *grovev1alpha1.PodCliqueSet: + pcsDeleteObserved = true + case *lpxv1alpha1.LPUPipelineRequest: + requestDeleteObserved = true } + return delegated.Delete(ctx, object, opts...) + }, + }) + r.Client, r.apiReader = wrapped, wrapped + _, classification, err := r.reconcileLPXPublicationFence(ctx, child, scaled, pcs) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + require.True(t, pcsDeleteObserved) + require.False(t, requestDeleteObserved) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.False(t, pcs.DeletionTimestamp.IsZero()) + require.Contains(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames, retiredTemplate) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.True(t, stale.DeletionTimestamp.IsZero()) - t.Log("A later edit still waits for the failed request's scheduler finalizer") - pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) - require.False(t, pending.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Generation++ - require.NoError(t, r.Update(ctx, child)) - _, err := r.Reconcile(ctx, request) + t.Log("Fence replacement until owner garbage collection removes every old LPR") + pcs.Finalizers = nil + require.NoError(t, r.Update(ctx, pcs)) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{}))) + _, classification, err = r.reconcileLPXPublicationFence(ctx, child, scaled, nil) + require.NoError(t, err) + require.IsType(t, &lpxRetiring{}, classification) + + requests, err := r.listLPXRequestCandidates(ctx, child) + require.NoError(t, err) + for index := range requests.Items { + request := &requests.Items[index] + request.Finalizers = nil + require.NoError(t, r.Update(ctx, request)) + require.NoError(t, r.Delete(ctx, request)) + } + _, classification, err = r.reconcileLPXPublicationFence(ctx, child, scaled, nil) + require.NoError(t, err) + require.Nil(t, classification) + + t.Log("Create the replacement PCS with the scaled-in immutable shape") + replacement := renderLPXTestPodCliqueSet(t, ctx, r, child, scaledSource, scaled) + changed, synced, err := commoncontroller.SyncObservedResource( + ctx, r, child, nil, replacement, commoncontroller.WithPreservedListOrder(), + ) + require.NoError(t, err) + require.True(t, changed) + require.Equal(t, replacement.Spec, synced.Spec) + require.Equal(t, scaled.workloadDigest, synced.Annotations[lpx.WorkloadDigestAnnotation]) + require.Less(t, len(synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames), originalCliqueCount) +} + +func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { + t.Run("deadline", func(t *testing.T) { + t.Log("Observe a completed engine and a newer pending request in their shared PCS") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + serving := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(ctx, serving)) + pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) + pending.Status = deadlineTestRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + pending.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, pending)) + + t.Log("Persist failure before changing Grove or deleting scheduler intent") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) + _, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.True(t, pending.DeletionTimestamp.IsZero()) + + t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") + for range 3 { + _, err = r.Reconcile(ctx, request) require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) + require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(1), group.Spec.Replicas) + } + t.Log("A later edit still waits for the failed request's scheduler finalizer") + pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.False(t, pending.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) - t.Log("After cleanup the later edit can retry without replacing the serving engine") - retiredUID := pending.UID - pending.Finalizers = nil - require.NoError(t, r.Update(ctx, pending)) - for range 5 { - _, err = r.Reconcile(ctx, request) - require.NoError(t, err) - } - replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) - require.NotEqual(t, retiredUID, replacement.UID) - require.True(t, replacement.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - require.Equal(t, serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)) - }) + t.Log("After cleanup the later edit can retry without replacing the serving engine") + retiredUID := pending.UID + pending.Finalizers = nil + require.NoError(t, r.Update(ctx, pending)) + for range 5 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + } + replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NotEqual(t, retiredUID, replacement.UID) + require.True(t, replacement.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") + }) +} + +func TestLPXDeadlineContinuesDuringRequestRetirement(t *testing.T) { + t.Log("Publish two pending replicas and expire the tail first") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + prefix := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + prefix.Status = deadlineTestRequest(child, pcs, prefix.Name, time.Now(), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, prefix)) + tail := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) + tail.Status = deadlineTestRequest(child, pcs, tail.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + tail.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, tail)) + + t.Log("Retire the expired tail while its scheduler finalizer holds deletion") + reconcileRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 2 { + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) } + tail = getLPXRequest(t, ctx, r.Client, child.Namespace, tail.Name) + require.False(t, tail.DeletionTimestamp.IsZero()) + + t.Log("Continue enforcing the surviving request's independent deadline") + prefix = getLPXRequest(t, ctx, r.Client, child.Namespace, prefix.Name) + prefix.Status = deadlineTestRequest(child, pcs, prefix.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, prefix)) + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, prefix.Name) +} + +func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { + t.Log("Publish a request, then advance the deployment before that request expires") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + pcs := findLPXTestPodCliqueSet(t, objects) + pending.Status = deadlineTestRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, pending)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + failureGeneration := child.Generation + + t.Log("Expire and remove the request at the already-edited generation") + reconcileRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pending), &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("After the failure fence is durable, scale down and remove the expired request") + _, err = r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionTrue, failed.Status) + require.Equal(t, lpxSchedulingDeadlineExceededReason, failed.Reason) + require.Equal(t, failureGeneration, failed.ObservedGeneration) + schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, lpxSchedulingFailedCondition) + require.NotNil(t, schedulingFailed) + require.Equal(t, metav1.ConditionTrue, schedulingFailed.Status) + require.Equal(t, failureGeneration, schedulingFailed.ObservedGeneration) + + t.Log("Reconciliation at the failure generation does not recreate scheduler intent") + for range 3 { + _, err = r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + } + + t.Log("An edit after the failure permits a fresh request") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + for range 3 { + _, err = r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + } + replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NotEqual(t, pending.UID, replacement.UID) +} + +func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { + t.Log("Publish two requests and remove one without a terminal deadline decision") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + + t.Log("Advance the deployment and remove one request before it fails") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + + missing := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) + previousUID := missing.UID + require.NoError(t, r.Delete(ctx, missing)) + + t.Log("Reconciliation recreates deterministic scheduler intent without inheriting a scheduling clock") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + replacement := getLPXRequest(t, ctx, r.Client, missing.Namespace, missing.Name) + require.NotEqual(t, previousUID, replacement.UID) + require.Nil(t, replacement.Status) } func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { @@ -308,7 +637,7 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, afterSelected) require.Equal(t, beforePCS, afterPCS) require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) - _, err := reconcileSelectedLPXForTest(t.Context(), r, child, afterSelected) + _, err := reconcileSelectedLPXForTest(t, t.Context(), r, child, afterSelected) require.NoError(t, err) afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(t.Context(), afterRequests)) @@ -318,19 +647,13 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { require.Equal(t, beforeSource, source) } -func TestLPXCurrentAttemptSurvivesUnobservedReconciliation(t *testing.T) { - t.Log("A failed reconciliation may have a current durable attempt and an older result generation") +func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { + t.Log("A request-list failure prevents deriving any active deadline") source := newLPXTestSource(lpx.PipelineSingle, "build-v2") child := newLPXTestDeployment(t, source) source.Spec.Scheduling = deadlineTestScheduling() - child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ - ObservedGeneration: child.Generation, - DeadlineAt: ptr.To(metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(time.Minute))), - Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "current-request", UID: "current-uid"}}, - }} r := newLPXTestReconciler(t, nil, child, source) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - before := child.Status.Placement.DeepCopy() readError := errors.New("temporary request observation failure") r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { @@ -341,48 +664,13 @@ func TestLPXCurrentAttemptSurvivesUnobservedReconciliation(t *testing.T) { }, }) result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err, "active deadlines bound transient errors with a scheduled retry") - require.Positive(t, result.RequeueAfter) + require.ErrorIs(t, err, readError) + require.Zero(t, result) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) require.NotEqual(t, child.Generation, child.Status.ObservedGeneration) - require.Equal(t, before, child.Status.Placement) require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Failed").Message, readError.Error()) } -func TestLPXGPUCapacityReportsTheCompleteEngine(t *testing.T) { - for _, mode := range []string{"lpu-only", "hybrid", "speculative"} { - t.Run(mode, func(t *testing.T) { - t.Log("Reconcile the actual LPX workload and publish its per-engine GPU shape") - var child *v1alpha1.LPXGraphDeployment - var source *v1beta1.DynamoGraphDeployment - var registry lpx.ModelRegistry - var want int64 - switch mode { - case "lpu-only": - child, source, registry = newLPXTestDGD(t, lpx.PipelineSingle) - case "hybrid": - child, source, registry = newLPXTestDGD(t, lpx.PipelineLPX) - source.Spec.Components[0].Replicas = ptr.To(int32(3)) - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - pod := &lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec - pod.ResourceClaims = nil - pod.Containers[0].Resources.Limits = corev1.ResourceList{consts.KubeResourceGPUNvidia: resource.MustParse("4")} - want = 8 - case "speculative": - child, source, registry = newLPXSpecDecodeTestDGD(t) - } - r := newLPXTestReconciler(t, registry, child, source) - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - require.Contains(t, child.Status.Components, "lpx") - require.Equal(t, ptr.To(want), child.Status.Components["lpx"].GPUsPerEngine) - require.Equal(t, ptr.To(want), child.Status.Components["lpx"].GPUsPerReplica) - - }) - } -} - func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { for _, scenario := range []struct { name, reason, message string @@ -546,7 +834,7 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T }, }) r.Client, r.apiReader = wrapped, wrapped - retiring, err := r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + retiring, err := r.retireInvalidLPXWorkload(t.Context(), child, pcs, "invalid runtime") switch scenario { case "new child generation": require.ErrorContains(t, err, "authority changed") @@ -567,13 +855,13 @@ func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T if scenario == "pending finalizer" { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) require.False(t, pcs.DeletionTimestamp.IsZero()) - retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, pcs, "invalid runtime") require.NoError(t, err) require.NotNil(t, retiring) pcs.Finalizers = nil require.NoError(t, r.Update(t.Context(), pcs)) } - retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, "invalid runtime") + retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, nil, "invalid runtime") require.NoError(t, err) require.Nil(t, retiring) require.Equal(t, 3, deletes, "cleanup must not repeat deletes after they are accepted") @@ -664,7 +952,7 @@ func TestLPXEndpointLifecycle(t *testing.T) { t.Log("Retire the recreated discovery endpoint when the workload becomes invalid") source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - _, err := r.retireInvalidLPXWorkload(t.Context(), child, "invalid source") + _, err := r.retireInvalidLPXWorkload(t.Context(), child, nil, "invalid source") require.NoError(t, err) require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) require.NoError(t, r.Get(t.Context(), modelKey, modelService)) @@ -705,7 +993,7 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { } t.Log("Keep scheduler-request provenance rooted in the source DGD, not the independently named child") - requests, err := r.listOwnedLPXRequests(t.Context(), child) + requests, err := r.listOwnedLPXRequests(t.Context(), child, pcs) require.NoError(t, err) require.NotEmpty(t, requests) for _, request := range requests { @@ -763,12 +1051,39 @@ func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { require.NoError(t, err) require.NoError(t, r.Update(t.Context(), child)) - t.Log("Keep one PCS and endpoint while updating their serving component metadata") + t.Log("Preserve the PCS on rename and retire it when target and draft assignments change") _, err = r.Reconcile(t.Context(), request) require.NoError(t, err) + if change == "move conductor" { + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, &grovev1alpha1.PodCliqueSet{}))) + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + require.Equal(t, lpxRetiringReason, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + require.Equal(t, "stable-endpoint", string(endpoint.UID)) + + t.Log("Fence replacement until the retired PCS's requests are garbage collected") + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, &grovev1alpha1.PodCliqueSet{}))) + requests, err := r.listLPXRequestCandidates(t.Context(), child) + require.NoError(t, err) + require.NotEmpty(t, requests.Items) + for index := range requests.Items { + require.Equal(t, "stable-pcs", string(metav1.GetControllerOf(&requests.Items[index]).UID)) + require.NoError(t, r.Delete(t.Context(), &requests.Items[index])) + } + + t.Log("Recreate the workload under the same readable names after retirement") + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + } require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - require.Equal(t, "stable-pcs", string(pcs.UID)) + if change == "rename serving component" { + require.Equal(t, "stable-pcs", string(pcs.UID)) + } else { + require.NotEqual(t, "stable-pcs", string(pcs.UID)) + } require.Equal(t, "stable-endpoint", string(endpoint.UID)) require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) require.Equal(t, beforeGroup.Name, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Name) @@ -1048,7 +1363,7 @@ func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, err := r.reconcileWorkload(t.Context(), child, source, selected) + _, _, err := r.reconcileWorkload(t.Context(), child, source, selected, nil) require.ErrorContains(t, err, "input revision") allPCS := &grovev1alpha1.PodCliqueSetList{} require.NoError(t, r.List(t.Context(), allPCS)) diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement.go b/deploy/operator/internal/controller/lpx/deadline_retirement.go new file mode 100644 index 000000000000..0d3bd5807e4c --- /dev/null +++ b/deploy/operator/internal/controller/lpx/deadline_retirement.go @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "fmt" + "math" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// retireExpiredLPXRequests lowers the one live Grove group before deleting the +// affected engine suffix. pcs is nil after its disappearance, which leaves its +// requests to owner garbage collection. +func (r *graphReconciler) retireExpiredLPXRequests( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + source *nvidiacomv1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + requests []lpxv1alpha1.LPUPipelineRequest, + expired []*lpxv1alpha1.LPUPipelineRequest, +) error { + if len(expired) == 0 { + return nil + } + owner := metav1.GetControllerOf(expired[0]) + if owner == nil || owner.Name == "" || owner.UID == "" { + return fmt.Errorf("LPX request %q lacks an exact PodCliqueSet owner", expired[0].Name) + } + + // A missing, replaced, or deleting PCS leaves its requests to owner GC. + if pcs == nil || pcs.Name != owner.Name || pcs.UID != owner.UID || !pcs.DeletionTimestamp.IsZero() { + return nil + } + if !metav1.IsControlledBy(pcs, deployment) { + return fmt.Errorf("refusing to scale PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + } + for _, request := range expired { + if !metav1.IsControlledBy(request, pcs) { + return fmt.Errorf("expired LPX request %q does not have the current PodCliqueSet owner", request.Name) + } + } + + groupName, _, err := lpxScalingGroupTarget(expired[0]) + if err != nil { + return err + } + group := &grovev1alpha1.PodCliqueScalingGroup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: groupName}, group); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if !metav1.IsControlledBy(group, pcs) { + return fmt.Errorf("LPX scaling group lacks the current PCS owner") + } + if !group.DeletionTimestamp.IsZero() { + return nil + } + replicas, retiring, err := lpxExpiredEngineSuffix(requests, expired, group.Spec.Replicas) + if err != nil { + return err + } + if len(retiring) == 0 { + return nil + } + + if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { + return err + } + // Omitted replicas leave capacity externally managed; expire only scheduler + // intent so a later input revision can reuse the unchanged Grove ordinal. + if lpx.ServingComponent(source).Replicas != nil && group.Spec.Replicas > replicas { + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, + } + if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { + return err + } + } + + // Grove and the scheduler converge asynchronously after the ordered writes. + return r.deleteStaleLPXRequests(ctx, deployment, retiring) +} + +// lpxExpiredEngineSuffix returns complete engine replicas only when every +// expired replica belongs to one contiguous trailing suffix. It orders the +// expired requests last so transient sibling cleanup leaves durable expiry +// evidence for the next reconciliation. An interior failure blocks all +// scale-down so a healthy higher ordinal is never removed. +func lpxExpiredEngineSuffix( + requests []lpxv1alpha1.LPUPipelineRequest, + expired []*lpxv1alpha1.LPUPipelineRequest, + replicas int32, +) (int32, []*lpxv1alpha1.LPUPipelineRequest, error) { + if len(expired) == 0 { + return replicas, nil, nil + } + groupName, _, err := lpxScalingGroupTarget(expired[0]) + if err != nil { + return 0, nil, err + } + failedReplicas := make(map[int64]struct{}, len(expired)) + for _, request := range expired { + name, replica, err := lpxScalingGroupTarget(request) + if err != nil { + return 0, nil, err + } + if name != groupName { + return 0, nil, fmt.Errorf("expired LPX requests target different scaling groups") + } + failedReplicas[replica] = struct{}{} + } + + targetReplicas := int64(replicas) + for targetReplicas > 0 { + if _, failed := failedReplicas[targetReplicas-1]; !failed { + break + } + targetReplicas-- + } + for replica := range failedReplicas { + if replica < targetReplicas { + return replicas, nil, nil + } + } + + expiredNames := make(map[string]struct{}, len(expired)) + for _, request := range expired { + expiredNames[request.Name] = struct{}{} + } + retiring := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) + failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) + for index := range requests { + request := &requests[index] + name, replica, err := lpxScalingGroupTarget(request) + if err != nil { + return 0, nil, err + } + if name != groupName { + return 0, nil, fmt.Errorf("LPX requests target different scaling groups") + } + if replica < targetReplicas { + continue + } + if _, expired := expiredNames[request.Name]; expired { + failed = append(failed, request) + } else { + retiring = append(retiring, request) + } + } + return int32(targetReplicas), append(retiring, failed...), nil +} + +func lpxScalingGroupTarget(request *lpxv1alpha1.LPUPipelineRequest) (string, int64, error) { + target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef + if target == nil || target.Name == "" || target.ReplicaIndex < 0 || target.ReplicaIndex >= math.MaxInt32 { + return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) + } + return target.Name, target.ReplicaIndex, nil +} diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go new file mode 100644 index 000000000000..034200b79e64 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go @@ -0,0 +1,278 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestLPXExpiredScaleOutPreservesServingEngine(t *testing.T) { + t.Log("Publish one bound engine and one expired scale-out request") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + serving := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(ctx, serving)) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + + t.Log("Lower Grove and delete only the expired scheduler intent in one reconciliation") + err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) + require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") +} + +func TestLPXExpiredModelRetiresItsEngineSiblings(t *testing.T) { + t.Log("Publish the multiple model requests belonging to one speculative engine") + ctx := t.Context() + child, source, registry := newLPXSpecDecodeTestDGD(t) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + require.Greater(t, len(desired.requests), 1) + requests, err := r.listOwnedLPXRequests(ctx, child, findLPXTestPodCliqueSet(t, objects)) + require.NoError(t, err) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[len(desired.requests)-1].requestName) + + t.Log("Map one expired model request to every request sharing its engine suffix") + replicas, retiring, err := lpxExpiredEngineSuffix(requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, 1) + require.NoError(t, err) + require.Zero(t, replicas) + require.Len(t, retiring, len(requests)) +} + +func TestLPXExpiredModelPreservesExpiryProofUntilSiblingCleanupStarts(t *testing.T) { + t.Log("Publish multiple model requests for one engine and expire one of them") + ctx := t.Context() + child, source, registry := newLPXSpecDecodeTestDGD(t) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + require.Greater(t, len(requests), 1) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + var sibling *lpxv1alpha1.LPUPipelineRequest + for index := range requests { + if requests[index].Name != expired.Name { + sibling = &requests[index] + break + } + } + require.NotNil(t, sibling) + + t.Log("Fail the first sibling deletion and retain the expired request for a retry") + deleteErr := errors.New("transient sibling deletion failure") + base := r.Client + r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok && request.Name == sibling.Name { + return deleteErr + } + return delegated.Delete(ctx, object, opts...) + }, + }) + err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) + require.ErrorIs(t, err, deleteErr) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("Retry from the retained expiry proof and remove the complete engine request set") + r.Client = base + requests, err = r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + expired = getLPXRequest(t, ctx, r.Client, child.Namespace, expired.Name) + require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) + for index := range desired.requests { + requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, desired.requests[index].requestName) + } +} + +func TestLPXExpiredInteriorReplicaDoesNotScaleDown(t *testing.T) { + t.Log("Publish three engine replicas and expire only the middle ordinal") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(3)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + pcs := findLPXTestPodCliqueSet(t, objects) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + + t.Log("Leave the PCSG, its PodCliques, and the interior LPR untouched") + require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(3), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + for _, object := range objects { + if clique, ok := object.(*grovev1alpha1.PodClique); ok { + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(clique), &grovev1alpha1.PodClique{})) + } + } +} + +func TestLPXExpiredInteriorReplicaBlocksTrailingSuffix(t *testing.T) { + t.Log("Publish three engines with both an interior failure and a failed tail") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(3)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + pcs := findLPXTestPodCliqueSet(t, objects) + interior := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + tail := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[2].requestName) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + + t.Log("Do not partially mutate the group while an expired hole remains") + require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{interior, tail})) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(3), group.Spec.Replicas) + for _, request := range []*lpxv1alpha1.LPUPipelineRequest{interior, tail} { + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{})) + } +} + +func TestLPXExpiredMaximalTrailingRunScalesDown(t *testing.T) { + t.Log("Publish four engines and expire the complete trailing pair") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(4)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + pcs := findLPXTestPodCliqueSet(t, objects) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + expired := []*lpxv1alpha1.LPUPipelineRequest{ + getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[2].requestName), + getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[3].requestName), + } + + require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, expired)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + for index, projection := range desired.requests { + err := r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: projection.requestName}, &lpxv1alpha1.LPUPipelineRequest{}) + if index < 2 { + require.NoError(t, err) + } else { + require.True(t, apierrors.IsNotFound(err)) + } + } +} + +func TestLPXDeadlineRetirementPreservesImplicitReplicaCapacity(t *testing.T) { + t.Log("Publish one externally managed engine and expire its scheduler intent") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = nil + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + + t.Log("Delete the expired LPR without making observed zero capacity the new desired state") + require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) + + t.Log("Continue deriving the implicit desired count from the preserved Grove capacity") + selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) + require.NoError(t, err) + require.Nil(t, classification) + require.Equal(t, int32(1), selected.plan.Replicas) +} + +func TestLPXExpiredRequestWaitsForMissingScalingGroup(t *testing.T) { + t.Log("Keep an expired request while its live PCS has not materialized a scaling group") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + require.NoError(t, r.Delete(ctx, group)) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("Once Grove creates the group, issue scale-down before deleting the LPR") + group.ResourceVersion = "" + group.UID = "replacement-group" + group.DeletionTimestamp = nil + group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + require.NoError(t, r.Create(ctx, group)) + err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Zero(t, group.Spec.Replicas) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) +} + +func TestLPXExpiredRequestsWaitForOwnerGarbageCollection(t *testing.T) { + t.Log("Keep an expired request observable after its owner PCS disappears") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) + requests, err := r.listOwnedLPXRequests(ctx, child, pcs) + require.NoError(t, err) + require.NoError(t, r.Delete(ctx, pcs)) + + err = r.retireExpiredLPXRequests(ctx, child, source, nil, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) +} diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 2d6fffb147b5..15d5e250f981 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -9,7 +9,6 @@ import ( "context" "crypto/sha256" "encoding/binary" - "encoding/json" "errors" "fmt" "hash" @@ -37,11 +36,10 @@ import ( ) const ( - lpxAttemptDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-attempt-digest" lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation lpxOwnerUIDLabel = lpxDeploymentUIDAnnotation lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" - lpxPCSUIDAnnotation = "scheduling.lpu.nvidia.com/podcliqueset-uid" + lpxPodCliqueSetKind = "PodCliqueSet" lpxRetirementRequeueAfter = 5 * time.Second maxDGDConditionMessageSize = 32768 ) @@ -63,33 +61,38 @@ type lpxRejected struct { } type lpxMaterializing struct { - workloadDigest string - requests []lpxModelMaterializing - workload *lpx.SelectedWorkload - plan *lpx.MaterializationPlan - deadlineSeconds *int64 + workloadDigest string + requests []lpxModelMaterializing + workload *lpx.SelectedWorkload + plan *lpx.MaterializationPlan } type reconcileOutcome struct { - State nvidiacomv1beta1.DGDState - Reason string - Message string + State nvidiacomv1beta1.DGDState + Reason string + Message string + retirementScope lpxRetirementScope } +type lpxRetirementScope uint8 + +const ( + lpxWorkloadRetirement lpxRetirementScope = iota + lpxRequestRetirement +) + // lpxModelMaterializing is the request-local state for one model projection. // The enclosing lpxMaterializing owns the shared Grove identity. type lpxModelMaterializing struct { requestName string - attemptDigest string modelProjection *lpx.ModelProjection replicaIndex int32 agentCliqueName string } type lpxGroveIdentity struct { - pcsUID types.UID - pcs *grovev1alpha1.PodCliqueSet - plan *lpx.MaterializationPlan + pcs *grovev1alpha1.PodCliqueSet + plan *lpx.MaterializationPlan } type lpxClosed struct { @@ -104,6 +107,7 @@ type lpxSchedulerObserved lpxv1alpha1.LPUPipelineRequestStatus type lpxRetiring struct { retirementReason string + scope lpxRetirementScope } func (*lpxRejected) lpxClassification() {} @@ -131,12 +135,14 @@ func lpxResult(classification lpxClassification) reconcileOutcome { result.Message = "LPX scheduling is bound; runtime activation and serving readiness remain external" case *lpxSchedulerObserved: result = lpxSchedulerResult(state) - result.Message = lpxMessageWithDiagnostics(result.Message, state.Diagnostics) case *lpxRetiring: result.Reason = lpxRetiringReason result.Message = state.retirementReason - case *lpxDeadlineTransition: - result = state.result + result.retirementScope = state.scope + case *lpxDeadlineExceeded: + result.State = nvidiacomv1beta1.DGDStateFailed + result.Reason = lpxSchedulingDeadlineExceededReason + result.Message = "An LPX request exceeded its scheduling deadline" default: result.State = nvidiacomv1beta1.DGDStateFailed result.Reason = "LPXInternalClassificationError" @@ -145,7 +151,7 @@ func lpxResult(classification lpxClassification) reconcileOutcome { return result } -// lpxSchedulerResult classifies the receipt without formatting diagnostics before selection. +// lpxSchedulerResult classifies one receipt and formats its diagnostics once. func lpxSchedulerResult(state *lpxSchedulerObserved) reconcileOutcome { result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} var summary string @@ -238,15 +244,17 @@ func overlayLPXResult( // reconcileLPXKnownIntentFence deletes a publication before any new graph can // be rendered when selection, DGD generation, model, or DGD incarnation changes. +// pcs is nil when its stable name was not found. func (r *graphReconciler) reconcileLPXKnownIntentFence( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, requests []lpxv1alpha1.LPUPipelineRequest, ) (lpxClassification, []lpxv1alpha1.LPUPipelineRequest, error) { var err error if requests == nil { - requests, err = r.listOwnedLPXRequests(ctx, deployment) + requests, err = r.listOwnedLPXRequests(ctx, deployment, pcs) } if err != nil { return nil, nil, err @@ -265,17 +273,20 @@ func (r *graphReconciler) reconcileLPXKnownIntentFence( if selected { reason = "The DGD incarnation, generation, model, or selected input changed" } - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, reason, false) + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, reason) return retiring, nil, retireErr } return nil, requests, nil } +// prepareLPXMaterializing resolves the selected workload against the observed +// Grove capacity. pcs is nil when the stable PodCliqueSet name was not found. func (r *graphReconciler) prepareLPXMaterializing( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, -) (*lpxMaterializing, *lpxRejected, error) { + pcs *grovev1alpha1.PodCliqueSet, +) (*lpxMaterializing, lpxClassification, error) { workload, err := lpx.ResolveSelectedWorkload(ctx, source, r.modelRegistry) if err != nil { if errors.Is(err, lpx.ErrBuildSnapshotAcquisition) { @@ -296,12 +307,13 @@ func (r *graphReconciler) prepareLPXMaterializing( return nil, nil, err } } else { - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: plan.PodCliqueSetName}, pcs); err != nil { - return nil, nil, err + // Reject an unrelated PCS before consulting any group it happens to own. + if pcs != nil && !metav1.IsControlledBy(pcs, deployment) { + return nil, nil, fmt.Errorf("refusing to use replicas from a foreign LPX PodCliqueSet") } - if !metav1.IsControlledBy(pcs, deployment) || !metav1.IsControlledBy(group, pcs) { - return nil, nil, fmt.Errorf("refusing to use replicas from a foreign LPX scaling group") + // A group outliving its PCS is normal during background Grove cleanup. + if pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(group, pcs) { + return nil, &lpxRetiring{retirementReason: "Waiting for the previous LPX scaling group to finish cleanup"}, nil } plan.Replicas = group.Spec.Replicas if err := plan.ValidateReplicaCount(); err != nil { @@ -329,19 +341,20 @@ func (r *graphReconciler) prepareLPXMaterializing( } } return &lpxMaterializing{ - workloadDigest: workloadDigest, - requests: requests, - workload: workload, - plan: plan, - deadlineSeconds: lpxAttemptDeadlineSeconds(source), + workloadDigest: workloadDigest, + requests: requests, + workload: workload, + plan: plan, }, nil, nil } // retireInvalidLPXWorkload also retires staging that never acquired an LPR. // Healthy publication and spec-write fences intentionally do not use this path. +// pcs is nil when its stable name was not found. func (r *graphReconciler) retireInvalidLPXWorkload( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, reason string, ) (*lpxRetiring, error) { selector := client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)} @@ -376,7 +389,7 @@ func (r *graphReconciler) retireInvalidLPXWorkload( } } - requests, err := r.listOwnedLPXRequests(ctx, deployment) + requests, err := r.listOwnedLPXRequests(ctx, deployment, pcs) if err != nil { return nil, err } @@ -434,8 +447,8 @@ func writeLPXHashField(h hash.Hash, tag, value string) { } } -// hasCurrentLPXAttemptAnnotations requires nonnil desired and its DGD; a nil object interface does not match. -func (desired *lpxMaterializing) hasCurrentLPXAttemptAnnotations( +// hasCurrentLPXPublicationAnnotations requires nonnil desired and its DGD; a nil object interface does not match. +func (desired *lpxMaterializing) hasCurrentLPXPublicationAnnotations( object metav1.Object, deployment *nvidiacomv1alpha1.LPXGraphDeployment, ) bool { @@ -458,12 +471,13 @@ func lpxRequestName(dgdName, digest string) string { } // reconcileLPXPublicationFence requires a nonnil desired whose requests remain immutable during the call. -// A missing PCS cannot be recreated until garbage collection removes every request from its previous incarnation. +// pcs is nil when the desired PodCliqueSet was not found; it cannot be recreated until garbage collection +// removes every request from its previous incarnation. func (r *graphReconciler) reconcileLPXPublicationFence( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, - podCliqueSetMissing bool, + pcs *grovev1alpha1.PodCliqueSet, ) (map[string]*lpxv1alpha1.LPUPipelineRequest, lpxClassification, error) { // Retain each desired position for digest lookup and collision precedence. desiredByName := make(map[string]int, len(desired.requests)) @@ -480,10 +494,7 @@ func (r *graphReconciler) reconcileLPXPublicationFence( for index := range requests.Items { request := &requests.Items[index] desiredIndex, desiredName := desiredByName[request.Name] - ownedByChild, err := r.ownsLPXRequest(ctx, deployment, request) - if err != nil { - return nil, nil, err - } + ownedByChild := ownsLPXRequest(deployment, pcs, request) if !ownedByChild { // Preserve desired-order error precedence without retaining collision state. if desiredName { @@ -497,63 +508,52 @@ func (r *graphReconciler) reconcileLPXPublicationFence( } stale = append(stale, request) } - if len(stale) > 0 { - // Persist cancellation before deleting scheduler intent for removed engines. - if attempt := currentLPXAttemptStatus(deployment); attempt != nil && attempt.ExceededAt == nil { - want := desiredLPXAttemptStatus(desired, deployment.Generation, attempt.PodCliqueSetUID) - if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, want, currents); transition != nil { - return currents, transition, nil - } - } - if err := r.deleteStaleLPXRequests(ctx, deployment, stale); err != nil { - return nil, nil, err - } - return currents, &lpxRetiring{retirementReason: "One or more engines are no longer requested"}, nil - } - // Keep exact cleanup authority until the replaced request is authoritatively absent. - if attempt := currentLPXAttemptStatus(deployment); attempt != nil { - for index, row := range attempt.Requests { - if row.UID == "" || row.AttemptDigest != "" { - continue - } - if request := currents[row.Name]; request != nil && request.UID == row.UID { - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, "The engine's previous request is retiring", true) - return currents, retiring, retireErr - } - next := attempt.DeepCopy() - next.Requests[index].UID = "" - return currents, newLPXDeadlineTransition(next, nvidiacomv1beta1.DGDStatePending, - lpxAttemptPreparedReason, "The previous engine request has finished cleanup", time.Nanosecond), nil - } - } // A deterministic-name collision must never be adopted or deleted. if foreignCollision < len(desired.requests) { return nil, nil, fmt.Errorf("LPX request name %q is occupied by a foreign owner", desired.requests[foreignCollision].requestName) } - if podCliqueSetMissing && len(currents) > 0 { + if pcs == nil && len(currents)+len(stale) > 0 { return currents, &lpxRetiring{retirementReason: "Waiting for garbage collection to remove the previous LPX requests"}, nil } - - // Keep scale-up fenced until terminal cleanup can retire the failed engines. - if attempt := currentLPXAttemptStatus(deployment); attempt != nil { - _, _, missing := exactLPXAttemptState(attempt, currents) - if attempt.ExceededAt != nil || missing { - return currents, &lpxRetiring{retirementReason: "Waiting for the terminal scheduling batch to finish cleanup"}, nil + if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != desired.workloadDigest { + if err := r.deleteLPXPodCliqueSet(ctx, deployment, pcs.Name, pcs.UID); err != nil { + return nil, nil, err } + return currents, &lpxRetiring{retirementReason: "The selected LPX workload shape changed"}, nil + } + if pcs != nil && !pcs.DeletionTimestamp.IsZero() { + return currents, &lpxRetiring{retirementReason: "Waiting for the current PodCliqueSet to finish cleanup"}, nil + } + if len(stale) > 0 { + if err := r.deleteStaleLPXRequests(ctx, deployment, stale); err != nil { + return nil, nil, err + } + return currents, &lpxRetiring{ + retirementReason: "One or more engines are no longer requested", + scope: lpxRequestRetirement, + }, nil } // Grove must not recreate pods while a retiring request still protects their names. for _, request := range desired.requests { if live := currents[request.requestName]; live != nil && !live.DeletionTimestamp.IsZero() { - return currents, &lpxRetiring{retirementReason: "Waiting for the current LPX request to finish cleanup"}, nil + return currents, &lpxRetiring{ + retirementReason: "Waiting for the current LPX request to finish cleanup", + scope: lpxRequestRetirement, + }, nil } } + + // A recorded deadline failure fences this generation even if the scheduler later reports a terminal phase. + if lpxDeadlineFailureCurrent(deployment) { + return currents, &lpxDeadlineExceeded{}, nil + } return currents, nil, nil } -// listLPXRequestCandidates uses the propagated child identity only -// to narrow the authoritative read. Owner references and recorded UIDs remain -// the authority for adopting or deleting any returned request. +// listLPXRequestCandidates uses the propagated deployment identity to avoid a +// namespace-wide scan. Ownership is still checked before a returned request is +// observed or changed. func (r *graphReconciler) listLPXRequestCandidates( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -566,30 +566,47 @@ func (r *graphReconciler) listLPXRequestCandidates( return list, err } -// ownsLPXRequest follows the request's exact controller chain, including a PCS -// whose name predates the current source input. -func (r *graphReconciler) ownsLPXRequest(ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, request *lpxv1alpha1.LPUPipelineRequest) (bool, error) { +// observeCurrentLPXPodCliqueSet returns nil when the deployment's stable PCS +// name is absent from the informer cache. +func (r *graphReconciler) observeCurrentLPXPodCliqueSet( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, +) (*grovev1alpha1.PodCliqueSet, error) { + pcs := &grovev1alpha1.PodCliqueSet{} + err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: dynamo.PCSNameForLPX(deployment)}, pcs) + if apierrors.IsNotFound(err) { + return nil, nil + } + return pcs, err +} + +// ownsLPXRequest follows the request's exact controller chain using the PCS +// already observed by the caller. pcs is nil after that PCS disappears. +func ownsLPXRequest( + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + request *lpxv1alpha1.LPUPipelineRequest, +) bool { owner := metav1.GetControllerOf(request) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != "PodCliqueSet" { - return false, nil + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != lpxPodCliqueSetKind { + return false } - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: owner.Name}, pcs); err == nil { - return metav1.IsControlledBy(pcs, deployment) && metav1.IsControlledBy(request, pcs), nil - } else if !apierrors.IsNotFound(err) { - return false, err + if pcs != nil { + return metav1.IsControlledBy(pcs, deployment) && metav1.IsControlledBy(request, pcs) } - // Every labeled request for the recorded PCS remains retireable after garbage - // collection removes that owner, including requests that had already left the - // active deadline batch. - attempt := currentLPXAttemptStatus(deployment) - return attempt != nil && owner.UID == attempt.PodCliqueSetUID, nil + // The deployment-UID label keeps owner-GC observation level-driven after the + // PCS disappears. The request's controller kind still prevents adopting an + // unrelated labeled object while the exact UID remains in its owner reference. + return request.Labels[lpxOwnerUIDLabel] == string(deployment.UID) } +// listOwnedLPXRequests filters the deployment-scoped list against the caller's +// PCS observation. pcs is nil after that owner disappears. func (r *graphReconciler) listOwnedLPXRequests( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, ) ([]lpxv1alpha1.LPUPipelineRequest, error) { requests, err := r.listLPXRequestCandidates(ctx, deployment) if err != nil { @@ -598,11 +615,7 @@ func (r *graphReconciler) listOwnedLPXRequests( owned := make([]lpxv1alpha1.LPUPipelineRequest, 0) for index := range requests.Items { request := &requests.Items[index] - ownedByChild, err := r.ownsLPXRequest(ctx, deployment, request) - if err != nil { - return nil, err - } - if ownedByChild { + if ownsLPXRequest(deployment, pcs, request) { owned = append(owned, *request) } } @@ -618,26 +631,26 @@ func (r *graphReconciler) deleteLPXPodCliqueSet( deployment *nvidiacomv1alpha1.LPXGraphDeployment, pcsName string, pcsUID types.UID, -) (bool, error) { +) error { pcs := &grovev1alpha1.PodCliqueSet{} if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { if apierrors.IsNotFound(err) { - return false, nil + return nil } - return false, err + return err } if pcs.UID != pcsUID { - return false, nil + return nil } if !metav1.IsControlledBy(pcs, deployment) { - return false, fmt.Errorf("refusing to delete PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) + return fmt.Errorf("refusing to delete PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) } if !pcs.DeletionTimestamp.IsZero() { - return true, nil + return nil } if deployment.DeletionTimestamp.IsZero() { if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return false, err + return err } } propagation := metav1.DeletePropagationBackground @@ -645,9 +658,9 @@ func (r *graphReconciler) deleteLPXPodCliqueSet( Preconditions: &metav1.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, PropagationPolicy: &propagation, }); err != nil && !apierrors.IsNotFound(err) { - return false, err + return err } - return true, nil + return nil } // deleteStaleLPXRequests removes scaled-in scheduler intent without disturbing @@ -679,6 +692,25 @@ func (r *graphReconciler) deleteStaleLPXRequests( return nil } +// retireLPXEngineRequest removes the PCS whose immutable request no longer +// matches the selected workload. +func (r *graphReconciler) retireLPXEngineRequest( + ctx context.Context, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + request *lpxv1alpha1.LPUPipelineRequest, + reason string, +) (lpxClassification, error) { + owner := metav1.GetControllerOf(request) + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || + owner.Kind != lpxPodCliqueSetKind || owner.UID == "" { + return nil, fmt.Errorf("LPX request %q does not have an exact PodCliqueSet controller owner", request.Name) + } + if err := r.deleteLPXPodCliqueSet(ctx, deployment, owner.Name, owner.UID); err != nil { + return nil, err + } + return &lpxRetiring{retirementReason: reason}, nil +} + func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -699,37 +731,17 @@ func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( return classification, err } - // Record the immutable PCS and request identities before publication. The - // attempt status is the durable ownership fence used during GC-driven cleanup; - // a scheduling deadline only adds a clock to that record. - attemptProjection := desiredLPXAttemptStatus(desired, deployment.Generation, pcs.UID) - if transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents); transition != nil { - return transition, nil - } for index := range desired.requests { request := &desired.requests[index] if currents[request.requestName] != nil { continue } - // Classify known authority loss before attempting a replacement publication. - attempt := currentLPXAttemptStatus(deployment) - if _, _, missing := exactLPXAttemptState(attempt, currents); missing { - return newLPXDeadlineTransition(attempt, nvidiacomv1beta1.DGDStateFailed, - lpxAttemptAuthorityLostReason, "An exact LPX request disappeared; retiring its scheduling batch without replacement", 0), nil - } - if err := r.revalidateLPXAttemptPublication(ctx, deployment, desired.plan.PodCliqueSetName); err != nil { - return nil, err - } - groveIdentity := &lpxGroveIdentity{pcsUID: pcs.UID, pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} + groveIdentity := &lpxGroveIdentity{pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity) if createErr != nil { return nil, createErr } currents[request.requestName] = live - transition := r.reconcileLPXAttemptPreparation(deployment, desired.deadlineSeconds, attemptProjection, currents) - if transition != nil { - return transition, nil - } } return classifyLPXCurrentRequests(desired.requests, currents), nil @@ -777,7 +789,7 @@ func (r *graphReconciler) reconcileSelectedLPXPodCliqueSet( } for _, request := range desired.requests { if live := currents[request.requestName]; live != nil { - retiring, err := r.retireLPXEngineRequest(ctx, deployment, live, fmt.Sprintf("The published PCS identity became unavailable: %q", incomplete), false) + retiring, err := r.retireLPXEngineRequest(ctx, deployment, live, fmt.Sprintf("The published PCS identity became unavailable: %q", incomplete)) return nil, retiring, err } } @@ -787,15 +799,10 @@ func (r *graphReconciler) reconcileSelectedLPXPodCliqueSet( // The PCS owns each request; LPX resolves the replica-local clique references itself. for index := range desired.requests { request := &desired.requests[index] - identity := &lpxGroveIdentity{pcsUID: pcs.UID, pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} - spec, err := json.Marshal(request.modelProjection.RequestSpec(identity.plan, request.agentCliqueName)) - if err != nil { - return nil, nil, err - } - request.attemptDigest = fmt.Sprintf("sha256:%x", sha256.Sum256(spec)) + identity := &lpxGroveIdentity{pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} if live := currents[request.requestName]; live != nil { if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, live, err.Error(), true) + retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, live, err.Error()) return nil, retiring, retireErr } } @@ -838,7 +845,7 @@ func (r *graphReconciler) createPublishedLPXRequest( consts.KubeLabelDynamoGraphDeploymentName: metav1.GetControllerOf(deployment).Name, lpxOwnerUIDLabel: string(deployment.UID), }, - Annotations: lpxRequestAnnotations(deployment, materializing, grove), + Annotations: lpxRequestAnnotations(deployment, materializing), }, Spec: spec, } @@ -859,17 +866,14 @@ func (r *graphReconciler) createPublishedLPXRequest( func lpxRequestAnnotations( deployment *nvidiacomv1alpha1.LPXGraphDeployment, materializing *lpxModelMaterializing, - grove *lpxGroveIdentity, ) map[string]string { // Callers validate the exact source controller owner before constructing requests. sourceOwner := metav1.GetControllerOf(deployment) return map[string]string{ - lpxAttemptDigestAnnotation: materializing.attemptDigest, lpx.DGDUIDAnnotation: string(sourceOwner.UID), lpx.DeploymentNameAnnotation: deployment.Name, lpxDeploymentUIDAnnotation: string(deployment.UID), lpxModelAnnotation: materializing.modelProjection.Model(), - lpxPCSUIDAnnotation: string(grove.pcsUID), lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), lpxv1alpha1.CompilerSnapshotDigestAnnotation: materializing.modelProjection.CompilerSnapshotDigest(), } @@ -887,15 +891,15 @@ func validateCurrentLPXRequest( if request.UID == "" { return fmt.Errorf("LPX request %q has no immutable UID", request.Name) } - wantAnnotations := lpxRequestAnnotations(deployment, desired, grove) + wantAnnotations := lpxRequestAnnotations(deployment, desired) for key, value := range wantAnnotations { if request.Annotations[key] != value { - return fmt.Errorf("LPX request %q immutable annotation %q no longer matches the current attempt", request.Name, key) + return fmt.Errorf("LPX request %q immutable annotation %q no longer matches the current deployment", request.Name, key) } } wantSpec := desired.modelProjection.RequestSpec(grove.plan, desired.agentCliqueName) if !apiequality.Semantic.DeepEqual(request.Spec, wantSpec) { - return fmt.Errorf("LPX request %q immutable intent no longer matches the current attempt", request.Name) + return fmt.Errorf("LPX request %q immutable intent no longer matches the current publication", request.Name) } return nil } @@ -922,7 +926,7 @@ func (r *graphReconciler) observeLPXPodCliqueSet( } incomplete := "" - if !desired.hasCurrentLPXAttemptAnnotations(pcs, deployment) { + if !desired.hasCurrentLPXPublicationAnnotations(pcs, deployment) { incomplete = "The selected Grove PodCliqueSet is stale" } if pcs.Spec.Replicas != 1 { diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 5db4da600ee3..8c57edd76e1f 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -115,8 +115,13 @@ func requirePreparedLPX( source *nvidiacomv1beta1.DynamoGraphDeployment, ) (*lpxMaterializing, *lpxRejected) { t.Helper() - desired, rejected, err := reconciler.prepareLPXMaterializing(ctx, dgd, source) + desired, classification, err := reconciler.prepareLPXMaterializing(ctx, dgd, source, nil) require.NoError(t, err) + if classification == nil { + return desired, nil + } + rejected, ok := classification.(*lpxRejected) + require.True(t, ok, "classification=%T", classification) return desired, rejected } @@ -158,7 +163,7 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { require.NotContains(t, group.Spec.CliqueNames, "") cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) require.NotContains(t, cyborg.Spec.StartsAfter, "") - require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) + require.Equal(t, corev1.DefaultSchedulerName, cyborg.Spec.PodSpec.SchedulerName) createLPXTestObjects(t, ctx, reconciler.Client, objects...) classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) @@ -222,7 +227,7 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.IsType(t, &lpxOpen{}, classification) - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd, findLPXTestPodCliqueSet(t, objects)) require.NoError(t, err) require.Len(t, requests, 3) @@ -265,13 +270,49 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( }}, } require.NoError(t, reconciler.Update(ctx, target)) - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.NoError(t, err) require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) require.Equal(t, "LPXUnsupported", lpxResult(classification).Reason) require.Contains(t, lpxResult(classification).Message, "TargetUnsupported") } +func TestLPXImplicitReplicaObservationWaitsForMatchingPodCliqueSet(t *testing.T) { + t.Log("Render the stable Grove identities for an externally managed replica count") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = nil + base, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, base, child, source, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + replacement := pcs.DeepCopy() + replacement.UID = "replacement-pcs" + + for _, test := range []struct { + name string + pcs *grovev1alpha1.PodCliqueSet + }{ + {name: "PCS has already disappeared"}, + {name: "replacement PCS does not own the old group", pcs: replacement}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Keep the old PCSG visible during background owner cleanup") + seed := []client.Object{group.DeepCopy()} + if test.pcs != nil { + seed = append(seed, test.pcs.DeepCopy()) + } + r := newLPXTestReconciler(t, registry, child.DeepCopy(), source.DeepCopy(), seed...) + + t.Log("Classify the transient owner mismatch as retirement, not reconciliation failure") + selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, test.pcs) + require.NoError(t, err) + require.Nil(t, selected) + require.IsType(t, &lpxRetiring{}, classification) + }) + } +} + func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { for _, test := range []struct { name string @@ -522,7 +563,7 @@ func TestLPXPublicationFencePrecedesPodCliqueSetWrite(t *testing.T) { }) t.Log("A failed publication observation prevents initial creation") - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, true) + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, nil) require.ErrorIs(t, err, fenceErr) require.Nil(t, classification) require.Nil(t, currents) @@ -533,7 +574,7 @@ func TestLPXPublicationFencePrecedesPodCliqueSetWrite(t *testing.T) { t.Log("Recover the observation and create the exact child-owned PCS") failFence = false writes, lists = 0, 0 - currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, true) + currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, nil) require.NoError(t, err) require.Nil(t, classification) require.Empty(t, currents) @@ -584,6 +625,34 @@ func TestLPXPublicationFencePrecedesPodCliqueSetWrite(t *testing.T) { require.Equal(t, live.Annotations[lpx.WorkloadDigestAnnotation], stored.Annotations[lpx.WorkloadDigestAnnotation]) } +func TestLPXPublicationFenceReusesObservedPodCliqueSetForRequestOwnership(t *testing.T) { + t.Log("Publish two requests owned by the already-observed PodCliqueSet") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + pcsGets := 0 + r.apiReader = interceptor.NewClient(r.apiReader.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + pcsGets++ + } + return delegated.Get(ctx, key, object, opts...) + }, + }) + + t.Log("Classify every LPR against that observation without per-request API reads") + currents, classification, err := r.reconcileLPXPublicationFence(ctx, child, desired, pcs) + require.NoError(t, err) + require.Nil(t, classification) + require.Len(t, currents, len(desired.requests)) + require.Zero(t, pcsGets) +} + func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodCliqueSet(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -596,18 +665,18 @@ func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodClique require.NoError(t, reconciler.Delete(ctx, pcs)) t.Log("Keep the replacement PCS fenced while its predecessor's request remains persisted") - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) require.NoError(t, err) require.Len(t, currents, 1) require.IsType(t, &lpxRetiring{}, classification) - t.Log("Garbage collection alone cannot authorize replacing a recorded scheduling attempt") + t.Log("Permit idempotent replacement after garbage collection removes the previous request") request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) require.NoError(t, reconciler.Delete(ctx, request)) - currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) require.NoError(t, err) require.Empty(t, currents) - require.IsType(t, &lpxRetiring{}, classification) + require.Nil(t, classification) } func TestLPXScaleDownDeletesOnlyStaleRequests(t *testing.T) { @@ -631,18 +700,18 @@ func TestLPXScaleDownDeletesOnlyStaleRequests(t *testing.T) { t.Log("Scale from three replicas to one while every request remains pending") scaledDown := withLPXTestReplicas(t, desired, 1) - _, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + _, classification, err := reconciler.reconcileLPXPublicationFence( + ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), + ) require.NoError(t, err) - transition, ok := classification.(*lpxDeadlineTransition) - require.True(t, ok) - require.Len(t, transition.attempt.Requests, 1) - dgd.Status.Placement.LPXAttempt = transition.attempt - require.NoError(t, reconciler.Status().Update(ctx, dgd)) + require.IsType(t, &lpxRetiring{}, classification) t.Log("Delete every stale LPR concurrently without replacing the shared PCS") - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + currents, classification, err := reconciler.reconcileLPXPublicationFence( + ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), + ) require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) + require.Nil(t, classification) require.Len(t, currents, 1) for _, projection := range desired.requests[1:] { requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, projection.requestName) @@ -671,20 +740,18 @@ func TestLPXScaleUpWaitsForTerminatingScaledDownRequest(t *testing.T) { t.Log("Scale down and begin direct request deletion") scaledDown := withLPXTestReplicas(t, desired, 1) - _, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) - require.NoError(t, err) - transition, ok := classification.(*lpxDeadlineTransition) - require.True(t, ok) - dgd.Status.Placement.LPXAttempt = transition.attempt - require.NoError(t, reconciler.Status().Update(ctx, dgd)) - _, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, scaledDown, false) + _, classification, err := reconciler.reconcileLPXPublicationFence( + ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), + ) require.NoError(t, err) require.IsType(t, &lpxRetiring{}, classification) terminating := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) require.False(t, terminating.DeletionTimestamp.IsZero()) t.Log("Scale back up while the same request name is still terminating") - _, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, false) + _, classification, err = reconciler.reconcileLPXPublicationFence( + ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), + ) require.NoError(t, err) require.IsType(t, &lpxRetiring{}, classification) storedPCS := &grovev1alpha1.PodCliqueSet{} @@ -704,6 +771,58 @@ func TestLPXScaleUpWaitsForTerminatingScaledDownRequest(t *testing.T) { require.Equal(t, pcsUID, storedPCS.UID) } +func TestLPXElapsedDeadlineDoesNotFenceBoundGeneration(t *testing.T) { + t.Log("Publish a request that reaches Bound before its persisted deadline is observed again") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Status = deadlineTestRequest(dgd, deadlineTestPCS(dgd, metav1.GetControllerOf(request).UID), request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, reconciler.Update(ctx, request)) + + t.Log("Terminal scheduler disposition wins the deadline race and permits ordinary observation") + currents, classification, err := reconciler.reconcileLPXPublicationFence( + ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), + ) + require.NoError(t, err) + require.Len(t, currents, 1) + require.Nil(t, classification) + classification, wake, err := reconciler.reconcileLPXRequestDeadlines( + ctx, dgd, source, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), desired.requests, + ) + require.NoError(t, err) + require.Nil(t, classification) + require.True(t, wake.IsZero()) +} + +func TestLPXRecordedDeadlineFailureFencesLateBoundRequest(t *testing.T) { + t.Log("Record expiry before the scheduler reports a terminal phase") + ctx := t.Context() + dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + source.Spec.Scheduling = deadlineTestScheduling() + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) + publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) + request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + request.Status = deadlineTestRequest(dgd, deadlineTestPCS(dgd, metav1.GetControllerOf(request).UID), request.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, reconciler.Update(ctx, request)) + dgd.Status.Conditions = []metav1.Condition{{ + Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, + Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, + }} + + t.Log("Keep the recorded failure authoritative for the current generation") + currents, classification, err := reconciler.reconcileLPXPublicationFence( + ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), + ) + require.NoError(t, err) + require.Len(t, currents, 1) + require.IsType(t, &lpxDeadlineExceeded{}, classification) +} + func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -910,12 +1029,10 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { } rederived, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) require.Nil(t, rejected) - wantDigest := desired.requests[0].attemptDigest desired = rederived - classification, err := reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err := reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) - require.Equal(t, wantDigest, desired.requests[0].attemptDigest) t.Log("Observe the exact current scheduler receipt after the controller restart") request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) @@ -923,7 +1040,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { request.Status = newLPXNodeLocalBoundStatus(7, 3, "sha256:bound-plan") require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.NoError(t, err) bound, ok := classification.(*lpxBound) require.True(t, ok) @@ -941,7 +1058,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) request.Status.ObservedGeneration = ptr.To(int64(6)) require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) @@ -953,7 +1070,7 @@ func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { Code: "StaleUnsupported", Subject: "request/old", Detail: "belongs to an older generation", }} require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.NoError(t, err) require.IsType(t, &lpxOpen{}, classification) require.NotContains(t, lpxResult(classification).Message, "StaleUnsupported") @@ -1099,6 +1216,8 @@ func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { require.Contains(t, result.Message, "ExactDiagnostic") require.Contains(t, result.Message, "requests/current") require.Contains(t, result.Message, "the exact scheduler detail") + require.Equal(t, 1, strings.Count(result.Message, "LPX diagnostics:")) + require.Equal(t, 1, strings.Count(result.Message, "the exact scheduler detail")) if test.wantMessage != "" { require.Contains(t, result.Message, test.wantMessage) } @@ -1138,7 +1257,7 @@ func TestLPXSchedulerDiagnosticMessageRespectsConditionLimit(t *testing.T) { require.True(t, strings.HasSuffix(message, "…")) } -func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { +func TestLPXRequestIdentityAndRetirementFence(t *testing.T) { type digestInput struct { namespace string name string @@ -1180,28 +1299,30 @@ func TestLPXAttemptIdentityAndRetirementFence(t *testing.T) { dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - t.Log("Fence the exact PCS owner and immutable attempt annotations") + t.Log("Fence the exact PCS owner and immutable deployment annotations") projection := &desired.requests[0] - identity := &lpxGroveIdentity{pcsUID: "pcs-uid", pcs: deadlineTestPCS(dgd, "pcs-uid"), plan: desired.plan} + identity := &lpxGroveIdentity{pcs: deadlineTestPCS(dgd, "pcs-uid"), plan: desired.plan} current := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection, identity), OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(identity.pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}}, + ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection), OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(identity.pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}}, Spec: projection.modelProjection.RequestSpec(identity.plan, projection.agentCliqueName), } require.NoError(t, validateCurrentLPXRequest(dgd, projection, identity, current)) changed := *identity - changed.pcsUID = "replacement-pcs-uid" - require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "immutable annotation") + changed.pcs = identity.pcs.DeepCopy() + changed.pcs.UID = "replacement-pcs-uid" + require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "exact PCS owner") createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) dgd.Generation++ - classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + pcs := observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired) + classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, pcs, nil) require.NoError(t, err) require.Nil(t, classification) require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.requests[0].requestName}, &lpxv1alpha1.LPUPipelineRequest{})) - classification, _, err = reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + classification, _, err = reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, pcs, nil) require.NoError(t, err) require.Nil(t, classification) } @@ -1212,7 +1333,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { {"Grove", "Grove is disabled"}, } { t.Run(scenario.name, func(t *testing.T) { - t.Log("Publish a complete workload, discovery service and scheduling attempt") + t.Log("Publish a complete workload, discovery service and scheduling request") ctx := t.Context() child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) @@ -1225,10 +1346,6 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { require.NoError(t, err) } require.NoError(t, r.Get(ctx, key, child)) - require.NotNil(t, child.Status.Placement) - require.NotNil(t, child.Status.Placement.LPXAttempt) - require.NotNil(t, child.Status.Placement.LPXAttempt.DeadlineAt) - placement := child.Status.Placement.DeepCopy() before := []client.ObjectList{ &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}, &corev1.ServiceList{}, &lpxv1alpha1.LPUPipelineRequestList{}, @@ -1249,16 +1366,12 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { originalSpec := source.Spec.DeepCopy() source.Spec.Components[0].LPX.BuildID = "edited-while-disabled" require.NoError(t, r.Update(ctx, source)) - child.Status.Placement.LPXAttempt.DeadlineAt = ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))) - require.NoError(t, r.Status().Update(ctx, child)) - disabledPlacement := child.Status.Placement.DeepCopy() for range 2 { result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.NoError(t, err) require.Zero(t, result) } require.NoError(t, r.Get(ctx, key, child)) - require.Equal(t, disabledPlacement, child.Status.Placement) failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") require.NotNil(t, failed) require.Equal(t, metav1.ConditionTrue, failed.Status) @@ -1275,8 +1388,6 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { t.Log("Re-enable unchanged intent and resume the original PCS and request identities") source.Spec = *originalSpec require.NoError(t, r.Update(ctx, source)) - child.Status.Placement = placement - require.NoError(t, r.Status().Update(ctx, child)) r.runtimeConfig.Gate.LPX, r.runtimeConfig.Gate.Grove = true, true r.modelRegistry = registry _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) @@ -1329,7 +1440,7 @@ func TestLPXFinalizationAfterRequestAPIRemoval(t *testing.T) { }) t.Log("Ordinary reconciliation still fails when requests cannot be listed") - _, err := r.listOwnedLPXRequests(ctx, child) + _, err := r.listOwnedLPXRequests(ctx, child, nil) require.ErrorIs(t, err, scenario.listErr) t.Log("Release the legacy finalizer without reading requests; owner garbage collection owns cleanup") @@ -1358,7 +1469,7 @@ func TestLPXControllerPreservesPublishedWorkWhenProviderEditIsInvalid(t *testing require.Nil(t, request.DeletionTimestamp) } -func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t *testing.T) { +func TestLPXPublicationFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t *testing.T) { t.Log("Give foreign requests the desired names in the opposite lexical order") ctx := t.Context() source := newLPXTestSource(lpx.PipelineSingle, "build-v2") @@ -1386,7 +1497,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t reconciler := newLPXTestReconciler(t, nil, dgd, source) t.Log("Complete an empty preflight fence before foreign requests occupy the desired names") - currents, retiring, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + currents, retiring, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) require.NoError(t, err) require.Empty(t, currents) require.Nil(t, retiring) @@ -1395,10 +1506,10 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t } t.Log("Exclude foreign requests from owned publications and the known-intent retirement fence") - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd) + requests, err := reconciler.listOwnedLPXRequests(ctx, dgd, nil) require.NoError(t, err) require.Empty(t, requests) - classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil) + classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil, nil) require.NoError(t, err) require.Nil(t, classification) @@ -1433,7 +1544,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t }, }) - currents, retiring, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, true) + currents, retiring, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) require.EqualError(t, err, fmt.Sprintf("LPX request name %q is occupied by a foreign owner", foreign.Name)) require.Nil(t, currents) require.Nil(t, retiring) @@ -1445,7 +1556,7 @@ func TestLPXAttemptFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t require.Equal(t, lpxOwnerUIDLabel+"="+string(dgd.UID), requestListCalls[0].LabelSelector.String()) t.Log("Refresh publication state and refuse the foreign requests created after the empty preflight") - classification, err = reconcileSelectedLPXForTest(ctx, reconciler, dgd, desired) + classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) require.ErrorContains(t, err, "occupied by a foreign owner") require.Nil(t, classification) require.Len(t, requestListCalls, 2, "final publication must refresh the authoritative request snapshot") @@ -1511,7 +1622,7 @@ func renderLPXTestPodCliqueSet( ) *grovev1alpha1.PodCliqueSet { t.Helper() rendered, _, err := renderPodCliqueSet(ctx, source, reconciler.Config, reconciler.runtimeConfig, - reconciler.Client, reconciler.DockerSecretRetriever, desired.workload, desired.plan, dgd) + reconciler.DockerSecretRetriever, desired.workload, desired.plan, dgd) require.NoError(t, err) return rendered } @@ -1877,22 +1988,36 @@ func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.C } } +func observedLPXTestPodCliqueSet( + t *testing.T, + ctx context.Context, + reconciler *graphReconciler, + deployment *nvidiacomv1alpha1.LPXGraphDeployment, + desired *lpxMaterializing, +) *grovev1alpha1.PodCliqueSet { + t.Helper() + if desired.plan == nil { + return nil + } + pcs := &grovev1alpha1.PodCliqueSet{} + err := reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs) + if apierrors.IsNotFound(err) { + return nil + } + require.NoError(t, err) + return pcs +} + func reconcileSelectedLPXForTest( + t *testing.T, ctx context.Context, reconciler *graphReconciler, deployment *nvidiacomv1alpha1.LPXGraphDeployment, desired *lpxMaterializing, ) (lpxClassification, error) { - podCliqueSetMissing := false - if desired.plan != nil { - pcs := &grovev1alpha1.PodCliqueSet{} - err := reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs) - if err != nil && !apierrors.IsNotFound(err) { - return nil, err - } - podCliqueSetMissing = apierrors.IsNotFound(err) - } - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, deployment, desired, podCliqueSetMissing) + t.Helper() + pcs := observedLPXTestPodCliqueSet(t, ctx, reconciler, deployment, desired) + currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, deployment, desired, pcs) if classification != nil || err != nil { return classification, err } @@ -1907,21 +2032,9 @@ func publishSelectedLPXForTest( desired *lpxMaterializing, ) lpxClassification { t.Helper() - for range len(desired.requests) + 2 { - classification, err := reconcileSelectedLPXForTest(ctx, reconciler, deployment, desired) - require.NoError(t, err) - transition, preparing := classification.(*lpxDeadlineTransition) - if !preparing { - return classification - } - if deployment.Status.Placement == nil { - deployment.Status.Placement = &nvidiacomv1beta1.PlacementStatus{} - } - deployment.Status.Placement.LPXAttempt = transition.attempt - require.NoError(t, reconciler.Status().Update(ctx, deployment)) - } - t.Fatal("LPX publication did not converge") - return nil + classification, err := reconcileSelectedLPXForTest(t, ctx, reconciler, deployment, desired) + require.NoError(t, err) + return classification } func getLPXRequest(t *testing.T, ctx context.Context, kubeClient client.Reader, namespace, name string) *lpxv1alpha1.LPUPipelineRequest { diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 5c3ef3ece676..ae4aa0ea9f33 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -283,14 +283,10 @@ func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: ptr.To(metav1.Now()), } - child.Status.Placement = &v1beta1.PlacementStatus{LPXAttempt: &v1beta1.LPXAttemptStatus{ - ObservedGeneration: child.Generation, - DeadlineAt: ptr.To(metav1.NewTime(time.Now().Add(-time.Minute))), - Requests: []v1beta1.LPXAttemptRequestStatus{{Name: "previous-request", UID: "previous-uid"}}, - }} + child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Generation++ previousGeneration := child.Status.ObservedGeneration - require.Nil(t, lpxAttemptDeadlineSeconds(source)) + require.Nil(t, lpxRequestDeadlineSeconds(source)) registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestSecondBuildID: true}, nil) r := newLPXTestReconciler(t, registry, child, source) r.runtimeConfig.Gate = features.Gates{LPX: true} diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go index f996a714988f..d030f3a331a7 100644 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ b/deploy/operator/internal/controller/lpx/preflight.go @@ -22,6 +22,7 @@ import ( "fmt" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -30,13 +31,15 @@ import ( // reconcileLPXSafetyPreflight classifies and fences scheduler-owned state // after the LPX controller validates source identity and the Grove provider. +// pcs is nil when its stable name was not found. func (r *graphReconciler) reconcileLPXSafetyPreflight( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, requests []lpxv1alpha1.LPUPipelineRequest, ) (*lpxMaterializing, lpxClassification, error) { - classification, requests, err := r.reconcileLPXKnownIntentFence(ctx, deployment, source, requests) + classification, requests, err := r.reconcileLPXKnownIntentFence(ctx, deployment, source, pcs, requests) if err != nil { return nil, nil, fmt.Errorf("failed to fence prior LPX intent: %w", err) } @@ -51,23 +54,25 @@ func (r *graphReconciler) reconcileLPXSafetyPreflight( return nil, nil, nil } - return r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source) + return r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source, pcs) } // reconcileSelectedLPXSafetyPreflight resolves and validates one selected LPX -// workload after its caller establishes the required download order. +// workload after its caller establishes the required download order. pcs is +// nil when its stable name was not found. func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, ) (*lpxMaterializing, lpxClassification, error) { // Resolve one immutable workload projection for selection and rendering. - selectedLPX, rejected, prepareErr := r.prepareLPXMaterializing(ctx, deployment, source) + selectedLPX, classification, prepareErr := r.prepareLPXMaterializing(ctx, deployment, source, pcs) if prepareErr != nil { return nil, nil, fmt.Errorf("failed to resolve the LPX workload: %w", prepareErr) } - if rejected != nil { - return nil, rejected, nil + if classification != nil { + return nil, classification, nil } // Grove accepts the desired spec before individual engine requests are reconciled. diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 71de28ddccec..b0093d3bd00e 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -30,7 +30,6 @@ func renderPodCliqueSet( source *v1beta1.DynamoGraphDeployment, operatorConfig *configv1alpha1.OperatorConfiguration, runtimeConfig *controller_common.RuntimeConfig, - kubeClient client.Reader, secretsRetriever dynamo.SecretsRetriever, workload *dynamolpx.SelectedWorkload, plan *dynamolpx.MaterializationPlan, @@ -38,7 +37,7 @@ func renderPodCliqueSet( ) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { // Apply shared Dynamo defaults before materializing the independently owned LPX roles. pcs, input, err := dynamo.RenderLPXBasePodCliqueSet(ctx, source, operatorConfig, runtimeConfig, - kubeClient, secretsRetriever, workload, plan) + secretsRetriever, workload, plan) if err != nil { return nil, nil, err } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index bb5f4c897d34..b6e4851bfbed 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -32,7 +32,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -64,9 +63,8 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{} - kube := newTestLPXClient(t) runtimeConfig := &controller_common.RuntimeConfig{} - pcs, _, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + pcs, _, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, selected, plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) serialized, err := json.Marshal(pcs) @@ -74,7 +72,7 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { t.Log("Accept exactly one MiB including final identity, discovery and scheduler metadata") source.Annotations["kai.scheduler/padding"] = strings.Repeat("x", lpx.MaxRenderedPodCliqueSetBytes-len(serialized)) - pcs, _, err = renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + pcs, _, err = renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, selected, plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) serialized, err = json.Marshal(pcs) @@ -86,78 +84,13 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { t.Log("Reject one additional final-metadata byte as a selected-render failure before publication") source.Annotations["kai.scheduler/padding"] += "x" - pcs, resources, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, kube, nil, + pcs, resources, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, selected, plan, newLPXRenderDeployment(t, source)) require.ErrorContains(t, err, "rendered LPX PodCliqueSet is 1048577 bytes; maximum is 1048576") require.Nil(t, pcs) require.Nil(t, resources) } -func TestLPXHybridPreservesKVTransferTopology(t *testing.T) { - t.Log("Create immutable native builds for this story's hybrid cases") - registry := newTestDataModelRegistry(t, t.TempDir()) - - for _, fixture := range []string{"node-local-v2-hybrid", "node-local-v3-hx-hybrid"} { - t.Run(fixture, func(t *testing.T) { - t.Log("Resolve a hybrid engine using a real ClusterTopologyBinding for KV transfer") - payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + fixture + ".input.yaml") - require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - source.Spec.Experimental = &v1beta1.DynamoGraphDeploymentExperimentalSpec{ - KvTransferPolicy: &v1beta1.KvTransferPolicy{ClusterTopologyName: "fabric", Domain: "rack"}, - } - binding := &grovev1alpha1.ClusterTopologyBinding{ObjectMeta: metav1.ObjectMeta{Name: "fabric"}, - Spec: grovev1alpha1.ClusterTopologyBindingSpec{Levels: []grovev1alpha1.TopologyLevel{ - {Domain: "zone", Key: "topology.kubernetes.io/zone"}, {Domain: "rack", Key: "nvidia.com/rack"}, - }}, - } - scheme := runtime.NewScheme() - require.NoError(t, corev1.AddToScheme(scheme)) - require.NoError(t, grovev1alpha1.AddToScheme(scheme)) - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(binding).Build() - child := newLPXRenderDeployment(t, source) - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) - require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) - require.NoError(t, err) - config := &configv1alpha1.OperatorConfiguration{} - pcs, _, err := renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) - require.NoError(t, err) - - t.Log("The Cyborg role receives every topology domain, not an empty mounted directory") - items := map[string]string{} - for _, clique := range pcs.Spec.Template.Cliques { - if clique.Name != plan.CyborgTemplate { - continue - } - require.Equal(t, binding.Name, clique.Annotations[commonconsts.KubeAnnotationTopologyClusterTopologyName]) - for _, volume := range clique.Spec.PodSpec.Volumes { - if volume.Name == "topology-labels" { - require.NotNil(t, volume.DownwardAPI) - for _, item := range volume.DownwardAPI.Items { - items[item.Path] = item.FieldRef.FieldPath - } - } - } - } - require.Equal(t, map[string]string{ - "zone": "metadata.labels['" + commonconsts.DynamoTopologyLabelKey("zone") + "']", - "rack": "metadata.labels['" + commonconsts.DynamoTopologyLabelKey("rack") + "']", - }, items) - - t.Log("Binding changes and deletion must invalidate the render instead of silently losing routing metadata") - binding.Spec.Levels = binding.Spec.Levels[:1] - require.NoError(t, kube.Update(t.Context(), binding)) - _, _, err = renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) - require.ErrorContains(t, err, `domain "rack" does not exist`) - require.NoError(t, kube.Delete(t.Context(), binding)) - _, _, err = renderPodCliqueSet(t.Context(), source, config, &controller_common.RuntimeConfig{}, kube, nil, selected, plan, child) - require.ErrorContains(t, err, "was not found") - }) - } -} - func TestLPXRenderingPreservesInputs(t *testing.T) { const restartToken = "2026-09-08T00:00:00Z" t.Log("Create immutable native builds while keeping each rendering's mutable inputs separate") @@ -274,10 +207,10 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { t.Log("Render twice and require identical resources with no input or shared-metadata mutation") first, firstResources, err := renderPodCliqueSet(t.Context(), source, config, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, child) + &controller_common.RuntimeConfig{}, nil, selected, plan, child) require.NoError(t, err) second, secondResources, err := renderPodCliqueSet(t.Context(), source, config, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, child) + &controller_common.RuntimeConfig{}, nil, selected, plan, child) require.NoError(t, err) require.Equal(t, first, second) require.Equal(t, firstResources, secondResources) @@ -330,7 +263,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, selected.Digest(), equivalentSelected.Digest()) require.Equal(t, plan, equivalentPlan) equivalentPCS, equivalentResources, err := renderPodCliqueSet(t.Context(), equivalent, config, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, equivalentSelected, equivalentPlan, child) + &controller_common.RuntimeConfig{}, nil, equivalentSelected, equivalentPlan, child) require.NoError(t, err) require.Equal(t, first, equivalentPCS) require.Equal(t, firstResources, equivalentResources) @@ -354,7 +287,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { scaledPlan, err := workload.PlanNodeLocalMaterialization(plan.PodCliqueSetName) require.NoError(t, err) pcs, resources, err := renderPodCliqueSet(t.Context(), scaled, config, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, workload, scaledPlan, scaledChild) + &controller_common.RuntimeConfig{}, nil, workload, scaledPlan, scaledChild) require.NoError(t, err) require.Equal(t, firstResources, resources) require.Equal(t, first.Spec.Template.Cliques, pcs.Spec.Template.Cliques) @@ -480,7 +413,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { t.Log("Render one shared conductor without changing either component's Agent template") pcs, _, err := renderPodCliqueSet(t.Context(), source, &configv1alpha1.OperatorConfiguration{}, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, source)) + &controller_common.RuntimeConfig{}, nil, selected, plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) t.Log("Keep Nova separate from Quasar and preserve each role's metadata and storage") @@ -829,7 +762,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { &deployment, controllerConfig, &controller_common.RuntimeConfig{}, - nil, nil, + nil, selected, mustPlanSelectedLPX(t, &deployment, selected), child, @@ -974,7 +907,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { registryRoot := t.TempDir() testdataModelRegistry := newTestDataModelRegistry(t, registryRoot) - kubeClient := newTestLPXClient(t) + kubeClient := fake.NewClientBuilder().Build() runtimeConfig := &controller_common.RuntimeConfig{} tests := []string{ @@ -1009,7 +942,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { t.Log("Render all LPX roles with the separate LPX PCS identity") got, extraResources, err := renderPodCliqueSet( t.Context(), &dynamoDeployment, controllerConfig, runtimeConfig, - kubeClient, nil, selected, + nil, selected, plan, newLPXRenderDeployment(t, &dynamoDeployment), ) @@ -1170,7 +1103,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test } pcs, extraResources, err := renderPodCliqueSet( t.Context(), dgd, controllerConfig, &controller_common.RuntimeConfig{}, - newTestLPXClient(t), nil, selected, plan, newLPXRenderDeployment(t, dgd), + nil, selected, plan, newLPXRenderDeployment(t, dgd), ) require.NoError(t, err) @@ -1335,7 +1268,6 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing.T) { t.Log("Share the immutable build registry across entrypoint cases") registry := newTestDataModelRegistry(t, t.TempDir()) - kubeClient := newTestLPXClient(t) modes := []struct { name string file string @@ -1465,7 +1397,6 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. dgd, &configv1alpha1.OperatorConfiguration{}, &controller_common.RuntimeConfig{}, - kubeClient, nil, selected, mustPlanSelectedLPX(t, dgd, selected), @@ -1520,11 +1451,6 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. } } -func newTestLPXClient(t *testing.T) client.Client { - t.Helper() - return fake.NewClientBuilder().Build() -} - func envValueSource(envs []corev1.EnvVar, name string) *corev1.EnvVarSource { for index := range envs { if envs[index].Name == name { @@ -1571,7 +1497,7 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { plan := mustPlanSelectedLPX(t, source, selected) pcs, _, err := renderPodCliqueSet(t.Context(), source, &configv1alpha1.OperatorConfiguration{}, - &controller_common.RuntimeConfig{}, newTestLPXClient(t), nil, selected, + &controller_common.RuntimeConfig{}, nil, selected, plan, newLPXRenderDeployment(t, source)) require.NoError(t, err) cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 21817ca130d6..554a71edadb2 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -189,6 +189,14 @@ func TestLPXSourceWatchIgnoresOrdinaryChurn(t *testing.T) { changed.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "new-runtime" require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) changed = source.DeepCopy() + changed.Generation++ + changed.Spec.Components[0].TopologyConstraint = &nvidiacomv1beta1.TopologyConstraint{PackDomain: "rack"} + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed = source.DeepCopy() + changed.Generation++ + changed.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} + require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) + changed = source.DeepCopy() changed.Spec.Restart = &nvidiacomv1beta1.Restart{ID: "restart"} require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) changed.Status.Restart = &nvidiacomv1beta1.RestartStatus{ObservedID: "restart", Phase: nvidiacomv1beta1.RestartPhaseRestarting, InProgress: []string{"lpx"}} diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index f824755479e6..644ec8e9da96 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" ) // RenderLPXBasePodCliqueSet applies shared Dynamo defaults to the Grove envelope @@ -35,7 +34,6 @@ func RenderLPXBasePodCliqueSet( dynamoDeployment *v1beta1.DynamoGraphDeployment, operatorConfig *configv1alpha1.OperatorConfiguration, runtimeConfig *controller_common.RuntimeConfig, - kubeClient client.Reader, secretsRetriever SecretsRetriever, selectedWorkload *dynamolpx.SelectedWorkload, plan *dynamolpx.MaterializationPlan, @@ -59,26 +57,16 @@ func RenderLPXBasePodCliqueSet( pcs.Labels[commonconsts.KubeLabelKaiSchedulerQueue] = queue } - // Cyborg uses ordinary worker KV-transfer defaults, including the binding's domains. - var topologyDomains []v1beta1.TopologyDomain - if dynamoDeployment.Spec.Experimental != nil { - topologyDomains, err = resolveGroveClusterTopologyDomains(ctx, kubeClient, dynamoDeployment.Spec.Experimental.KvTransferPolicy) - if err != nil { - return nil, nil, err - } - } - // Preserve the complete source graph as the independent role-default context. component := dynamolpx.ServingComponent(dynamoDeployment) input, gpuCliques, err := renderLPXComponents(cliqueParams{ component: component, componentName: component.ComponentName, dynamoDeployment: dynamoDeployment, operatorConfig: operatorConfig, runtimeConfig: runtimeConfig, - secretsRetriever: secretsRetriever, - discoveryBackend: controller_common.GetDiscoveryBackend(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), - discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), - validatedQueueName: queue, - groveClusterTopologyDomains: topologyDomains, + secretsRetriever: secretsRetriever, + discoveryBackend: controller_common.GetDiscoveryBackend(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), + discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), + validatedQueueName: queue, }, selectedWorkload, plan) if err != nil { return nil, nil, err @@ -98,7 +86,6 @@ const ( LPXDeploymentUIDAnnotation = dynamolpx.DeploymentUIDAnnotation LPXRestartAnnotation = "lpx.nvidia.com/restart-id" LPXServingLabel = "lpx.nvidia.com/serving" - lpxGPUExecutionRole = "gpu" ) // LPXRestartToken advances only from the DGD's persisted restart selection. @@ -157,9 +144,8 @@ type lpxInputRevisionPayload struct { Environment []corev1.EnvVar `json:"Env"` PriorityClass string BackendFramework string - RestartToken string `json:"Restart"` - KVTransferPolicy *v1beta1.KvTransferPolicy TopologyConstraint *v1beta1.SpecTopologyConstraint + RestartToken string `json:"Restart"` ProviderOverride *v1beta1.ProviderOverride SchedulingLabels map[string]string `json:",omitempty"` EPPEnabled bool `json:",omitempty"` @@ -171,8 +157,9 @@ type lpxInputRevisionPayload struct { // LPXInputRevision hashes all LPX components and their shared render inputs. // Source identity is checked separately by ValidateLPXSource. Ordinary component payloads, // DGD bookkeeping and raw restart requests are excluded: restart is the effective -// token selected by persisted DGD restart state. Operator configuration and external -// dependencies are not source revisions; the LPX controller observes them separately. +// token selected by persisted DGD restart state. Operator configuration is not a +// source revision; rejected topology inputs remain included because they change +// preflight results. func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (string, error) { components := dynamolpx.Components(dgd) if len(components) == 0 { @@ -199,10 +186,6 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin annotations[key] = value } } - var policy *v1beta1.KvTransferPolicy - if dgd.Spec.Experimental != nil { - policy = dgd.Spec.Experimental.KvTransferPolicy - } input := lpxInputRevisionPayload{ Components: canonical, Scheduling: dgd.Spec.Scheduling, @@ -212,9 +195,8 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin Environment: dgd.Spec.Env, PriorityClass: dgd.Spec.PriorityClassName, BackendFramework: dgd.Spec.BackendFramework, - RestartToken: restart, - KVTransferPolicy: policy, TopologyConstraint: dgd.Spec.TopologyConstraint, + RestartToken: restart, ProviderOverride: dgd.Spec.ProviderOverride, SchedulingLabels: lpxSchedulingMetadata(dgd.Labels), EPPEnabled: dgd.HasEPPComponent(), @@ -301,12 +283,11 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) lpuDefaults := &podTemplateRuntimeDefaults{ComponentDefaults: &BaseComponentDefaults{}} lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, lpuDefaults, nil) + p.discoveryContext, lpuDefaults) if err != nil { return nil, nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) } lpuTemplate.Labels[dynamolpx.StageLabel] = component.ComponentName - lpuTemplate.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" input.Stages[component.ComponentName] = *lpuTemplate conductor := component.ComponentRole(v1beta1.ComponentRoleLPXConductor) if component != p.component { @@ -317,12 +298,11 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p if workload.Pipeline() != dynamolpx.PipelineLPX { role := lpxRoleComponent(component, conductor.PodTemplate, p.dynamoDeployment, p.discoveryBackend) input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, lpuDefaults, nil) + p.discoveryContext, lpuDefaults) if err != nil { return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) } input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName - input.Conductor.Labels[dynamolpx.ExecutionRoleLabel] = "lpu" continue } @@ -350,7 +330,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p gpu.componentName = component.ComponentName gpu.r = ServiceRole{Name: plan.CyborgTemplate, Role: RoleMain, Replicas: *role.Replicas} gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, - p.discoveryContext, defaults, p.groveClusterTopologyDomains) + p.discoveryContext, defaults) if err != nil { return nil, nil, fmt.Errorf("rendering %s.conductor: failed to generate podSpec for role %s: %w", component.ComponentName, gpu.r.Name, err) } @@ -362,7 +342,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p clique.Spec.MinAvailable = ptr.To(clique.Spec.Replicas) clique.Labels[commonconsts.KubeLabelDynamoComponentType] = string(v1beta1.ComponentTypeLPX) clique.Labels[dynamolpx.StageLabel] = component.ComponentName - clique.Labels[dynamolpx.ExecutionRoleLabel] = lpxGPUExecutionRole gpuCliques = append(gpuCliques, clique) } return input, gpuCliques, nil @@ -433,7 +412,7 @@ func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentCont } // renderSelectedLPXRole consumes a private component copy; other inputs are read-only. -// alphaComponent may be nil; topologyDomains may be nil when no topology is consumed. +// alphaComponent may be nil. func renderSelectedLPXRole( component *v1beta1.DynamoComponentDeploymentSharedSpec, dgd *v1beta1.DynamoGraphDeployment, @@ -442,12 +421,11 @@ func renderSelectedLPXRole( secretsRetriever SecretsRetriever, discoveryContext DiscoveryContext, defaults ComponentDefaults, - topologyDomains []v1beta1.TopologyDomain, ) (*corev1.PodTemplateSpec, error) { componentName := component.ComponentName // Capture authored precedence before PodSpec defaults fill the role's metadata. metadata := generatePodMetadata(component, dgd, alphaComponent, componentName, discoveryContext) - applyDGDTemplateDefaults(component, dgd, topologyDomains) + applyDGDTemplateDefaults(component, dgd, nil) basePodSpec, err := generateBasePodSpecWithDefaults( component, BackendFrameworkNoop, diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 2be5765852bd..8a5eab3cba47 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -97,7 +97,7 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { t.Log("Render the role before runtime-specific lowering") template, err := renderSelectedLPXRole(component, source, nil, &configv1alpha1.OperatorConfiguration{}, &mockSecretsRetriever{}, DiscoveryContext{}, - &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()}, nil) + &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()}) require.NoError(t, err) t.Log("Retain authored security context and default only an absent one") @@ -235,6 +235,9 @@ func TestLPXInputRevision(t *testing.T) { {"component/compilation-cache", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).CompilationCache = &v1beta1.CompilationCacheConfig{PVCName: "cache"} }}, + {"component/topology", true, func(d *v1beta1.DynamoGraphDeployment) { + lpx.ServingComponent(d).TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} + }}, {"component/build", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).LPX.BuildID = "next-build" }}, {"component/settings", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":false}`)} @@ -271,9 +274,6 @@ func TestLPXInputRevision(t *testing.T) { {"shared/environment", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Env = []corev1.EnvVar{{Name: "SHARED", Value: "new"}} }}, {"shared/priority", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.PriorityClassName = "inference" }}, {"shared/backend", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.BackendFramework = "vllm" }}, - {"shared/kv-transfer", true, func(d *v1beta1.DynamoGraphDeployment) { - d.Spec.Experimental = &v1beta1.DynamoGraphDeploymentExperimentalSpec{KvTransferPolicy: &v1beta1.KvTransferPolicy{LabelKey: "topology.kubernetes.io/zone", Domain: "zone"}} - }}, {"shared/topology", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "cluster", PackDomain: "rack"} }}, diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 3d2c6efc85a4..47e862d68b25 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -9,13 +9,10 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dra" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,41 +21,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -// ResolveLPXGPUShape counts one complete engine replica from a non-nil rendered -// LPX PCS. GPU-role main containers define engine width; all role Pods define -// replica cost. Outer scaling-group replicas never multiply either value. -func ResolveLPXGPUShape(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet) (GPUShape, error) { - enginePods := make([]dra.PodSpecMultiplicity, 0, len(pcs.Spec.Template.Cliques)) - replicaPods := make([]dra.PodSpecMultiplicity, 0, len(pcs.Spec.Template.Cliques)) - for _, clique := range pcs.Spec.Template.Cliques { - pod := &clique.Spec.PodSpec - replicaPods = append(replicaPods, dra.PodSpecMultiplicity{PodSpec: pod, Count: clique.Spec.Replicas}) - if clique.Labels[lpx.ExecutionRoleLabel] != lpxGPUExecutionRole { - continue - } - engine := &corev1.PodSpec{ResourceClaims: pod.ResourceClaims} - for i := range pod.Containers { - if pod.Containers[i].Name == consts.MainContainerName { - engine.Containers = pod.Containers[i : i+1] - break - } - } - if len(engine.Containers) == 0 { - return GPUShape{}, fmt.Errorf("LPX GPU role %q has no main container", clique.Name) - } - enginePods = append(enginePods, dra.PodSpecMultiplicity{PodSpec: engine, Count: clique.Spec.Replicas}) - } - engineGPUs, err := dra.ResolvePodSetGPUCount(ctx, reader, pcs.Namespace, enginePods) - if err != nil { - return GPUShape{}, fmt.Errorf("resolve LPX engine GPUs: %w", err) - } - replicaGPUs, err := dra.ResolvePodSetGPUCount(ctx, reader, pcs.Namespace, replicaPods) - if err != nil { - return GPUShape{}, fmt.Errorf("resolve LPX replica GPUs: %w", err) - } - return GPUShape{GPUsPerEngine: int64(engineGPUs), GPUsPerReplica: int64(replicaGPUs)}, nil -} - // EvaluateLPXGroveReadiness observes one complete engine, including every role // in every replica. Ordinary DGD components are deliberately not evaluated here. func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet) (GroveReadiness, error) { diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go index 550eac707933..d15e2df88b2f 100644 --- a/deploy/operator/internal/dynamo/grove_lpx_test.go +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -8,16 +8,8 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T) { @@ -48,80 +40,3 @@ func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) require.Equal(t, source.GetDynamoNamespaceForComponent(&serving), readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) } - -func TestResolveLPXGPUShape(t *testing.T) { - for _, test := range []struct { - name string - want GPUShape - }{ - {"scalar-init-peak", GPUShape{GPUsPerEngine: 8, GPUsPerReplica: 21}}, - {"claim-template", GPUShape{GPUsPerEngine: 4, GPUsPerReplica: 7}}, - {"shared-claim", GPUShape{GPUsPerEngine: 2, GPUsPerReplica: 5}}, - {"lpu-only", GPUShape{}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Count two GPU workers and three auxiliary Pods in each complete engine replica") - nativeSidecar := gpuContainer("native-sidecar", "1") - nativeSidecar.RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) - gpu := &grovev1alpha1.PodCliqueTemplateSpec{ - Name: "cyborg", Labels: map[string]string{lpx.ExecutionRoleLabel: lpxGPUExecutionRole}, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: 2, PodSpec: corev1.PodSpec{ - Containers: []corev1.Container{gpuContainer(consts.MainContainerName, "4"), gpuContainer("sidecar", "1")}, - InitContainers: []corev1.Container{nativeSidecar, gpuContainer("prepare", "8")}, - }}, - } - auxiliary := &grovev1alpha1.PodCliqueTemplateSpec{ - Name: "agents", Labels: map[string]string{lpx.ExecutionRoleLabel: "lpu"}, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: 3, PodSpec: corev1.PodSpec{ - Containers: []corev1.Container{gpuContainer("agent", "0"), gpuContainer("metrics", "1")}, - }}, - } - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Namespace: "test"}, - Spec: grovev1alpha1.PodCliqueSetSpec{Template: grovev1alpha1.PodCliqueSetTemplateSpec{ - Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{gpu, auxiliary}, - PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{{Replicas: ptr.To(int32(3))}}, - }}, - } - claim := &resourcev1.ResourceClaim{ObjectMeta: metav1.ObjectMeta{Namespace: pcs.Namespace, Name: "gpu"}, - Spec: resourcev1.ResourceClaimSpec{Devices: resourcev1.DeviceClaim{Requests: []resourcev1.DeviceRequest{{ - Name: "gpu", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: "gpu.nvidia.com", Count: 2, AllocationMode: resourcev1.DeviceAllocationModeExactCount}, - }}}}, - } - claimTemplate := &resourcev1.ResourceClaimTemplate{ObjectMeta: claim.ObjectMeta, - Spec: resourcev1.ResourceClaimTemplateSpec{Spec: claim.Spec}, - } - scheme := runtime.NewScheme() - require.NoError(t, resourcev1.AddToScheme(scheme)) - reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(claim, claimTemplate, - &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: "gpu.nvidia.com"}}).Build() - switch test.name { - case "claim-template", "shared-claim": - pod := &gpu.Spec.PodSpec - pod.InitContainers = nil - for i := range pod.Containers { - pod.Containers[i].Resources = corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "gpu"}}} - } - pod.ResourceClaims = []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To(claimTemplate.Name)}} - if test.name == "shared-claim" { - pod.ResourceClaims[0].ResourceClaimTemplateName = nil - pod.ResourceClaims[0].ResourceClaimName = ptr.To(claim.Name) - } - case "lpu-only": - pcs.Spec.Template.Cliques = []*grovev1alpha1.PodCliqueTemplateSpec{auxiliary} - auxiliary.Spec.PodSpec.Containers = auxiliary.Spec.PodSpec.Containers[:1] - } - before := pcs.DeepCopy() - shape, err := ResolveLPXGPUShape(t.Context(), reader, pcs) - require.NoError(t, err) - require.Equal(t, test.want, shape) - require.Equal(t, before, pcs, "GPU accounting must not mutate shared Pod specs") - - if test.name == "claim-template" { - t.Log("Missing dependencies must not produce a misleading zero-GPU result") - require.NoError(t, reader.Delete(t.Context(), claimTemplate)) - _, err := ResolveLPXGPUShape(t.Context(), reader, pcs) - require.ErrorContains(t, err, "resolve LPX engine GPUs") - } - }) - } -} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 2c683cdde0d3..124aa2cd5dc2 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -49,6 +49,8 @@ func configureHybridCyborg( return err } + cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" + delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) cyborg.Annotations = roleAnnotations( cyborg.Annotations, lpxv1alpha1.PodRoleCyborgWorker, diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 7045d0b0a5fd..40fda5572a20 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -7,6 +7,7 @@ package lpx import ( "encoding/json" + "math" "strings" "testing" @@ -31,7 +32,7 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { projection.configuredBuild.IOFanoutFactor = 1 workload := &SelectedWorkload{ modelProjections: []*ModelProjection{projection}, - scalingGroupReplicas: int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes), + scalingGroupReplicas: math.MaxInt32, } pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) plan, err := workload.PlanNodeLocalMaterialization(pcsName) @@ -44,8 +45,8 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { wantError bool }{ {name: "one GPU per engine", replicas: 1}, - {name: "hostname at DNS limit", replicas: 100_000_000}, - {name: "hostname over DNS limit", replicas: 100_000_001, wantError: true}, + {name: "hostname at DNS limit", replicas: 100}, + {name: "hostname over DNS limit", replicas: 101, wantError: true}, } { t.Run(test.name, func(t *testing.T) { t.Log("Render the GPU width beneath the same readable PCS name") diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 151f17c4bbd7..808db9ec90bd 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -6,7 +6,6 @@ package lpx import ( - "crypto/sha256" "fmt" "slices" "strings" @@ -26,15 +25,6 @@ const ( const MaxPodCliqueSetNameLength = commonconsts.MaxCombinedGroveResourceNameLength - len(lpxScalingGroupTemplateName) - len(conductorTemplateName) -// LPXAttemptRequestStatus stores a name (lpxRequestName uses at most 60 bytes), -// sha256 digest, and server-issued UUID. Reserve a full DNS label plus JSON syntax -// and an array separator; all generated values are ASCII. -const maximumSchedulingAttemptRequestBytes = len(`{"name":"","attemptDigest":"","uid":""},`) + - validation.DNS1123LabelMaxLength + len("sha256:") + sha256.Size*2 + 36 - -// Leave half the object allowance for the enclosing status, spec, and metadata. -const schedulingAttemptRequestBytesBudget = MaxRenderedPodCliqueSetBytes / 2 - // ExpectedAgent binds one projected model to the exact Grove // PodClique identity that must materialize it. The controller uses this plan // to observe objects; it never reconstructs a row from a generated Pod name. @@ -111,11 +101,10 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi return plan, plan.ValidateReplicaCount() } -// ValidateReplicaCount checks conductor/Agent hostnames and bounds request/status allocation. +// ValidateReplicaCount checks conductor and Agent hostnames. func (p *MaterializationPlan) ValidateReplicaCount() error { - // Charge every model/replica for its complete persisted request identity. - if p.Replicas < 0 || int64(p.Replicas)*int64(len(p.Agents)) > int64(schedulingAttemptRequestBytesBudget/maximumSchedulingAttemptRequestBytes) { - return fmt.Errorf("LPX replica count exceeds the scheduling status size budget") + if p.Replicas < 0 { + return fmt.Errorf("LPX replica count must not be negative") } if p.ConductorTemplate != "" { if err := p.validatePodHostname("conductor", p.ConductorTemplate, 0); err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index dc2d028f3c57..acc913a7b1bb 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -6,67 +6,17 @@ package lpx import ( - "encoding/json" - "fmt" + "math" "strings" "testing" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" ) -func TestMaterializationBoundsCompleteSchedulingAttempt(t *testing.T) { - t.Parallel() - - t.Log("Account for every populated API field and the array separator") - row := nvidiacomv1beta1.LPXAttemptRequestStatus{ - Name: strings.Repeat("a", validation.DNS1123LabelMaxLength), - AttemptDigest: "sha256:" + strings.Repeat("f", 64), - UID: types.UID("01234567-89ab-cdef-0123-456789abcdef"), - } - encoded, err := json.Marshal(row) - require.NoError(t, err) - require.Equal(t, maximumSchedulingAttemptRequestBytes, len(encoded)+1) - - const maximumRequests = int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes) - - t.Log("Accept the exact row limit and reject larger single- and multi-model allocations") - for _, test := range []struct { - name string - replicas int32 - models int - wantErr bool - }{ - {name: "zero", models: 1}, - {name: "nine replicas", replicas: 9, models: 1}, - {name: "twelve replicas", replicas: 12, models: 1}, - {name: "limit", replicas: maximumRequests, models: 1}, - {name: "one over", replicas: maximumRequests + 1, models: 1, wantErr: true}, - {name: "two models at limit", replicas: maximumRequests / 2, models: 2}, - {name: "two models over limit", replicas: maximumRequests/2 + 1, models: 2, wantErr: true}, - {name: "16384 replicas", replicas: 16384, models: 1, wantErr: true}, - {name: "negative", replicas: -1, models: 1, wantErr: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Validate the complete identity storage before materializing replicas") - plan := &MaterializationPlan{LPXScalingGroup: "group", Replicas: test.replicas, Agents: make([]ExpectedAgent, test.models)} - for i := range plan.Agents { - plan.Agents[i] = ExpectedAgent{TemplateName: fmt.Sprintf("agent-%d", i), Replicas: 1} - } - if test.wantErr { - require.ErrorContains(t, plan.ValidateReplicaCount(), "scheduling status size budget") - } else { - require.NoError(t, plan.ValidateReplicaCount()) - } - }) - } -} - func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { t.Parallel() @@ -101,7 +51,7 @@ func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { } workload := &SelectedWorkload{ modelProjections: projections, - scalingGroupReplicas: int32(schedulingAttemptRequestBytesBudget / maximumSchedulingAttemptRequestBytes / test.models), + scalingGroupReplicas: math.MaxInt32, } pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) plan, err := workload.PlanNodeLocalMaterialization(pcsName) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index ff2a5494b9f6..5fe840b11faa 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -24,12 +24,8 @@ import ( // admission metadata and transport overhead. const MaxRenderedPodCliqueSetBytes = 1 << 20 -const ( - // StageLabel records the authored LPX component association. - StageLabel = "lpx.nvidia.com/stage" - // ExecutionRoleLabel distinguishes internal LPU and GPU Pod templates. - ExecutionRoleLabel = "lpx.nvidia.com/execution-role" -) +// StageLabel records the authored LPX component association. +const StageLabel = "lpx.nvidia.com/stage" // RenderInput contains the fresh stage templates and runtime settings consumed by rendering. type RenderInput struct { @@ -240,11 +236,6 @@ func RenderSelectedNodeLocal( MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), }} - // Select Grove's LPX backend for the complete engine, including its KAI fallback. - for _, clique := range pcs.Spec.Template.Cliques { - clique.Spec.PodSpec.SchedulerName = SchedulerName - } - explicit := grovev1alpha1.CliqueStartupTypeExplicit pcs.Spec.Template.StartupType = &explicit return extraResources, nil diff --git a/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go index 9ba60eda68b5..315e2ae9fb27 100644 --- a/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go +++ b/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1/types.go @@ -679,6 +679,7 @@ type LPUPipelineRequestStatus struct { LastPlanRevision int64 `json:"lastPlanRevision"` ObservedGeneration *int64 `json:"observedGeneration,omitempty"` Phase RequestPhase `json:"phase"` + SchedulingStartedAt *metav1.Time `json:"schedulingStartedAt,omitempty"` } func (in *LPUPipelineRequestStatus) DeepCopyInto(out *LPUPipelineRequestStatus) { @@ -709,6 +710,9 @@ func (in *LPUPipelineRequestStatus) DeepCopyInto(out *LPUPipelineRequestStatus) value := *in.ObservedGeneration out.ObservedGeneration = &value } + if in.SchedulingStartedAt != nil { + out.SchedulingStartedAt = in.SchedulingStartedAt.DeepCopy() + } } func (in *LPUPipelineRequestStatus) DeepCopy() *LPUPipelineRequestStatus { diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 22b97afb70d1..49ad235ebc2d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -219,7 +219,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -441,7 +440,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/role: cyborg-worker labels: - lpx.nvidia.com/execution-role: gpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -596,7 +594,7 @@ spec: - mountPath: /dev/shm name: shared-memory restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 6ccab754927b..ad30f181f42d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -83,7 +83,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -256,7 +255,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/role: cyborg-worker labels: - lpx.nvidia.com/execution-role: gpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -391,7 +389,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index f438eabcfb73..a6522a179d1e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -85,7 +85,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -227,7 +226,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/role: conductor labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -371,7 +369,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 7756a2f58da4..c208953abe4c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -154,7 +154,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx @@ -301,7 +300,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx @@ -448,7 +446,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -592,7 +589,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 scheduling.lpu.nvidia.com/role: conductor labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -735,7 +731,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index a9210393749d..b98e975968d0 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -78,7 +78,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxStrictHybrid labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -250,7 +249,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 scheduling.lpu.nvidia.com/role: cyborg-worker labels: - lpx.nvidia.com/execution-role: gpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -367,7 +365,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index d008b1d641c8..3c49808a191f 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -77,7 +77,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -252,7 +251,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/role: conductor labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -435,7 +433,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index e0083f475ebf..1182ba239efb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -133,7 +133,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx @@ -313,7 +312,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx @@ -493,7 +491,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -670,7 +667,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 scheduling.lpu.nvidia.com/role: conductor labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -853,7 +849,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 5c0c6649abe1..ba5300e16305 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -161,7 +161,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx @@ -307,7 +306,6 @@ spec: scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/role: conductor labels: - lpx.nvidia.com/execution-role: lpu lpx.nvidia.com/serving: "true" lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu @@ -456,7 +454,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: lpx-scheduler + schedulerName: default-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 484f3b398a74..a7e113621b0b 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -327,9 +327,11 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( allErrs = append(allErrs, field.Required(componentsPath, "must have at least one component")) } components := componentsByName(spec.Components) + hasLPXComponent := false for i := range spec.Components { component := &spec.Components[i] componentPath := componentsPath.Index(i) + hasLPXComponent = hasLPXComponent || component.IsLPX() // Externally managed components validate their generated names in their own controller. if opts.grovePathway && !component.ManagedByExternalController() { @@ -484,6 +486,13 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( opts.grovePathway, opts.grovePathwayRequirement, )...) + // A deployment-wide KV policy cannot be applied consistently once LPX owns a serving component. + if hasLPXComponent && spec.Experimental.KvTransferPolicy != nil { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("experimental", "kvTransferPolicy"), + "is not supported when an LPX component is selected", + )) + } } return allErrs diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index c7ef0c17ad33..a704154ff84b 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -123,6 +123,18 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: betaLPXDGDForAdmission(nil), wantWebhookErrs: []string{`spec.components: Forbidden: requires the Grove pathway, but workload provider "component" is selected`}, }, + { + name: "LPX rejects the deployment-wide KV transfer policy", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Experimental = &nvidiacomv1beta1.DynamoGraphDeploymentExperimentalSpec{ + KvTransferPolicy: &nvidiacomv1beta1.KvTransferPolicy{ + LabelKey: "topology.kubernetes.io/zone", + Domain: "zone", + }, + } + }), + wantWebhookErrs: []string{`spec.experimental.kvTransferPolicy: Forbidden: is not supported when an LPX component is selected`}, + }, { name: "LPX topology and checkpoint support is deferred to the child controller", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 0b96df09883e..e5e606341a96 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -961,45 +961,6 @@ _Appears in:_ | `preferredWeight` _float_ | PreferredWeight is required and used only when enforcement is
"preferred". Higher values create a stronger same-domain routing
preference, but do not guarantee same-domain selection. The value is not
a probability; worker selection still depends on load and other routing
inputs. A value of 0 disables the topology preference; 1 is the strongest
supported preference. | | Maximum: 1
Minimum: 0
Optional: \{\}
| -#### LPXAttemptRequestStatus - - - -LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - - - -_Appears in:_ -- [LPXAttemptStatus](#lpxattemptstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `name` _string_ | | | | -| `attemptDigest` _string_ | | | | -| `uid` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | - - -#### LPXAttemptStatus - - - -LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. - - - -_Appears in:_ -- [PlacementStatus](#placementstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `observedGeneration` _integer_ | | | | -| `podCliqueSetUID` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | -| `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | | - - #### ModelReference @@ -1112,7 +1073,6 @@ _Appears in:_ | --- | --- | --- | --- | | `score` _float_ | Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0]
where higher is better and 1.0 is the best possible placement. | | Maximum: 1
Minimum: 0
Optional: \{\}
| | `state` _[PlacementScoreState](#placementscorestate)_ | State indicates placement score reporting state. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| -| `lpxAttempt` _[LPXAttemptStatus](#lpxattemptstatus)_ | LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. | | Optional: \{\}
| #### ProfilingConfigSpec @@ -2545,45 +2505,6 @@ _Appears in:_ | `preferredWeight` _float_ | preferredWeight is required and used only when enforcement is
"preferred". Higher values create a stronger same-domain routing
preference, but do not guarantee same-domain selection. The value is not
a probability; worker selection still depends on load and other routing
inputs. A value of 0 disables the topology preference; 1 is the strongest
supported preference. | | Maximum: 1
Minimum: 0
Optional: \{\}
| -#### LPXAttemptRequestStatus - - - -LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - - - -_Appears in:_ -- [LPXAttemptStatus](#lpxattemptstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `name` _string_ | | | | -| `attemptDigest` _string_ | | | | -| `uid` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | - - -#### LPXAttemptStatus - - - -LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. - - - -_Appears in:_ -- [PlacementStatus](#placementstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `observedGeneration` _integer_ | | | | -| `podCliqueSetUID` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | | | | -| `deadlineAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `exceededAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `disarmedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | | | | -| `requests` _[LPXAttemptRequestStatus](#lpxattemptrequeststatus) array_ | | | | - - #### LPXConfig @@ -2795,7 +2716,6 @@ _Appears in:_ | --- | --- | --- | --- | | `score` _float_ | score is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation
uses the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| | `state` _[PlacementScoreState](#placementscorestate)_ | state indicates placement score reporting state. See PlacementScoreState
for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| -| `lpxAttempt` _[LPXAttemptStatus](#lpxattemptstatus)_ | lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. | | Optional: \{\}
| #### ProfilingPhase @@ -3048,7 +2968,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `attemptDeadlineSeconds` _integer_ | attemptDeadlineSeconds limits one aggregate LPX scheduling attempt.
Omission means unlimited; the value is not a solver budget. | | Maximum: 9.223372036e+09
Minimum: 1
Optional: \{\}
| +| `attemptDeadlineSeconds` _integer_ | attemptDeadlineSeconds limits how long each LPR may remain pending.
Omission means unlimited; the value is not a solver budget. | | Maximum: 9.223372036e+09
Minimum: 1
Optional: \{\}
| #### SearchStrategy @@ -3400,7 +3320,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `enabled` _boolean_ | Enabled opts the operator into LPX request production. When true, startup
requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and
scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. | false | | +| `enabled` _boolean_ | Enabled opts the operator into LPX request production. When true, startup
requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API to
be installed separately. | false | | | `modelRegistryURL` _string_ | ModelRegistryURL configures the location of the LPU model registry used by LPX.
Supported values are an absolute local path (or file:// URL) and gs:// URLs. | | | diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 4296c3b5e292..77e7c397c378 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -74,8 +74,8 @@ spec: - Wall-clock lifetime of the aggregate LPX attempt, from the first request's API-server creation time through `Bound`, `NoFit`, or `Unsupported`. Valid values are `1` through `9,223,372,036`; omission means unlimited. - Dynamo durably records the absolute deadline. Resumed scheduling keeps it; late terminal status loses to expiry. Expiry reports `LPXSchedulingDeadlineExceeded` and retires the exact attempt through LPX cleanup. It is neither a solver budget nor retry authorization. + Wall-clock limit for each pending LPU pipeline request, measured from the scheduler-published start of its current scheduling cycle until it reports `Bound`, `NoFit`, or `Unsupported`. The deadline is not armed until the scheduler publishes `status.schedulingStartedAt`, and a later scheduling cycle resets that timestamp. Valid values are `1` through `9,223,372,036`; omission means unlimited. + Expiry reports `LPXSchedulingDeadlineExceeded` and deletes the expired request. When `replicas` is explicit, it also lowers the Grove replica count to the surviving prefix; externally managed capacity is left unchanged. A later LPX input revision authorizes another publication. This field is not a solver budget. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 7dd6af346dcd..1d3a68a37fa3 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,11 +10,11 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 3 Kubernetes API packages with 175 typed CRD and config sections. +Dynamo publishes 3 Kubernetes API packages with 171 typed CRD and config sections. -71 types -75 types +69 types +73 types 29 types @@ -1506,55 +1506,6 @@ PreferredWeight is required and used only when enforcement is "preferred". Highe - -LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - - -**Kind:** `type` -**Appears in:** [LPXAttemptStatus](#lpxattemptstatus) - - - - - - - - -See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). - - - - -LPXAttemptStatus mirrors the v1beta1 durable LPX attempt authority record. - - -**Kind:** `type` -**Appears in:** [PlacementStatus](#placementstatus) - - - - - -See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [LPXAttemptRequestStatus](#lpxattemptrequeststatus). - - - ModelReference identifies a model served by this component @@ -1664,12 +1615,6 @@ State indicates placement score reporting state. See [PlacementScoreState](#placementscorestate). **Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} - - -LPXAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. -See [LPXAttemptStatus](#lpxattemptstatus). -**Validation:** Optional: \{\} - @@ -3721,55 +3666,6 @@ preferredWeight is required and used only when enforcement is "preferred". Highe - -LPXAttemptRequestStatus identifies one exact request in the aggregate attempt. - - -**Kind:** `type` -**Appears in:** [LPXAttemptStatus](#v1beta1-lpxattemptstatus) - - - - - - - - -See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). - - - - -LPXAttemptStatus is Dynamo's durable authority record for one aggregate LPX attempt. - - -**Kind:** `type` -**Appears in:** [PlacementStatus](#v1beta1-placementstatus) - - - - - -See [UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). - - - -See [LPXAttemptRequestStatus](#v1beta1-lpxattemptrequeststatus). - - - LPXConfig identifies the component's compiled model and runtime settings. @@ -3982,12 +3878,6 @@ state indicates placement score reporting state. See PlacementScoreState for the See [PlacementScoreState](#v1beta1-placementscorestate). **Validation:** Enum: [Reported Partial Unsupported Unknown] Optional: \{\} - - -lpxAttempt is Dynamo's durable aggregate LPX scheduling-attempt authority record. -See [LPXAttemptStatus](#v1beta1-lpxattemptstatus). -**Validation:** Optional: \{\} - @@ -4253,7 +4143,7 @@ SchedulingSpec configures LPX scheduling attempts. **Appears in:** [DynamoGraphDeploymentSpec](#dynamographdeploymentspec), [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) -attemptDeadlineSeconds limits one aggregate LPX scheduling attempt. Omission means unlimited; the value is not a solver budget. +attemptDeadlineSeconds limits how long each LPR may remain pending. Omission means unlimited; the value is not a solver budget. **Validation:** Maximum: 9.223372036e+09 Minimum: 1 Optional: \{\} @@ -4626,7 +4516,7 @@ LPXConfiguration holds LPX scheduler integration and model-registry settings. **Appears in:** [OperatorConfiguration](#operatorconfiguration) -Enabled opts the operator into LPX request production. When true, startup requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs to be installed separately. +Enabled opts the operator into LPX request production. When true, startup requires the scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API to be installed separately. diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index efe9a8d7be1f..e9cff13ccfcb 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -48,8 +48,8 @@ # (PodReference, PodSnapshotSource/Spec/Status, # PodSnapshotContentSource/Spec/Status, PodSnapshotReference), which are # owned by github.com/ai-dynamo/snapshot. - "nvidia.com/v1alpha1": 71, - "nvidia.com/v1beta1": 75, + "nvidia.com/v1alpha1": 69, + "nvidia.com/v1beta1": 73, "operator.config.dynamo.nvidia.com/v1alpha1": 29, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( From f0e89cba2c430f50c0db3f847aa57f9ac77fe6d8 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:19:25 -0700 Subject: [PATCH 085/196] fix(operator): use Grove LPX scheduler backend (#402) Signed-off-by: Steven Davidovitz (cherry picked from commit e0be861b6b2ef1699c9cbdf88335a9ce51594644) Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/lifecycle_test.go | 2 +- deploy/operator/internal/controller/lpx/render_test.go | 7 +++++++ deploy/operator/internal/dynamo/lpx/cyborg_render.go | 2 -- deploy/operator/internal/dynamo/lpx/pod_roles.go | 2 +- deploy/operator/internal/dynamo/lpx/render.go | 6 +++++- deploy/operator/internal/dynamo/lpx/render_test.go | 1 + .../dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 2 +- .../lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml | 2 +- .../lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml | 2 +- .../testdata/from_dgd_yaml/node-local-v2-specdecode.yaml | 2 +- .../testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml | 2 +- .../testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 2 +- .../from_dgd_yaml/node-local-v3-hx-specdecode.yaml | 2 +- .../dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml | 2 +- 14 files changed, 23 insertions(+), 13 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 8c57edd76e1f..8089376e6074 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -163,7 +163,7 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { require.NotContains(t, group.Spec.CliqueNames, "") cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) require.NotContains(t, cyborg.Spec.StartsAfter, "") - require.Equal(t, corev1.DefaultSchedulerName, cyborg.Spec.PodSpec.SchedulerName) + require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) createLPXTestObjects(t, ctx, reconciler.Client, objects...) classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index b6e4851bfbed..9a7be9ff0959 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -119,6 +119,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { source.Labels = map[string]string{ "priorityClassName": "inference", "kai.scheduler/preemptibility": "NonPreemptible", "project": "test", "unrelated": "ignored", + commonconsts.KubeLabelKaiSchedulerQueue: "lpx-queue", } metav1.SetMetaDataAnnotation(&source.ObjectMeta, "kai.scheduler/topology", "source-topology") metav1.SetMetaDataAnnotation(&source.ObjectMeta, "unrelated", "ignored") @@ -139,6 +140,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, "selected-metadata", "from-component") metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, commonconsts.RestartAnnotation, "2026-09-07T00:00:00Z") metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, "shared-label", "from-component") + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, commonconsts.KubeLabelKaiSchedulerQueue, "lpx-queue") metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, commonconsts.KubeLabelDynamoNamespace, "authored-namespace") } } @@ -216,6 +218,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, firstResources, secondResources) require.Equal(t, "inference", first.Labels["priorityClassName"]) require.Equal(t, "NonPreemptible", first.Labels["kai.scheduler/preemptibility"]) + require.Equal(t, "lpx-queue", first.Labels[commonconsts.KubeLabelKaiSchedulerQueue]) require.Equal(t, "test", first.Labels["project"]) require.Equal(t, "explicit-pcs-topology", first.Annotations["kai.scheduler/topology"]) require.NotContains(t, first.Annotations, commonconsts.RestartAnnotation) @@ -302,6 +305,8 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { var conductor, gpuClique *grovev1alpha1.PodCliqueTemplateSpec serving := 0 for index, clique := range first.Spec.Template.Cliques { + require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName, "role %s must use the LPX backend", clique.Name) + require.Equal(t, "lpx-queue", clique.Labels[commonconsts.KubeLabelKaiSchedulerQueue]) if clique.Labels[dynamo.LPXServingLabel] == commonconsts.KubeLabelValueTrue { serving++ require.NotEqual(t, lpxv1alpha1.PodRoleAgent, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) @@ -948,6 +953,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { ) require.NoError(t, err) for _, clique := range got.Spec.Template.Cliques { + require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) component := dynamoDeployment.GetComponentByName(clique.Labels[commonconsts.KubeLabelDynamoComponent]) require.NotNil(t, component) require.True(t, component.IsLPX()) @@ -966,6 +972,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { require.NoError(t, err) require.NotEqual(t, normal.Name, got.Name) for _, clique := range normal.Spec.Template.Cliques { + require.NotEqual(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) require.NotEqual(t, selected.LPXComponentName(), clique.Labels[commonconsts.KubeLabelDynamoComponent]) } podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 124aa2cd5dc2..2c683cdde0d3 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -49,8 +49,6 @@ func configureHybridCyborg( return err } - cyborg.Spec.PodSpec.SchedulerName = "default-scheduler" - delete(cyborg.Labels, commonconsts.KubeLabelKaiSchedulerQueue) cyborg.Annotations = roleAnnotations( cyborg.Annotations, lpxv1alpha1.PodRoleCyborgWorker, diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index 84149cc3cf21..bec31880c379 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -15,7 +15,7 @@ import ( ) const ( - // SchedulerName is the Kubernetes scheduler used for LPX Agent pods. + // SchedulerName selects Grove's LPX backend for the complete PodCliqueSet. SchedulerName = "lpx-scheduler" v2LPUResourceName corev1.ResourceName = "lpu.nvidia.com/lpu" diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 5fe840b11faa..70b645d8f4f7 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -236,6 +236,11 @@ func RenderSelectedNodeLocal( MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), }} + // Keep every LPX role in one backend gang, including the KAI fallback roles. + for _, clique := range pcs.Spec.Template.Cliques { + clique.Spec.PodSpec.SchedulerName = SchedulerName + } + explicit := grovev1alpha1.CliqueStartupTypeExplicit pcs.Spec.Template.StartupType = &explicit return extraResources, nil @@ -250,7 +255,6 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo configureAgentScheduling(agentPodSpec, workload.BuildFamily()) // Placement is already resolved; shape only the actual conductor's LPX-owned fields. if conductorPodSpec != nil { - conductorPodSpec.SchedulerName = corev1.DefaultSchedulerName stripLPUResources(conductorPodSpec) if err := withLPUConfigVolume(conductorPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { return err diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 9945b0be860b..9a64845c9f66 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -122,6 +122,7 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { require.Equal(t, projection.Digest().String(), rendered.Annotations[WorkloadDigestAnnotation]) require.NotContains(t, rendered.Annotations, ExecutionBackendAnnotation) for _, clique := range rendered.Spec.Template.Cliques { + require.Equal(t, SchedulerName, clique.Spec.PodSpec.SchedulerName) require.NotContains(t, clique.Annotations, ExecutionBackendAnnotation) if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { require.Equal(t, projection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 49ad235ebc2d..9ab5f3724735 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -594,7 +594,7 @@ spec: - mountPath: /dev/shm name: shared-memory restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index ad30f181f42d..7e3ebff30622 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -389,7 +389,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index a6522a179d1e..e28a92c3414e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -369,7 +369,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index c208953abe4c..7b45d83594f1 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -731,7 +731,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index b98e975968d0..c3b1119dd046 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -365,7 +365,7 @@ spec: - name: gpu resourceClaimTemplateName: decode-gpu restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 3c49808a191f..59875cc449ce 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -433,7 +433,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 1182ba239efb..0f34c89fb63e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -849,7 +849,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index ba5300e16305..a9e72c334a23 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -454,7 +454,7 @@ spec: name: ssh-secret readOnly: true restartPolicy: Always - schedulerName: default-scheduler + schedulerName: lpx-scheduler securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch From 7ff723d8d92ca56bcdd045d9df6c21bf7ff1b857 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:54:16 +0300 Subject: [PATCH 086/196] refactor(operator)!: let LPX pod templates own pod settings (#521) Signed-off-by: Andrew Paprotsky (cherry picked from commit 94d4dee76e1bfa5120b53937db5458295e2b1960) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 177 +++++++++++++++--- deploy/operator/internal/dynamo/graph_lpx.go | 19 +- .../internal/dynamo/graph_lpx_test.go | 45 +++-- .../internal/dynamo/lpx/cyborg_render_test.go | 6 - .../internal/dynamo/lpx/cyborg_runtime.go | 17 +- .../operator/internal/dynamo/lpx/lpu_pod.go | 54 ------ .../operator/internal/dynamo/lpx/pod_roles.go | 4 +- .../internal/dynamo/lpx/pod_roles_test.go | 87 +-------- deploy/operator/internal/dynamo/lpx/render.go | 19 +- .../internal/dynamo/lpx/render_test.go | 8 +- .../internal/dynamo/lpx/runtime_agent.go | 59 +----- .../dynamo/lpx/runtime_startup_test.go | 12 +- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 68 ++++++- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 17 +- .../node-local-v2-hybrid.input.yaml | 65 ++++++- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 17 +- .../node-local-v2-lpu-only.input.yaml | 79 +++++--- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 17 -- .../node-local-v2-specdecode.input.yaml | 83 +++++--- .../node-local-v2-specdecode.yaml | 45 +---- .../node-local-v3-hx-hybrid.input.yaml | 66 ++++++- .../node-local-v3-hx-hybrid.yaml | 17 +- .../node-local-v3-hx-lpu-only.input.yaml | 72 ++++--- .../node-local-v3-hx-lpu-only.yaml | 20 -- .../node-local-v3-hx-specdecode.input.yaml | 76 +++++--- .../node-local-v3-hx-specdecode.yaml | 54 +----- .../from_dgd_yaml/single_v2.input.yaml | 85 ++++++--- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 17 -- 28 files changed, 699 insertions(+), 606 deletions(-) delete mode 100644 deploy/operator/internal/dynamo/lpx/lpu_pod.go diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 9a7be9ff0959..38928e42489a 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -6,6 +6,7 @@ package lpx import ( "encoding/json" "fmt" + "maps" "os" "path/filepath" "slices" @@ -30,6 +31,7 @@ import ( "gotest.tools/v3/golden" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -1162,7 +1164,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, "1024", runtimeEnv["GLUE_RDMA_PATH_MTU"]) require.Equal(t, "mlx5_0", runtimeEnv["NIC_NAME"]) require.Equal(t, "19878", runtimeEnv["READINESS_PORT"]) - require.Equal(t, "default", runtimeEnv["LPU_MODEL_NAME"]) + require.Equal(t, "metadata.annotations['"+lpxv1alpha1.PodModelAnnotation+"']", envValueSource(main.Env, "LPU_MODEL_NAME").FieldRef.FieldPath) gasDirSource := envValueSource(main.Env, "GAS_DIR") require.NotNil(t, gasDirSource) require.NotNil(t, gasDirSource.ConfigMapKeyRef) @@ -1177,16 +1179,11 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NotContains(t, runtimeEnv, "PARTITION_ID") require.NotContains(t, runtimeEnv, "RANK_IN_PARTITION") - t.Log("Preserve custom tolerations and mounts alongside the required Agent networking and storage") + t.Log("Preserve authored Agent tolerations, networking and storage") require.True(t, agent.Spec.PodSpec.HostIPC) require.True(t, agent.Spec.PodSpec.HostNetwork) require.Equal(t, corev1.DNSClusterFirstWithHostNet, agent.Spec.PodSpec.DNSPolicy) - require.Equal(t, []corev1.Toleration{ - {Key: "cluster.example/custom", Operator: corev1.TolerationOpExists}, - {Key: "lpu.nvidia.com/node", Operator: corev1.TolerationOpExists}, - {Key: "lpu.nvidia.com/lpu", Operator: corev1.TolerationOpExists}, - {Key: "lpu.nvidia.com/node-v2", Operator: corev1.TolerationOpExists}, - }, agent.Spec.PodSpec.Tolerations) + require.Equal(t, agentTemplate.Spec.Tolerations, agent.Spec.PodSpec.Tolerations) for _, name := range []string{"config", "hugepages", "host-dev", "host-sys", "ssh-secret"} { require.True(t, slices.ContainsFunc(agent.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }), "missing volume %s", name) } @@ -1197,7 +1194,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) require.Contains(t, agent.Spec.PodSpec.Volumes, customConfigVolume) require.True(t, main.Resources.Requests.Cpu().Equal(resource.MustParse("62"))) - require.Equal(t, resource.MustParse("4096Mi"), main.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"]) + require.True(t, main.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"].Equal(resource.MustParse("4096Mi"))) t.Log("Verify Cyborg receives its generated config and InfiniBand bindings instead of fallback PVCs") require.Contains(t, cyborg.Spec.PodSpec.Containers, sidecar) @@ -1275,6 +1272,21 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing.T) { t.Log("Share the immutable build registry across entrypoint cases") registry := newTestDataModelRegistry(t, t.TempDir()) + staticEnvNames := []string{ + "POD_IP", "POD_NAME", "POD_NAMESPACE", "POD_UID", "LPU_MODEL_NAME", "CYBORG_FPGA_GPI_REPLICA_INDEX", + "DYN_SYSTEM_ENABLED", "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", "DYN_SYSTEM_PORT", "DYN_HEALTH_CHECK_ENABLED", + "NIXL_TELEMETRY_ENABLE", "NIXL_TELEMETRY_EXPORTER", "NIXL_TELEMETRY_PROMETHEUS_PORT", "DYN_FORWARDPASS_METRIC_PORT", + } + + t.Log("Use ordinary component defaults as the shared environment and port contract") + commonDefaults, err := (&dynamo.BaseComponentDefaults{}).GetBaseContainer(dynamo.ComponentContext{}) + require.NoError(t, err) + workerDefaults, err := dynamo.NewWorkerDefaults().GetBaseContainer(dynamo.ComponentContext{}) + require.NoError(t, err) + roleDefaults := map[string]corev1.Container{ + lpxv1alpha1.PodRoleAgent: commonDefaults, lpxv1alpha1.PodRoleConductor: commonDefaults, + lpxv1alpha1.PodRoleCyborgWorker: workerDefaults, + } modes := []struct { name string file string @@ -1289,19 +1301,19 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. {name: "HX hybrid", file: "node-local-v3-hx-hybrid.input.yaml", hybrid: true, hx: true}, } intents := []struct { - name string - command []string - args []string - cyborgCommand []string - cyborgArgs []string - hybridOnly bool - configPath *string - configRole int - envFrom bool - customProbes bool + name string + command []string + args []string + cyborgCommand []string + cyborgArgs []string + hybridOnly bool + configPath *string + configRole int + envFrom bool + customPodSettings bool }{ {name: "image entrypoint and image command"}, - {name: "custom exec health probes", command: []string{"/opt/custom-runtime"}, customProbes: true}, + {name: "custom pod settings and exec health probes", command: []string{"/opt/custom-runtime"}, customPodSettings: true}, {name: "default startup without config mount", configPath: ptr.To(""), configRole: 1}, {name: "default startup with misplaced config mount", configPath: ptr.To("/custom"), configRole: 1}, {name: "default Agent without config mount", configPath: ptr.To("")}, @@ -1347,11 +1359,12 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. continue } t.Run(intent.name, func(t *testing.T) { - t.Log("Render the LPX LPU and GPU roles from the same command intent") + t.Log("Author LPX LPU and GPU runtime settings independently of fixture defaults") dgd := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, dgd)) component := dgd.GetComponentByName("lpu") require.NotNil(t, component) + component.SharedMemorySize = nil agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) templates := []*corev1.PodTemplateSpec{agent.PodTemplate, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate} for roleIndex, template := range templates { @@ -1365,8 +1378,62 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. } main.Command = slices.Clone(command) main.Args = slices.Clone(args) + template.Spec.SecurityContext, main.SecurityContext = nil, nil + template.Spec.HostNetwork, template.Spec.HostIPC = false, false + template.Spec.HostUsers, template.Spec.DNSConfig = nil, nil + template.Spec.DNSPolicy, template.Spec.RestartPolicy = "", "" + template.Spec.TerminationGracePeriodSeconds, template.Spec.Tolerations = nil, nil + main.Ports = nil + main.Env = slices.DeleteFunc(main.Env, func(variable corev1.EnvVar) bool { return slices.Contains(staticEnvNames, variable.Name) }) + template.Spec.Volumes = slices.DeleteFunc(template.Spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == commonconsts.KubeValueNameSharedMemory }) + main.VolumeMounts = slices.DeleteFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { + return mount.MountPath == commonconsts.DefaultSharedMemoryMountPath + }) + requests, limits := main.Resources.Requests, main.Resources.Limits + main.Resources.Requests, main.Resources.Limits = corev1.ResourceList{}, corev1.ResourceList{} + maps.Copy(main.Resources.Requests, requests) + maps.Copy(main.Resources.Limits, limits) + for _, resources := range []corev1.ResourceList{main.Resources.Requests, main.Resources.Limits} { + maps.DeleteFunc(resources, func(name corev1.ResourceName, _ resource.Quantity) bool { + return name != corev1.ResourceName(commonconsts.KubeResourceGPUNvidia) + }) + } main.StartupProbe, main.LivenessProbe, main.ReadinessProbe = nil, nil, nil - if intent.customProbes { + if intent.customPodSettings { + component.SharedMemorySize = ptr.To(resource.MustParse("0")) + template.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(1000)), RunAsGroup: ptr.To(int64(2000)), RunAsNonRoot: ptr.To(true), + FSGroup: ptr.To(int64(3000)), FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), + SupplementalGroups: []int64{4000}, Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } + main.SecurityContext = &corev1.SecurityContext{ + Privileged: ptr.To(false), RunAsUser: ptr.To(int64(1000)), RunAsGroup: ptr.To(int64(2000)), + RunAsNonRoot: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), + } + template.Spec.HostNetwork, template.Spec.HostIPC = true, true + template.Spec.HostUsers = ptr.To(true) + template.Spec.DNSPolicy = corev1.DNSNone + template.Spec.DNSConfig = &corev1.PodDNSConfig{Nameservers: []string{"192.0.2.1"}, Searches: []string{"custom.test"}} + template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure + template.Spec.TerminationGracePeriodSeconds = ptr.To(int64(17)) + template.Spec.Tolerations = []corev1.Toleration{{Key: "custom.example/runtime", Operator: corev1.TolerationOpEqual, Value: "allowed", Effect: corev1.TaintEffectNoSchedule}} + main.Ports = []corev1.ContainerPort{{Name: "custom", ContainerPort: 4321, Protocol: corev1.ProtocolUDP}} + for _, name := range staticEnvNames { + main.Env = append(main.Env, corev1.EnvVar{Name: name, Value: "authored-" + name}) + } + template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{ + Name: commonconsts.KubeValueNameSharedMemory, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("32Mi")), + }}, + }) + main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: commonconsts.KubeValueNameSharedMemory, MountPath: commonconsts.DefaultSharedMemoryMountPath}) + main.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("500m") + main.Resources.Limits[corev1.ResourceCPU] = resource.MustParse("2") + for _, resources := range []corev1.ResourceList{main.Resources.Requests, main.Resources.Limits} { + resources[corev1.ResourceMemory] = resource.MustParse("128Mi") + resources[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("2Mi") + } main.StartupProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "started"}}}, FailureThreshold: 12, PeriodSeconds: 4, TimeoutSeconds: 2, @@ -1380,6 +1447,12 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. FailureThreshold: 2, PeriodSeconds: 5, TimeoutSeconds: 1, } } + template.Spec.Containers = append(template.Spec.Containers, corev1.Container{ + Name: "observer", Image: "observer:test", Command: []string{"/custom-observer"}, + Env: []corev1.EnvVar{{Name: "POD_IP", Value: "sidecar-owned"}}, + SecurityContext: &corev1.SecurityContext{RunAsUser: ptr.To(int64(2000))}, + }) + main = &template.Spec.Containers[0] main.Env = append(main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) if intent.envFrom && roleIndex == 0 { main.EnvFrom = []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated-env"}}}} @@ -1416,19 +1489,70 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. } require.NoError(t, err) - t.Log("Verify every role preserves authored startup and health probes, including omission") + t.Log("Verify shared defaults remain overridable while runtime-specific omissions stay empty") var agents, conductors, cyborgs int for _, clique := range pcs.Spec.Template.Cliques { - main := clique.Spec.PodSpec.Containers[0] - authored := templates[1].Spec.Containers[0] + pod := clique.Spec.PodSpec + main := pod.Containers[0] + authoredPod := templates[1].Spec if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { - authored = templates[0].Spec.Containers[0] + authoredPod = templates[0].Spec + } + authored := authoredPod.Containers[0] + if !intent.customPodSettings { + defaults := roleDefaults[clique.Annotations[lpxv1alpha1.PodRoleAnnotation]] + authoredPod.RestartPolicy = corev1.RestartPolicyAlways + authoredPod.TerminationGracePeriodSeconds = ptr.To(int64(60)) + authored.Ports = defaults.Ports + authored.Env = append(slices.Clone(defaults.Env), corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}) + authoredPod.Volumes = append(slices.Clone(authoredPod.Volumes), corev1.Volume{ + Name: commonconsts.KubeValueNameSharedMemory, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("8Gi")), + }}, + }) + authored.VolumeMounts = append(slices.Clone(authored.VolumeMounts), corev1.VolumeMount{ + Name: commonconsts.KubeValueNameSharedMemory, MountPath: commonconsts.DefaultSharedMemoryMountPath, + }) } require.Equal(t, authored.Command, main.Command, "%s command", clique.Name) require.True(t, slices.Equal(authored.Args, main.Args), "%s args", clique.Name) require.Equal(t, authored.StartupProbe, main.StartupProbe, "%s startup", clique.Name) require.Equal(t, authored.LivenessProbe, main.LivenessProbe, "%s liveness", clique.Name) require.Equal(t, authored.ReadinessProbe, main.ReadinessProbe, "%s readiness", clique.Name) + require.Equal(t, authored.SecurityContext, main.SecurityContext, "%s container security", clique.Name) + require.Equal(t, authoredPod.SecurityContext, pod.SecurityContext, "%s pod security", clique.Name) + require.Equal(t, authoredPod.HostNetwork, pod.HostNetwork, "%s host network", clique.Name) + require.Equal(t, authoredPod.HostIPC, pod.HostIPC, "%s host IPC", clique.Name) + require.Equal(t, authoredPod.HostUsers, pod.HostUsers, "%s host users", clique.Name) + require.Equal(t, authoredPod.DNSPolicy, pod.DNSPolicy, "%s DNS policy", clique.Name) + require.Equal(t, authoredPod.DNSConfig, pod.DNSConfig, "%s DNS config", clique.Name) + require.Equal(t, authoredPod.RestartPolicy, pod.RestartPolicy, "%s restart", clique.Name) + require.Equal(t, authoredPod.TerminationGracePeriodSeconds, pod.TerminationGracePeriodSeconds, "%s grace period", clique.Name) + require.Equal(t, authoredPod.Tolerations, pod.Tolerations, "%s tolerations", clique.Name) + require.Equal(t, authored.Ports, main.Ports, "%s ports", clique.Name) + require.Equal(t, authoredPod.InitContainers, pod.InitContainers, "%s init containers", clique.Name) + require.Equal(t, authoredPod.Containers[1:], pod.Containers[1:], "%s sidecars", clique.Name) + actualEnv := slices.DeleteFunc(slices.Clone(main.Env), func(variable corev1.EnvVar) bool { return !slices.Contains(staticEnvNames, variable.Name) }) + expectedEnv := slices.DeleteFunc(slices.Clone(authored.Env), func(variable corev1.EnvVar) bool { return !slices.Contains(staticEnvNames, variable.Name) }) + require.ElementsMatch(t, expectedEnv, actualEnv, "%s static environment", clique.Name) + actualResources := main.Resources.DeepCopy() + delete(actualResources.Requests, "lpu.nvidia.com/lpu") + delete(actualResources.Limits, "lpu.nvidia.com/lpu") + delete(actualResources.Requests, "nvidia.com/lpu") + delete(actualResources.Limits, "nvidia.com/lpu") + require.True(t, apiequality.Semantic.DeepEqual(authored.Resources, *actualResources), "%s resources: expected %v, got %v", clique.Name, authored.Resources, *actualResources) + actualVolumes := slices.DeleteFunc(slices.Clone(pod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) + expectedVolumes := slices.DeleteFunc(slices.Clone(authoredPod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) + require.ElementsMatch(t, expectedVolumes, actualVolumes, "%s shared memory volumes", clique.Name) + actualMounts := slices.DeleteFunc(slices.Clone(main.VolumeMounts), func(mount corev1.VolumeMount) bool { + return mount.MountPath != commonconsts.DefaultSharedMemoryMountPath + }) + expectedMounts := slices.DeleteFunc(slices.Clone(authored.VolumeMounts), func(mount corev1.VolumeMount) bool { + return mount.MountPath != commonconsts.DefaultSharedMemoryMountPath + }) + require.ElementsMatch(t, expectedMounts, actualMounts, "%s shared memory mounts", clique.Name) switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { case lpxv1alpha1.PodRoleAgent: agents++ @@ -1441,7 +1565,6 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. require.NotContains(t, main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) case lpxv1alpha1.PodRoleCyborgWorker: cyborgs++ - require.NotEmpty(t, main.Ports) } } require.Positive(t, agents) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 644ec8e9da96..1c465b0d288d 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -312,9 +312,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p role.ComponentType = v1beta1.ComponentTypeDecode role.Replicas = ptr.To(replicas) role.MinAvailable = nil - var defaults ComponentDefaults = &podTemplateRuntimeDefaults{ - ComponentDefaults: ComponentDefaultsFactory(string(v1beta1.ComponentTypeDecode)), - } + var defaults ComponentDefaults = &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} if workload.BuildFamily() == dynamolpx.BuildFamilyXT { input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan, lpuTemplate.Spec) if err != nil { @@ -381,6 +379,14 @@ func (d *podTemplateRuntimeDefaults) GetBaseContainer(context ComponentContext) container.StartupProbe = nil container.LivenessProbe = nil container.ReadinessProbe = nil + + // Bind the pod address for LPX runtimes before authored environment overrides. + container.Env = append(container.Env, corev1.EnvVar{ + Name: commonconsts.PodIPEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }, + }) return container, nil } @@ -412,6 +418,7 @@ func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentCont } // renderSelectedLPXRole consumes a private component copy; other inputs are read-only. +// component and its PodTemplate must be non-nil. // alphaComponent may be nil. func renderSelectedLPXRole( component *v1beta1.DynamoComponentDeploymentSharedSpec, @@ -445,10 +452,8 @@ func renderSelectedLPXRole( return nil, err } - // Keep the LPX volume-permission default without changing authored security contexts. - if component.PodTemplate == nil || component.PodTemplate.Spec.SecurityContext == nil { - basePodSpec.SecurityContext.FSGroupChangePolicy = ptr.To(corev1.FSGroupChangeOnRootMismatch) - } + // The shared renderer's filesystem-group fallback is not part of the LPX template contract. + basePodSpec.SecurityContext = component.PodTemplate.Spec.SecurityContext.DeepCopy() // LPX supplies the generated config volume after merging; discard only inferred PVCs. explicitVolumes := component.PodTemplate.Spec.Volumes diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 8a5eab3cba47..6d14566074c9 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -36,23 +36,20 @@ func TestBackendDetectionSkipsNonWorkers(t *testing.T) { } } -func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { +func TestRenderSelectedLPXRolePodSettings(t *testing.T) { tests := []struct { - name string - authored *corev1.PodSecurityContext - expected *corev1.PodSecurityContext + name string + authored *corev1.PodSecurityContext + sharedMemorySize *resource.Quantity + volumes []corev1.Volume + mounts []corev1.VolumeMount }{ { name: "no authored security context", - expected: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), - }, }, { name: "authored fsGroup without policy", authored: &corev1.PodSecurityContext{FSGroup: ptr.To(int64(2000))}, - expected: &corev1.PodSecurityContext{FSGroup: ptr.To(int64(2000))}, }, { name: "authored always policy", @@ -60,25 +57,25 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { FSGroup: ptr.To(int64(2000)), FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), }, - expected: &corev1.PodSecurityContext{ - FSGroup: ptr.To(int64(2000)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), - }, }, { name: "authored empty security context", authored: &corev1.PodSecurityContext{}, - expected: &corev1.PodSecurityContext{}, }, { name: "authored root security context", authored: &corev1.PodSecurityContext{ RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), }, - expected: &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(0)), RunAsGroup: ptr.To(int64(0)), RunAsNonRoot: ptr.To(false), - }, }, + { + name: "explicit shared memory size", sharedMemorySize: ptr.To(resource.MustParse("64Mi")), + volumes: []corev1.Volume{{Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("64Mi")), + }}}}, + mounts: []corev1.VolumeMount{{Name: "shared-memory", MountPath: "/dev/shm"}}, + }, + {name: "explicit zero shared memory size", sharedMemorySize: ptr.To(resource.MustParse("0"))}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -87,7 +84,7 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "hybrid", Namespace: "test"}, } component := &v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "engine", ComponentType: v1beta1.ComponentTypeDecode, + ComponentName: "engine", ComponentType: v1beta1.ComponentTypeDecode, SharedMemorySize: test.sharedMemorySize, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: "main", Image: "cyborg:test"}}, SecurityContext: test.authored, @@ -100,8 +97,16 @@ func TestRenderSelectedLPXRoleSecurityContext(t *testing.T) { &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()}) require.NoError(t, err) - t.Log("Retain authored security context and default only an absent one") - require.Equal(t, test.expected, template.Spec.SecurityContext) + t.Log("Retain authored security and use the ordinary shared memory default unless resized or disabled") + if test.sharedMemorySize == nil { + test.volumes = []corev1.Volume{{Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("8Gi")), + }}}} + test.mounts = []corev1.VolumeMount{{Name: "shared-memory", MountPath: "/dev/shm"}} + } + require.Equal(t, test.authored, template.Spec.SecurityContext) + require.ElementsMatch(t, test.volumes, template.Spec.Volumes) + require.ElementsMatch(t, test.mounts, template.Spec.Containers[0].VolumeMounts) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 40fda5572a20..45ba6ad2e7be 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -133,12 +133,6 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ Name: "CYBORG_BATCH_SIZE", Value: "3", }) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: "CYBORG_FPGA_GPI_REPLICA_INDEX", - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.labels['grove.io/podclique-pod-index']", - }}, - }) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ Name: "GBUILD_MANIFEST_PATH", Value: "/models/model-build/manifest.v2.capnp.bin", }) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index d3b73974a784..e183089613aa 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -16,12 +16,11 @@ import ( const ( // CyborgBatchSizeEnv configures the batch handled by each Cyborg replica. - CyborgBatchSizeEnv = "CYBORG_BATCH_SIZE" - cyborgFpgaGpiIOFPGACountEnv = "CYBORG_FPGA_GPI_IO_FPGA_COUNT" - cyborgFpgaGpiReplicaIndexEnv = "CYBORG_FPGA_GPI_REPLICA_INDEX" - cyborgSwaCacheIDsEnv = "CYBORG_SWA_CACHE_IDS" - gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" - grovePodCliquePodIndexPath = "metadata.labels['grove.io/podclique-pod-index']" + CyborgBatchSizeEnv = "CYBORG_BATCH_SIZE" + cyborgFpgaGpiIOFPGACountEnv = "CYBORG_FPGA_GPI_IO_FPGA_COUNT" + cyborgSwaCacheIDsEnv = "CYBORG_SWA_CACHE_IDS" + gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" + grovePodCliquePodIndexPath = "metadata.labels['grove.io/podclique-pod-index']" ) // applyCyborgManifestPath projects an authoritative manifest location into one Cyborg container. @@ -41,12 +40,6 @@ func applyCyborgManifestPath(container *corev1.Container, projection *ModelProje func applyCyborgRuntimeIO(container *corev1.Container, cyborgBatchSize int, ioFPGACount int32) { setContainerEnv(container, corev1.EnvVar{Name: cyborgFpgaGpiIOFPGACountEnv, Value: strconv.FormatInt(int64(ioFPGACount), 10)}, - corev1.EnvVar{ - Name: cyborgFpgaGpiReplicaIndexEnv, - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: grovePodCliquePodIndexPath, - }}, - }, corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: strconv.Itoa(cyborgBatchSize)}, ) } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_pod.go b/deploy/operator/internal/dynamo/lpx/lpu_pod.go deleted file mode 100644 index 4eb89f9f4c35..000000000000 --- a/deploy/operator/internal/dynamo/lpx/lpu_pod.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "slices" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/ptr" -) - -const ( - lpuAgentContainerName = "agent" - defaultLPUAgentCPUs int64 = 62 -) - -func updateWorkerPodSpec(podSpec *corev1.PodSpec) { - // TODO can we remove some of this when we get the device-plugin? - podSpec.HostIPC = true - podSpec.HostNetwork = true - podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet -} - -// ensureLPUNodeTolerations adds standard LPU-node tolerations to podSpec, which must be nonnil. -func ensureLPUNodeTolerations(podSpec *corev1.PodSpec) { - // Derive each complete toleration from the platform's ordered taint keys. - for _, key := range [...]string{"lpu.nvidia.com/node", "lpu.nvidia.com/lpu", "lpu.nvidia.com/node-v2"} { - toleration := corev1.Toleration{Key: key, Operator: corev1.TolerationOpExists} - if !slices.Contains(podSpec.Tolerations, toleration) { - podSpec.Tolerations = append(podSpec.Tolerations, toleration) - } - } -} - -func applyLPUWorkerContainerBase(container *corev1.Container) { - // Workers need privileged device access. - container.SecurityContext = &corev1.SecurityContext{ - Privileged: ptr.To(true), - RunAsUser: ptr.To(int64(0)), - } - - // Agent scheduling already initialized device resources; add runtime CPU and hugepage requirements. - cpuRequest := *resource.NewQuantity(defaultLPUAgentCPUs, resource.DecimalSI) - if cpuLimit, ok := container.Resources.Limits[corev1.ResourceCPU]; ok && cpuLimit.Cmp(cpuRequest) < 0 { - cpuRequest = cpuLimit.DeepCopy() - } - container.Resources.Requests[corev1.ResourceCPU] = cpuRequest - container.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") - container.Resources.Limits[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("4096Mi") -} diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index bec31880c379..2efae24653a5 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -21,8 +21,8 @@ const ( v2LPUResourceName corev1.ResourceName = "lpu.nvidia.com/lpu" v3LPUResourceName corev1.ResourceName = "nvidia.com/lpu" - lpuModelNameEnvVar = "LPU_MODEL_NAME" - allocationEnvVar = "LPX_ALLOCATION" + lpuAgentContainerName = "agent" + allocationEnvVar = "LPX_ALLOCATION" ) // configureAgentScheduling consumes a fresh, nonnil Agent PodSpec containing main. diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 0ebd58e7cb22..9d50504f623c 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/ptr" ) func TestSelectedRolePodSpecsMaterializeFamilyResourceOnlyOnAgentMainContainer(t *testing.T) { @@ -111,35 +110,6 @@ func TestSelectedRolePodSpecsMaterializeFamilyResourceOnlyOnAgentMainContainer(t } } -func TestApplyLPUWorkerContainerBaseClampsCPURequestToLimit(t *testing.T) { - for _, test := range []struct { - name string - limit string - want string - }{ - {name: "lower limit", limit: "16", want: "16"}, - {name: "higher limit", limit: "64", want: "62"}, - {name: "no limit", want: "62"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Configure the worker CPU limit") - container := corev1.Container{Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{}, - Limits: corev1.ResourceList{}, - }} - if test.limit != "" { - container.Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(test.limit)} - } - - t.Log("Apply the LPX worker runtime defaults") - applyLPUWorkerContainerBase(&container) - - t.Log("Clamp the default request to the configured limit") - require.True(t, container.Resources.Requests.Cpu().Equal(resource.MustParse(test.want))) - }) - } -} - func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { lists := []corev1.ResourceList{ spec.InitContainers[0].Resources.Limits, @@ -156,9 +126,8 @@ func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { func TestConfigureNodeLocalRuntimeBindings(t *testing.T) { for _, role := range []string{"conductor", "agent"} { t.Run(role, func(t *testing.T) { - t.Log("Author main-container references and HX resource requirements") + t.Log("Author main-container references and resource requirements") pod := corev1.PodSpec{ - HostUsers: ptr.To(false), Containers: []corev1.Container{{ Name: commonconsts.MainContainerName, Env: []corev1.EnvVar{ @@ -184,69 +153,21 @@ func TestConfigureNodeLocalRuntimeBindings(t *testing.T) { t.Log("Bind the selected role and retarget every authored main-container reference") if role == "conductor" { - configureNodeLocalConductorRuntime(&pod, BuildFamilyHX, "agt") + configureNodeLocalConductorRuntime(&pod, "agt") } else { - configureNodeLocalAgentRuntime(&pod, BuildFamilyHX) + configureAgentIdentity(&pod) } container := pod.Containers[0] require.Equal(t, role, container.Name) require.Equal(t, role, testContainerEnvValue(container.Env, "CONTAINER_NAME")) require.Equal(t, role, testContainerEnvSource(container.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) require.Equal(t, role, pod.Volumes[0].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) - require.Equal(t, "status.podIP", testContainerEnvSource(container.Env, "POD_IP").FieldRef.FieldPath) require.Equal(t, before.Containers[0].Resources, container.Resources) - t.Log("Apply worker networking only to the Agent and placement only to the conductor") + t.Log("Bind placement only to the conductor") if role == "conductor" { require.Equal(t, "agt", testContainerEnvValue(container.Env, "LPX_ALLOCATION")) - require.Equal(t, before.HostUsers, pod.HostUsers) - require.False(t, pod.HostIPC) - require.False(t, pod.HostNetwork) - } else { - require.Nil(t, pod.HostUsers) - require.True(t, pod.HostIPC) - require.True(t, pod.HostNetwork) - require.Equal(t, corev1.DNSClusterFirstWithHostNet, pod.DNSPolicy) - } - }) - } -} - -func TestConfigureDirectHybridAgentRuntimePreservesPodSecurity(t *testing.T) { - for _, test := range []struct { - name string - security *corev1.PodSecurityContext - }{ - {name: "omitted"}, - {name: "authored", security: &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(1000)), - RunAsGroup: ptr.To(int64(1000)), - RunAsNonRoot: ptr.To(true), - FSGroup: ptr.To(int64(1000)), - FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), - SupplementalGroups: []int64{2000}, - Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, - SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, - }}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Configure a direct worker with the authored Pod security context") - pod := corev1.PodSpec{ - Containers: []corev1.Container{{Name: commonconsts.MainContainerName}}, - SecurityContext: test.security.DeepCopy(), - } - configureAgentScheduling(&pod, BuildFamilyXT) - configureDirectHybridAgentRuntime(&pod, "graph-lpu") - - t.Log("Change the worker identity to root while preserving every other security field") - want := test.security.DeepCopy() - if want == nil { - want = &corev1.PodSecurityContext{} } - want.RunAsUser = ptr.To(int64(0)) - want.RunAsGroup = ptr.To(int64(0)) - want.RunAsNonRoot = ptr.To(false) - require.Equal(t, want, pod.SecurityContext) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 70b645d8f4f7..9bbcd40c2bed 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -160,15 +160,6 @@ func RenderSelectedNodeLocal( podSpec = *podSpec.DeepCopy() } - // Resolve per-model placeholders only after separating siblings' backing data. - for containerIndex := range podSpec.Containers { - env := podSpec.Containers[containerIndex].Env - for envIndex := range env { - if env[envIndex].Name == lpuModelNameEnvVar { - env[envIndex] = corev1.EnvVar{Name: lpuModelNameEnvVar, Value: projection.Model()} - } - } - } annotations := roleAnnotations(template.Annotations, lpxv1alpha1.PodRoleAgent, projection.Digest().String()) annotations[commonconsts.AnnotationExtraResourcesHash] = configHash annotations[lpxv1alpha1.PodModelAnnotation] = projection.model @@ -260,21 +251,15 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo return err } } - if workload.Pipeline() == PipelineSpecDecode || workload.Pipeline() == PipelineLPX { - agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) - setContainerEnv(agent, corev1.EnvVar{Name: lpuModelNameEnvVar}) - } - if workload.Pipeline() == PipelineLPX { configureDirectHybridAgentRuntime(agentPodSpec, configMapName) } else { if conductorPodSpec != nil { - configureNodeLocalConductorRuntime(conductorPodSpec, workload.BuildFamily(), allocation) + configureNodeLocalConductorRuntime(conductorPodSpec, allocation) } - configureNodeLocalAgentRuntime(agentPodSpec, workload.BuildFamily()) + configureAgentIdentity(agentPodSpec) } - ensureLPUNodeTolerations(agentPodSpec) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 9a64845c9f66..361da8e6279f 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -197,7 +197,7 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { }}}}, }, }} - conductorPodSpec.Containers[0].Env = []corev1.EnvVar{{Name: lpuModelNameEnvVar, ValueFrom: modelAnnotationSource}} + conductorPodSpec.Containers[0].Env = []corev1.EnvVar{{Name: "LPU_MODEL_NAME", ValueFrom: modelAnnotationSource}} t.Log("Render conductor and Agent roles from the base PodSpec") rendered, err := renderSelectedForTest(renderTestPCS(false), []*ModelProjection{projection}, RenderInput{ @@ -224,9 +224,9 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { require.Equal(t, []string{"/bin/bash"}, agent.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"-c", "custom-agent"}, agent.Spec.PodSpec.Containers[0].Args) require.Contains(t, agent.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: lpuModelNameEnvVar, - Value: projection.Model(), + Name: "LPU_MODEL_NAME", ValueFrom: modelAnnotationSource, }) + require.Equal(t, projection.Model(), agent.Annotations[lpxv1alpha1.PodModelAnnotation]) } func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { @@ -260,6 +260,8 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { template.Labels = map[string]string{"owner": component.ComponentName} template.Annotations = map[string]string{"owner": component.ComponentName} template.Spec.Containers[0].Image = component.ComponentName + "-runtime" + template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AUTHORED_STAGE", Value: component.ComponentName}} + template.Spec.Tolerations = []corev1.Toleration{{Key: "custom.example/stage", Operator: corev1.TolerationOpEqual, Value: component.ComponentName}} stages[component.ComponentName] = *template.DeepCopy() } conductorTemplate := source.Spec.Components[1].ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 960f03ac75c6..7e762c548de8 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -6,13 +6,10 @@ package lpx import ( - "slices" - dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" ) // configureDirectHybridAgentRuntime lowers an LPX-scheduled PodSpec into the @@ -21,12 +18,7 @@ func configureDirectHybridAgentRuntime( agentPodSpec *corev1.PodSpec, lpuConfigMapName string, ) { - agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) - // Keep the authored startup while applying the Agent identity and worker resources. - agent.Name = lpuAgentContainerName - applyLPUWorkerContainerBase(agent) - - retargetMainContainerReferences(agentPodSpec, agent) + agent := configureAgentIdentity(agentPodSpec) agent.Env = append(agent.Env, corev1.EnvVar{ @@ -44,68 +36,27 @@ func configureDirectHybridAgentRuntime( }}, }, ) - setNodeLocalPodIPEnv(agent, true) - updateWorkerPodSpec(agentPodSpec) - - // Keep volume permissions and other Pod settings while forcing the Agent's root identity. - if agentPodSpec.SecurityContext == nil { - agentPodSpec.SecurityContext = &corev1.PodSecurityContext{} - } - agentPodSpec.SecurityContext.RunAsUser = ptr.To(int64(0)) - agentPodSpec.SecurityContext.RunAsGroup = ptr.To(int64(0)) - agentPodSpec.SecurityContext.RunAsNonRoot = ptr.To(false) } // configureNodeLocalConductorRuntime consumes a fresh conductor PodSpec with a validated main container. func configureNodeLocalConductorRuntime( conductorPodSpec *corev1.PodSpec, - targetFamily BuildFamily, allocation string, ) { // Bind placement data without interpreting the template's executable or arguments. conductor := common.FindContainerByName(conductorPodSpec.Containers, commonconsts.MainContainerName) - setNodeLocalPodIPEnv(conductor, targetFamily == BuildFamilyXT) conductor.Name = dynamov1beta1.ComponentRoleLPXConductor setContainerEnv(conductor, corev1.EnvVar{Name: allocationEnvVar, Value: allocation}) retargetMainContainerReferences(conductorPodSpec, conductor) } -// configureNodeLocalAgentRuntime consumes a fresh Agent PodSpec with a validated main container. -func configureNodeLocalAgentRuntime( - agentPodSpec *corev1.PodSpec, - targetFamily BuildFamily, -) { - // Apply the local LPU role identity and worker resources. +// configureAgentIdentity names the main runtime and retargets its container references. +// agentPodSpec must be non-nil and contain the validated main container. +func configureAgentIdentity(agentPodSpec *corev1.PodSpec) *corev1.Container { agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) - isXT := targetFamily == BuildFamilyXT - setNodeLocalPodIPEnv(agent, isXT) agent.Name = lpuAgentContainerName retargetMainContainerReferences(agentPodSpec, agent) - agentPodSpec.HostUsers = nil - updateWorkerPodSpec(agentPodSpec) - - // Apply the XT worker's resource and security requirements. - if isXT { - applyLPUWorkerContainerBase(agent) - agent.SecurityContext.RunAsGroup = ptr.To(int64(0)) - agent.SecurityContext.RunAsNonRoot = ptr.To(false) - } -} - -func setNodeLocalPodIPEnv(container *corev1.Container, isXT bool) { - if isXT && - slices.ContainsFunc(container.Env, func(existing corev1.EnvVar) bool { return existing.Name == "POD_IP" }) { - return - } - container.Env = append(container.Env, corev1.EnvVar{ - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "status.podIP", - }, - }, - }) + return agent } // retargetMainContainerReferences requires its non-nil target to point into the non-nil PodSpec's Containers. diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index aeb4a80a2733..f52902111a2d 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -99,11 +99,11 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { configureAgentScheduling(&pod, role.family) switch { case role.conductor: - configureNodeLocalConductorRuntime(&pod, role.family, "allocation") + configureNodeLocalConductorRuntime(&pod, "allocation") case role.direct: configureDirectHybridAgentRuntime(&pod, "config") default: - configureNodeLocalAgentRuntime(&pod, role.family) + configureAgentIdentity(&pod) } t.Log("Retain the sidecar and bind only the authored main runtime") @@ -118,14 +118,12 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { } else { require.Equal(t, lpuAgentContainerName, container.Name) } - if role.conductor || role.family == BuildFamilyHX && !role.direct { - require.Equal(t, authored.SecurityContext, container.SecurityContext) - } + require.Equal(t, authored.SecurityContext, container.SecurityContext) t.Log("Keep static environment values template-owned, including intentional omission") dynamicEnv := map[string]bool{ - "POD_IP": true, "LPX_ALLOCATION": role.conductor, - "TOPOLOGIES": role.direct, "GAS_DIR": role.direct, + "LPX_ALLOCATION": role.conductor, + "TOPOLOGIES": role.direct, "GAS_DIR": role.direct, } container.Env = slices.DeleteFunc(slices.Clone(container.Env), func(variable corev1.EnvVar) bool { return dynamicEnv[variable.Name] }) if role.conductor { diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index 229b5d0ce55c..e1b870e3f893 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -74,6 +74,10 @@ spec: value: '19877' - name: SERVER_HOSTS_FILE value: /tmp/lpu_servers + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] image: example.invalid/gpu-runtime:test name: main resources: @@ -127,7 +131,12 @@ spec: hostPath: path: /dev/infiniband type: Directory - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -203,21 +212,60 @@ spec: value: '19877' - name: READINESS_PORT value: '19878' + - name: LPU_MODEL_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] volumeMounts: - mountPath: /nfs name: model-storage - name: config mountPath: /configs - - {name: hugepages, mountPath: /dev/hugepages} - - {name: host-sys, mountPath: /sys} - - {name: host-dev, mountPath: /dev} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - name: hugepages + mountPath: /dev/hugepages + - name: host-sys + mountPath: /sys + - name: host-dev + mountPath: /dev + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + securityContext: + privileged: true + resources: + limits: + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists - name: prefill type: prefill replicas: 1 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 9ab5f3724735..f51d4026ba03 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -305,7 +305,9 @@ spec: - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME - value: default + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -313,7 +315,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -359,7 +360,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsUser: 0 startupProbe: failureThreshold: 60 httpGet: @@ -393,17 +393,10 @@ spec: securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -524,6 +517,10 @@ spec: value: prometheus - name: NIXL_TELEMETRY_PROMETHEUS_PORT value: "19090" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index 2183b8eddec9..e8e1d6c49191 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -40,6 +40,10 @@ spec: value: '19877' - name: SERVER_HOSTS_FILE value: /tmp/lpu_servers + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] startupProbe: failureThreshold: 720 httpGet: @@ -66,7 +70,8 @@ spec: mountPath: /configs - name: infiniband mountPath: /dev/infiniband - - {name: model-storage, mountPath: /nfs} + - name: model-storage + mountPath: /nfs resources: claims: - name: gpu @@ -75,7 +80,12 @@ spec: hostPath: path: /dev/infiniband type: Directory - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -89,6 +99,10 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi startupProbe: failureThreshold: 60 httpGet: @@ -118,20 +132,51 @@ spec: value: '19877' - name: READINESS_PORT value: '19878' + - name: LPU_MODEL_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] volumeMounts: - mountPath: /nfs name: model-storage - name: config mountPath: /configs - - {name: hugepages, mountPath: /dev/hugepages} - - {name: host-sys, mountPath: /sys} - - {name: host-dev, mountPath: /dev} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - name: hugepages + mountPath: /dev/hugepages + - name: host-sys + mountPath: /sys + - name: host-dev + mountPath: /dev + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + securityContext: + privileged: true volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 7e3ebff30622..01cfdeddc15d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -120,7 +120,9 @@ spec: - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME - value: default + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -128,7 +130,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -174,7 +175,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsUser: 0 startupProbe: failureThreshold: 60 httpGet: @@ -208,17 +208,10 @@ spec: securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -320,6 +313,10 @@ spec: value: prometheus - name: NIXL_TELEMETRY_PROMETHEUS_PORT value: "19090" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 5dc27dc9dc3f..32c4ebe67ecd 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -81,14 +81,13 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - securityContext: - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + - name: model-storage + mountPath: /nfs + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh initContainers: - args: - -m @@ -99,10 +98,6 @@ spec: - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key @@ -110,9 +105,18 @@ spec: name: ssh-secret readOnly: true volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 384 + - name: single-v2-ssh-key + emptyDir: {} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -126,6 +130,10 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi readinessProbe: failureThreshold: 3 initialDelaySeconds: 10 @@ -142,15 +150,42 @@ spec: name: model-storage - name: config mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - - {name: hugepages, mountPath: /dev/hugepages} + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + - name: hugepages + mountPath: /dev/hugepages + securityContext: + privileged: true volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index e28a92c3414e..bf77074bb257 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -121,7 +121,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -155,9 +154,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -186,10 +182,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -307,7 +299,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -334,10 +325,6 @@ spec: successThreshold: 1 timeoutSeconds: 1 resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -358,10 +345,6 @@ spec: image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index d19ffb8c9c52..e4ca8741d652 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -27,6 +27,10 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi command: - /bin/quasar-entrypoint args: @@ -42,23 +46,54 @@ spec: env: - name: LPU_RUN_SYSTEM_INIT value: 'true' + - name: LPU_MODEL_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] volumeMounts: - mountPath: /nfs name: model-storage - name: config mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - - {name: hugepages, mountPath: /dev/hugepages} + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + - name: hugepages + mountPath: /dev/hugepages + securityContext: + privileged: true volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists - name: lpu type: lpx replicas: 1 @@ -127,14 +162,13 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - securityContext: - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + - name: model-storage + mountPath: /nfs + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh initContainers: - args: - -m @@ -145,10 +179,6 @@ spec: - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key @@ -156,8 +186,17 @@ spec: name: ssh-secret readOnly: true volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 384 + - name: single-v2-ssh-key + emptyDir: {} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: *agent-template diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 7b45d83594f1..f98965de81c5 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -184,7 +184,9 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: LPU_MODEL_NAME - value: draft0 + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: LPU_RUN_SYSTEM_INIT value: "true" - name: NATS_SERVER @@ -192,7 +194,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -226,9 +227,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -257,10 +255,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -330,7 +324,9 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: LPU_MODEL_NAME - value: draft1 + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: LPU_RUN_SYSTEM_INIT value: "true" - name: NATS_SERVER @@ -338,7 +334,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -372,9 +367,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -403,10 +395,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -476,7 +464,9 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: LPU_MODEL_NAME - value: target + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: LPU_RUN_SYSTEM_INIT value: "true" - name: NATS_SERVER @@ -484,7 +474,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -518,9 +507,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -549,10 +535,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -669,7 +651,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -696,10 +677,6 @@ spec: successThreshold: 1 timeoutSeconds: 1 resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -720,10 +697,6 @@ spec: image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index 98ebaedff285..769722dfffaa 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -54,12 +54,23 @@ spec: periodSeconds: 5 timeoutSeconds: 4 volumeMounts: - - {name: model-storage, mountPath: /nfs} + - name: model-storage + mountPath: /nfs resources: claims: - name: gpu + env: + - name: CYBORG_FPGA_GPI_REPLICA_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['grove.io/podclique-pod-index'] volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -71,6 +82,10 @@ spec: resources: limits: lpu.nvidia.com/devices: '1' + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi command: - /bin/hydra-entrypoint startupProbe: @@ -102,20 +117,51 @@ spec: value: '19877' - name: READINESS_PORT value: '19878' + - name: LPU_MODEL_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] volumeMounts: - mountPath: /nfs name: model-storage - name: config mountPath: /configs - - {name: hugepages, mountPath: /dev/hugepages} - - {name: host-sys, mountPath: /sys} - - {name: host-dev, mountPath: /dev} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - name: hugepages + mountPath: /dev/hugepages + - name: host-sys + mountPath: /sys + - name: host-dev + mountPath: /dev + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + securityContext: + privileged: true volumes: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index c3b1119dd046..6f3684356ebb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -115,7 +115,9 @@ spec: - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME - value: default + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -123,7 +125,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -169,7 +170,6 @@ spec: nvidia.com/lpu: "16" securityContext: privileged: true - runAsUser: 0 startupProbe: exec: command: @@ -203,17 +203,10 @@ spec: securityContext: fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 terminationGracePeriodSeconds: 60 tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -311,6 +304,10 @@ spec: value: prometheus - name: NIXL_TELEMETRY_PROMETHEUS_PORT value: "19090" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index 586034a4bb74..744ae7737749 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -120,14 +120,13 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - securityContext: - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + - name: model-storage + mountPath: /nfs + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh initContainers: - args: - -m @@ -138,10 +137,6 @@ spec: - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key @@ -149,9 +144,18 @@ spec: name: ssh-secret readOnly: true volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 384 + - name: single-v2-ssh-key + emptyDir: {} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -199,17 +203,18 @@ spec: value: 'false' securityContext: privileged: true - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - mountPath: /nfs name: model-storage - name: config mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true initContainers: - command: - /bin/lp30-agent-v2-init @@ -222,9 +227,6 @@ spec: name: prepare-lp30 securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -234,6 +236,24 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 59875cc449ce..45ffc6897ba2 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -131,7 +131,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -164,9 +163,6 @@ spec: nvidia.com/lpu: "16" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -197,9 +193,6 @@ spec: resources: {} securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -214,10 +207,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -371,7 +360,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -398,10 +386,6 @@ spec: successThreshold: 1 timeoutSeconds: 1 resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -422,10 +406,6 @@ spec: image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 32d2f60004ae..6ff3c963efb5 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -63,19 +63,24 @@ spec: value: cvp - name: REDFISH_BMC_ENABLED value: 'false' + - name: LPU_MODEL_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] securityContext: privileged: true - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - name: model-storage mountPath: /nfs - name: config mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true initContainers: - command: - /bin/lp30-agent-v2-init @@ -88,9 +93,6 @@ spec: name: prepare-lp30 securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -100,9 +102,27 @@ spec: - name: model-storage persistentVolumeClaim: claimName: model-storage - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists - name: lpu type: lpx replicas: 1 @@ -211,14 +231,13 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - securityContext: - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + - name: model-storage + mountPath: /nfs + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh initContainers: - args: - -m @@ -229,10 +248,6 @@ spec: - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key @@ -240,8 +255,17 @@ spec: name: ssh-secret readOnly: true volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 384 + - name: single-v2-ssh-key + emptyDir: {} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: *agent-template diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 0f34c89fb63e..3983441dc247 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -183,13 +183,14 @@ spec: - name: LPU_FPGA_RECONFIGURE_METHOD value: cvp - name: LPU_MODEL_NAME - value: draft0 + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -222,9 +223,6 @@ spec: nvidia.com/lpu: "16" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -255,9 +253,6 @@ spec: resources: {} securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -272,10 +267,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -362,13 +353,14 @@ spec: - name: LPU_FPGA_RECONFIGURE_METHOD value: cvp - name: LPU_MODEL_NAME - value: draft1 + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -401,9 +393,6 @@ spec: nvidia.com/lpu: "16" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -434,9 +423,6 @@ spec: resources: {} securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -451,10 +437,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -541,13 +523,14 @@ spec: - name: LPU_FPGA_RECONFIGURE_METHOD value: cvp - name: LPU_MODEL_NAME - value: target + valueFrom: + fieldRef: + fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - name: NATS_SERVER value: nats-address - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -580,9 +563,6 @@ spec: nvidia.com/lpu: "16" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -613,9 +593,6 @@ spec: resources: {} securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /sys name: host-sys @@ -630,10 +607,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -787,7 +760,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -814,10 +786,6 @@ spec: successThreshold: 1 timeoutSeconds: 1 resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -838,10 +806,6 @@ spec: image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml index bef87e350737..6b7746e394d8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml @@ -97,14 +97,13 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 - securityContext: - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false volumeMounts: - - {name: model-storage, mountPath: /nfs} - - {name: config, mountPath: /configs} - - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + - name: model-storage + mountPath: /nfs + - name: config + mountPath: /configs + - name: single-v2-ssh-key + mountPath: /tmp/dynamo-lpu-ssh initContainers: - args: - -m @@ -115,10 +114,6 @@ spec: - /bin/install image: example.invalid/lpu-runtime:test name: prepare-ssh-key - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key @@ -126,9 +121,18 @@ spec: name: ssh-secret readOnly: true volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 384}} - - {name: single-v2-ssh-key, emptyDir: {}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 384 + - name: single-v2-ssh-key + emptyDir: {} + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch - name: agent podTemplate: spec: @@ -159,13 +163,48 @@ spec: name: model-storage - name: config mountPath: /configs - - {name: host-dev, mountPath: /dev} - - {name: host-sys, mountPath: /sys} - - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} - - {name: hugepages, mountPath: /dev/hugepages} + - name: host-dev + mountPath: /dev + - name: host-sys + mountPath: /sys + - name: ssh-secret + mountPath: /ssh-pk + readOnly: true + - name: hugepages + mountPath: /dev/hugepages + securityContext: + privileged: true + resources: + limits: + hugepages-2Mi: 4Gi + requests: + cpu: '62' + hugepages-2Mi: 4Gi volumes: - - {name: model-storage, persistentVolumeClaim: {claimName: model-storage}} - - {name: host-dev, hostPath: {path: /dev, type: Directory}} - - {name: host-sys, hostPath: {path: /sys, type: Directory}} - - {name: hugepages, emptyDir: {medium: HugePages}} - - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: host-sys + hostPath: + path: /sys + type: Directory + - name: hugepages + emptyDir: + medium: HugePages + - name: ssh-secret + secret: + secretName: ssh-secret + defaultMode: 420 + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + tolerations: + - key: lpu.nvidia.com/node + operator: Exists diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index a9e72c334a23..c0203b9387ae 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -201,7 +201,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -235,9 +234,6 @@ spec: lpu.nvidia.com/lpu: "8" securityContext: privileged: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -266,10 +262,6 @@ spec: tolerations: - key: lpu.nvidia.com/node operator: Exists - - key: lpu.nvidia.com/lpu - operator: Exists - - key: lpu.nvidia.com/node-v2 - operator: Exists volumes: - name: model-storage persistentVolumeClaim: @@ -392,7 +384,6 @@ spec: - name: POD_IP valueFrom: fieldRef: - apiVersion: v1 fieldPath: status.podIP - name: POD_NAME valueFrom: @@ -419,10 +410,6 @@ spec: successThreshold: 1 timeoutSeconds: 1 resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /nfs name: model-storage @@ -443,10 +430,6 @@ spec: image: example.invalid/lpu-runtime:test name: prepare-ssh-key resources: {} - securityContext: - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 volumeMounts: - mountPath: /tmp/dynamo-lpu-ssh name: single-v2-ssh-key From 37b36f1b1d1ff1db284de60dffdfd36cb59c0700 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:32:12 +0300 Subject: [PATCH 087/196] fix(operator): check LPX PCS ownership before scaling (#523) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7a3dfcf90e89567e8a0d4fd2a42b9c659b4bff11) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 3 +++ .../controller/lpx/controller_test.go | 21 +++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 8f98036f27b9..6af52ea2ebef 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -314,6 +314,9 @@ func (r *graphReconciler) reconcileWorkload( // PCS instead and must not try to mutate its old group. scaled := false if pcs != nil && pcs.DeletionTimestamp.IsZero() && pcs.Annotations[lpx.WorkloadDigestAnnotation] == selected.workloadDigest { + if err := commoncontroller.CheckControllerOwnership(pcs, deployment, r.Scheme()); err != nil { + return state, ctrl.Result{}, err + } scaled, err = r.reconcileLPXScalingGroupScale(ctx, source, pcs, selected.plan.LPXScalingGroup, true) if err != nil { return state, ctrl.Result{}, err diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 0a2c1782002a..1a15e299abbc 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -42,11 +42,15 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { name string replicas *int32 seed int32 + owner string err error wantUpdates int }{ {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, + {name: "foreign scale in", replicas: ptr.To(int32(2)), owner: "foreign"}, + {name: "ownerless scale in", replicas: ptr.To(int32(2)), owner: "absent"}, + {name: "wrong kind scale in", replicas: ptr.To(int32(2)), owner: "wrong kind"}, {name: "unchanged", replicas: ptr.To(int32(9))}, {name: "omitted scale out", seed: 3}, {name: "omitted scale in", seed: 12}, @@ -59,6 +63,14 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) objects := lpxMaterializedObjects(t, r, child, source, selected) pcs := findLPXTestPodCliqueSet(t, objects) + switch tc.owner { + case "foreign": + pcs.OwnerReferences[0].UID = "foreign-owner" + case "absent": + pcs.OwnerReferences = nil + case "wrong kind": + pcs.OwnerReferences[0].Kind = consts.ResourceTypeDynamoGraphDeployment + } if tc.replicas == nil { pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas = ptr.To(tc.seed) } @@ -99,11 +111,16 @@ func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { t.Log("Only explicit changes write scale, preserving the observed resource-version precondition") _, _, err = r.reconcileWorkload(t.Context(), child, source, selected, pcs) - require.ErrorIs(t, err, tc.err) + if tc.owner != "" { + var ownershipConflict *commoncontroller.OwnershipConflictError + require.ErrorAs(t, err, &ownershipConflict) + } else { + require.ErrorIs(t, err, tc.err) + } require.Equal(t, tc.wantUpdates, updates) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) want := int32(9) - if tc.replicas != nil && tc.err == nil { + if tc.replicas != nil && err == nil { want = *tc.replicas } require.Equal(t, want, group.Spec.Replicas) From 99a4a518680426e05c613afe58012117c64ba0a6 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:32:13 +0300 Subject: [PATCH 088/196] fix(operator): download uncached LPX successor builds (#524) Signed-off-by: Andrew Paprotsky (cherry picked from commit 39e2f0316b4931d4eeaf8536bafbe85004adf6ca) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 3 +- .../internal/controller/lpx/lifecycle_test.go | 48 ++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 6af52ea2ebef..29a4e159ec9f 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -191,7 +191,8 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } var classification lpxClassification selected, classification, err = r.reconcileLPXSafetyPreflight(ctx, deployment, source, pcs, nil) - if err != nil { + // An uncached successor must reach the download gate before snapshot resolution can succeed. + if err != nil && !errors.Is(err, lpx.ErrBuildSnapshotAcquisition) { return ctrl.Result{}, err } if classification != nil { diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 8089376e6074..e124c9366296 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -88,8 +88,9 @@ func (r *downloadOrderedLPXRegistry) BuildURL(string) (*url.URL, error) { } func (r *downloadOrderedLPXRegistry) EnsureDownloaded(context.Context, url.URL) (bool, error) { + firstDownload := !slices.Contains(r.calls, "download") r.calls = append(r.calls, "download") - if len(r.calls) == 1 { + if firstDownload { return false, nil } r.downloaded = true @@ -215,6 +216,51 @@ func TestSelectedLPXColdCacheDownloadsBeforeSnapshot(t *testing.T) { require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) } +func TestSelectedLPXSuccessorBuildDownloadsBeforePublication(t *testing.T) { + t.Log("Publish the previous build before switching to an uncached successor") + ctx := t.Context() + child, source, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) + r, selected := newPreparedLPXTestReconciler(t, baseRegistry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + previous := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + const nextBuild = "next-build" + registry := &downloadOrderedLPXRegistry{ + ModelRegistry: newLPXTestRegistryWithPartitionsAndMode(t, nextBuild, []int{7, 8}, manifestcapnpv2.CompilationMode_lpuOnly), + buildURL: url.URL{Scheme: lpx.BuildSchemeGCS, Host: "test-bucket", Path: "/" + nextBuild}, + } + r.modelRegistry = registry + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) + lpx.ServingComponent(source).LPX.BuildID = nextBuild + source.Generation++ + require.NoError(t, r.Update(ctx, source)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + var err error + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + require.NoError(t, r.Update(ctx, child)) + + t.Log("Start downloading despite snapshot unavailability, preserving the existing request") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.Equal(t, modelDownloadRequeueAfter, result.RequeueAfter) + require.Equal(t, []string{"snapshot", "download"}, registry.calls) + require.Equal(t, previous, getLPXRequest(t, ctx, r.Client, previous.Namespace, previous.Name)) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.Equal(t, child.Generation, child.Status.ObservedGeneration) + + t.Log("Resolve the successor after download completes on the next reconciliation") + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Equal(t, []string{"snapshot", "download", "snapshot", "download", "snapshot"}, registry.calls) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) + require.NotNil(t, child.Status.ModelDownload.LastCheckedAt) +} + func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection(t *testing.T) { ctx := t.Context() dgd, source, registry := newLPXSpecDecodeTestDGD(t) From 219afcc622725eac758e7525db477fc346600c8f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:32:13 +0300 Subject: [PATCH 089/196] fix(operator): retain exact LPX deadline cleanup requests (#526) Signed-off-by: Andrew Paprotsky (cherry picked from commit dd19a16c35e2afa53baa6bad45984919a9a88dd2) Signed-off-by: Andrew Paprotsky --- .../api/v1alpha1/lpxgraphdeployment_types.go | 6 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 6 ++ .../bases/nvidia.com_lpxgraphdeployments.yaml | 12 +++ .../controller/lpx/attempt_deadline.go | 82 ++++++++----------- .../controller/lpx/attempt_deadline_test.go | 67 ++++++++------- .../internal/controller/lpx/controller.go | 11 ++- .../controller/lpx/controller_test.go | 73 +++++++++++++++-- .../controller/lpx/deadline_retirement.go | 6 +- .../lpx/deadline_retirement_test.go | 8 ++ 9 files changed, 184 insertions(+), 87 deletions(-) diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index e89e89ac4fb4..568f53c4c399 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -6,6 +6,7 @@ package v1alpha1 import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // LPXGraphDeploymentSpec is the operator's handoff for all LPX components in a DGD. @@ -30,6 +31,11 @@ type LPXGraphDeploymentStatus struct { // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:"conditions,omitempty"` + // expiredRequestUIDs records requests selected for deadline cleanup until they disappear, + // even if their scheduler phase or the deployment input changes. + // +optional + // +listType=set + ExpiredRequestUIDs []types.UID `json:"expiredRequestUIDs,omitempty"` // components reports logical replicas by authored component name, never Agent Pods. // +optional Components map[string]v1beta1.ComponentReplicaStatus `json:"components,omitempty"` diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 22a4e1faa4cc..69688a5033ae 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -44,6 +44,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" configv1alpha1 "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1" ) @@ -1452,6 +1453,11 @@ func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ExpiredRequestUIDs != nil { + in, out := &in.ExpiredRequestUIDs, &out.ExpiredRequestUIDs + *out = make([]types.UID, len(*in)) + copy(*out, *in) + } if in.Components != nil { in, out := &in.Components, &out.Components *out = make(map[string]v1beta1.ComponentReplicaStatus, len(*in)) diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index d2133af7a132..73553a0dfefb 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -234,6 +234,18 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + expiredRequestUIDs: + description: |- + expiredRequestUIDs records requests selected for deadline cleanup until they disappear, + even if their scheduler phase or the deployment input changes. + items: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + type: array + x-kubernetes-list-type: set modelDownload: description: modelDownload retains the existing remote-build download progress. properties: diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index 2139939c106c..09e24b667f97 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -13,6 +13,7 @@ import ( grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -64,8 +65,8 @@ func completeLPXDeadline( // reconcileLPXRequestDeadlines derives every scheduling clock from the start // of the corresponding desired LPR's current scheduler-owned scheduling cycle. -// pcs is nil when no current PodCliqueSet was observed, which leaves no active -// clock. +// Recorded expiry survives phase/input changes until the exact request disappears. +// A nil PCS leaves no active clock; already recorded cleanup waits for owner GC. func (r *graphReconciler) reconcileLPXRequestDeadlines( ctx context.Context, deployment *nvidiacomv1alpha1.LPXGraphDeployment, @@ -74,7 +75,9 @@ func (r *graphReconciler) reconcileLPXRequestDeadlines( desired []lpxModelMaterializing, ) (lpxClassification, time.Time, error) { deadlineSeconds := lpxRequestDeadlineSeconds(source) - if deadlineSeconds == nil || pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) { + active := deadlineSeconds != nil && pcs != nil && pcs.DeletionTimestamp.IsZero() && metav1.IsControlledBy(pcs, deployment) + recorded := sets.New(deployment.Status.ExpiredRequestUIDs...) + if !active && recorded.Len() == 0 { return nil, time.Time{}, nil } requests, err := r.listOwnedLPXRequests(ctx, deployment, pcs) @@ -88,11 +91,22 @@ func (r *graphReconciler) reconcileLPXRequestDeadlines( // Expire each desired nonterminal request independently and retain the earliest future wake. now := time.Now() - duration := time.Duration(*deadlineSeconds) * time.Second expired := make([]*lpxv1alpha1.LPUPipelineRequest, 0) + deployment.Status.ExpiredRequestUIDs = nil + recordFailure := false var nextDeadline time.Time for index := range requests { request := &requests[index] + if recorded.Has(request.UID) { + deployment.Status.ExpiredRequestUIDs = append(deployment.Status.ExpiredRequestUIDs, request.UID) + if request.DeletionTimestamp.IsZero() { + expired = append(expired, request) + } + continue + } + if !active { + continue + } if _, current := desiredNames[request.Name]; !current { continue } @@ -103,27 +117,29 @@ func (r *graphReconciler) reconcileLPXRequestDeadlines( if !known { continue } - deadline := startedAt.Add(duration) + deadline := startedAt.Add(time.Duration(*deadlineSeconds) * time.Second) if !now.Before(deadline) { - expired = append(expired, request) + deployment.Status.ExpiredRequestUIDs = append(deployment.Status.ExpiredRequestUIDs, request.UID) + recordFailure = true continue } if nextDeadline.IsZero() || deadline.Before(nextDeadline) { nextDeadline = deadline } } - if len(expired) == 0 { - return nil, nextDeadline, nil - } - // Persist retry authorization before cleanup removes the request that proves expiry. - if !lpxDeadlineFailureCovers(deployment, expired) { + // Persist exact request identities and retry authorization before cleanup. + if recordFailure { return &lpxDeadlineExceeded{requeueAfter: time.Nanosecond, recordFailure: true}, time.Time{}, nil } + if recorded.Len() == 0 { + return nil, nextDeadline, nil + } // Lower Grove before deleting the affected engine suffix; convergence is asynchronous. + // Requeue even after the last recorded UID disappears so an authorized retry can proceed. retirementErr := r.retireExpiredLPXRequests(ctx, deployment, source, pcs, requests, expired) - return &lpxDeadlineExceeded{requeueAfter: lpxRetirementRequeueAfter}, time.Time{}, retirementErr + return &lpxDeadlineExceeded{requeueAfter: lpxRetirementRequeueAfter}, nextDeadline, retirementErr } func lpxRequestHasTerminalSchedulingDisposition(request *lpxv1alpha1.LPUPipelineRequest) bool { @@ -159,43 +175,13 @@ func lpxRequestDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) * return source.Spec.Scheduling.AttemptDeadlineSeconds } -// lpxDeadlineFailureCurrent reports whether the current generation must remain -// failed until a later input revision explicitly authorizes another publication. +// lpxDeadlineFailureCurrent fences publication until expired requests disappear +// and a later input revision explicitly authorizes a retry. func lpxDeadlineFailureCurrent(deployment *nvidiacomv1alpha1.LPXGraphDeployment) bool { failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) - return lpxDeadlineFailureRecorded(deployment) && - failed.ObservedGeneration >= deployment.Generation -} - -func lpxDeadlineFailureRecorded(deployment *nvidiacomv1alpha1.LPXGraphDeployment) bool { - failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) - return failed != nil && failed.Status == metav1.ConditionTrue && failed.Reason == lpxSchedulingDeadlineExceededReason -} - -// lpxDeadlineFailureCovers reports whether every expired scheduling cycle -// predates the recorded failure. A later cycle needs its own durable failure -// before cleanup. -func lpxDeadlineFailureCovers( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - requests []*lpxv1alpha1.LPUPipelineRequest, -) bool { - failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) - if failed == nil || failed.Status != metav1.ConditionTrue || failed.Reason != lpxSchedulingDeadlineExceededReason { - return false - } - if failed.ObservedGeneration < deployment.Generation { - return false - } - if failed.LastTransitionTime.IsZero() { - return false - } - for _, request := range requests { - startedAt, known := lpxSchedulingStartedAt(request) - if !known || !startedAt.Before(failed.LastTransitionTime.Time) { - return false - } - } - return true + return len(deployment.Status.ExpiredRequestUIDs) > 0 || (failed != nil && + failed.Status == metav1.ConditionTrue && failed.Reason == lpxSchedulingDeadlineExceededReason && + failed.ObservedGeneration >= deployment.Generation) } // projectLPXSchedulingFailureCondition keeps retry authorization independent @@ -224,7 +210,7 @@ func projectLPXSchedulingFailureCondition( if prior == nil { return } - if deployment.Generation <= prior.ObservedGeneration { + if deployment.Generation <= prior.ObservedGeneration || len(deployment.Status.ExpiredRequestUIDs) > 0 { return } meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 6bcf1fd12cda..39deb9424f04 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -42,6 +42,16 @@ func TestLPXDeadlineUsesEachSchedulingCycleStart(t *testing.T) { require.NoError(t, err) require.Nil(t, classification) require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) + + t.Log("Keep that deadline wake while a recorded expiry waits for its finalizer") + second.Finalizers = []string{"scheduler.example/cleanup"} + require.NoError(t, r.Update(t.Context(), second)) + require.NoError(t, r.Delete(t.Context(), second)) + dgd.Status.ExpiredRequestUIDs = []types.UID{second.UID} + classification, wake, err = r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(first, second)) + require.NoError(t, err) + require.IsType(t, &lpxDeadlineExceeded{}, classification) + require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) } func TestLPXDeadlineIgnoresRequestsOutsideCurrentPublication(t *testing.T) { @@ -203,35 +213,28 @@ func TestLPXDeadlineFailureRequiresPostFailureGeneration(t *testing.T) { require.False(t, lpxDeadlineFailureCurrent(dgd)) } -func TestLPXDeadlineFailureOnlyCoversPreviouslyPublishedRequests(t *testing.T) { - t.Log("Record a deadline failure for the current deployment generation") - dgd, _, _ := newLPXTestDGD(t, "single") - failureTime := time.Now().Add(-time.Minute) - dgd.Status.Conditions = []metav1.Condition{{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, - Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, - LastTransitionTime: metav1.NewTime(failureTime), - }} - - t.Log("The failure covers cleanup of a scheduling cycle that started before it") - pcs := deadlineTestPCS(dgd, "pcs-uid") - oldRequest := deadlineTestRequest(dgd, pcs, "old", failureTime.Add(-time.Minute), lpxv1alpha1.RequestPhasePending) - require.True(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{oldRequest})) - - t.Log("A failure from an earlier deployment generation cannot cover current cleanup") - dgd.Generation++ - require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{oldRequest})) - dgd.Generation-- - - t.Log("A later scheduling cycle on the same request requires a new durable failure") - restartedRequest := deadlineTestRequest(dgd, pcs, "restarted", failureTime.Add(-time.Hour), lpxv1alpha1.RequestPhasePending) - restartedRequest.Status.LastPlanRevision = 1 - restartedRequest.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(failureTime.Add(time.Second))) - require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{restartedRequest})) - - t.Log("An unknown cycle start is never covered by an older failure") - restartedRequest.Status.SchedulingStartedAt = nil - require.False(t, lpxDeadlineFailureCovers(dgd, []*lpxv1alpha1.LPUPipelineRequest{restartedRequest})) +func TestLPXDeadlineRetainsOnlyRecordedRequestUIDs(t *testing.T) { + for _, uid := range []types.UID{"recorded", "replacement"} { + t.Run(string(uid), func(t *testing.T) { + t.Log("Retain exact expiry identity even without a current deadline, desired request or PCS") + dgd, source, _ := newLPXTestDGD(t, "single") + dgd.Status.ExpiredRequestUIDs = []types.UID{"recorded"} + pcs := deadlineTestPCS(dgd, "pcs-uid") + request := deadlineTestRequest(dgd, pcs, "same-name", time.Now(), lpxv1alpha1.RequestPhaseBound) + request.UID = uid + r := newLPXTestReconciler(t, nil, dgd, source, request) + request = getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name) + classification, _, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, nil, nil) + require.NoError(t, err) + require.Positive(t, projectLPXLifecycleStatus(classification).RequeueAfter) + if uid == "recorded" { + require.Equal(t, []types.UID{uid}, dgd.Status.ExpiredRequestUIDs) + } else { + require.Nil(t, dgd.Status.ExpiredRequestUIDs) + } + require.Equal(t, request, getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name)) + }) + } } func TestProjectLPXDeadlineFailureAdvancesAStaleFailureFence(t *testing.T) { @@ -274,6 +277,12 @@ func TestProjectLPXDeadlineFailureRemainsStickyAtRecordedGeneration(t *testing.T require.NotNil(t, failed) require.Equal(t, metav1.ConditionTrue, failed.Status) require.Equal(t, lpxSchedulingDeadlineExceededReason, failed.Reason) + + t.Log("A later shape edit cannot clear retry authorization while expired requests survive") + dgd.Generation++ + dgd.Status.ExpiredRequestUIDs = []types.UID{"expired"} + projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{Reason: lpxRetiringReason}, false) + require.Equal(t, previous.Conditions[0], *meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition)) } func TestCompleteLPXDeadlineBoundsTransientErrors(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 29a4e159ec9f..8944452b86e8 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -174,6 +174,7 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } pcs, err = r.observeCurrentLPXPodCliqueSet(ctx, deployment) if err != nil { + pcs = nil return ctrl.Result{}, err } // Invalid source authority retires existing LPX state before reporting rejection. @@ -246,10 +247,16 @@ func (r *graphReconciler) completeReconcile( deadlineExceeded := false recordDeadlineFailure := false requestScopedRetirement := state.Reason == lpxRetiringReason && state.retirementScope == lpxRequestRetirement - if completion.unavailableReason == "" && completion.selected != nil && + if completion.unavailableReason == "" && completion.source != nil && + (completion.selected != nil || len(deployment.Status.ExpiredRequestUIDs) > 0) && (state.Reason != lpxRetiringReason || requestScopedRetirement) { + // Recorded cleanup survives errors preparing the next workload; only new clocks need selection. + var desired []lpxModelMaterializing + if completion.selected != nil { + desired = completion.selected.requests + } classification, wake, deadlineErr := r.reconcileLPXRequestDeadlines( - ctx, deployment, completion.source, completion.pcs, completion.selected.requests, + ctx, deployment, completion.source, completion.pcs, desired, ) deadlineAt, err = wake, errors.Join(err, deadlineErr) if classification != nil { diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 1a15e299abbc..e41d6622ed24 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -428,7 +428,8 @@ func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { } func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { - t.Run("deadline", func(t *testing.T) { + for _, phase := range []lpxv1alpha1.RequestPhase{lpxv1alpha1.RequestPhasePending, lpxv1alpha1.RequestPhaseBound} { + t.Logf("Scheduler phase after recorded expiry: %s", phase) t.Log("Observe a completed engine and a newer pending request in their shared PCS") ctx := t.Context() child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -440,7 +441,7 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { publishSelectedLPXForTest(t, ctx, r, child, selected) pcs := findLPXTestPodCliqueSet(t, objects) serving := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound).Status require.NoError(t, r.Update(ctx, serving)) pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) pending.Status = deadlineTestRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status @@ -450,12 +451,57 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { t.Log("Persist failure before changing Grove or deleting scheduler intent") request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) + persistErr := errors.New("status unavailable") + original := r.Client + r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if subresource == "status" { + return persistErr + } + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + }) _, err := r.Reconcile(ctx, request) + require.ErrorIs(t, err, persistErr) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Empty(t, child.Status.ExpiredRequestUIDs) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + require.True(t, getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name).DeletionTimestamp.IsZero()) + r.Client = original + _, err = r.Reconcile(ctx, request) require.NoError(t, err) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(2), group.Spec.Replicas) pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) require.True(t, pending.DeletionTimestamp.IsZero()) + if phase == lpxv1alpha1.RequestPhaseBound { + deleteErr := apierrors.NewConflict(lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource(), pending.Name, errors.New("scheduler updated status")) + r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { return deleteErr }, + }) + _, err = r.Reconcile(ctx, request) + require.ErrorIs(t, err, deleteErr) + r.Client = original + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Len(t, child.Status.ExpiredRequestUIDs, 1) + require.Equal(t, pending.UID, child.Status.ExpiredRequestUIDs[0]) + } + pending.Status.Phase = phase + require.NoError(t, r.Update(ctx, pending)) + conductorReplicas := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas + if phase == lpxv1alpha1.RequestPhaseBound { + t.Log("An edit after recorded expiry cannot rescue the old request") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) + source.Spec.Scheduling = nil + lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) + require.NoError(t, r.Update(ctx, source)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + } t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") for range 3 { @@ -470,14 +516,25 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) require.False(t, pending.DeletionTimestamp.IsZero()) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Generation++ - require.NoError(t, r.Update(ctx, child)) + if phase == lpxv1alpha1.RequestPhasePending { + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + } _, err = r.Reconcile(ctx, request) require.NoError(t, err) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(1), group.Spec.Replicas) t.Log("After cleanup the later edit can retry without replacing the serving engine") + if phase == lpxv1alpha1.RequestPhaseBound { + lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = conductorReplicas + require.NoError(t, r.Update(ctx, source)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + } retiredUID := pending.UID pending.Finalizers = nil require.NoError(t, r.Update(ctx, pending)) @@ -491,7 +548,7 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(2), group.Spec.Replicas) require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") - }) + } } func TestLPXDeadlineContinuesDuringRequestRetirement(t *testing.T) { @@ -526,8 +583,10 @@ func TestLPXDeadlineContinuesDuringRequestRetirement(t *testing.T) { prefix = getLPXRequest(t, ctx, r.Client, child.Namespace, prefix.Name) prefix.Status = deadlineTestRequest(child, pcs, prefix.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status require.NoError(t, r.Update(ctx, prefix)) - _, err := r.Reconcile(ctx, reconcileRequest) - require.NoError(t, err) + for range 2 { + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + } requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, prefix.Name) } diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement.go b/deploy/operator/internal/controller/lpx/deadline_retirement.go index 0d3bd5807e4c..bce09d323a7f 100644 --- a/deploy/operator/internal/controller/lpx/deadline_retirement.go +++ b/deploy/operator/internal/controller/lpx/deadline_retirement.go @@ -81,7 +81,11 @@ func (r *graphReconciler) retireExpiredLPXRequests( } // Omitted replicas leave capacity externally managed; expire only scheduler // intent so a later input revision can reuse the unchanged Grove ordinal. - if lpx.ServingComponent(source).Replicas != nil && group.Spec.Replicas > replicas { + component := lpx.ServingComponent(source) + if component == nil { + return fmt.Errorf("cannot determine replica ownership without an LPX serving component") + } + if component.Replicas != nil && group.Spec.Replicas > replicas { scale := &autoscalingv1.Scale{ ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go index 034200b79e64..b50f20ff3f27 100644 --- a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go +++ b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go @@ -39,6 +39,14 @@ func TestLPXExpiredScaleOutPreservesServingEngine(t *testing.T) { requests, err := r.listOwnedLPXRequests(ctx, child, pcs) require.NoError(t, err) + t.Log("Reject ambiguous replica ownership without changing the live engine") + invalid := source.DeepCopy() + invalid.Spec.Components = nil + require.ErrorContains(t, r.retireExpiredLPXRequests(ctx, child, invalid, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}), "without an LPX serving component") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + t.Log("Lower Grove and delete only the expired scheduler intent in one reconciliation") err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) require.NoError(t, err) From 9d8403c0929b6dae372220a750c4b751ca60eb67 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:32:13 +0300 Subject: [PATCH 090/196] fix(operator): check missing LPX PCS request owner name (#527) Signed-off-by: Andrew Paprotsky (cherry picked from commit 5099c1e1e22a9430835910ed25f3a5a319b42339) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/lifecycle.go | 6 ++---- .../internal/controller/lpx/lifecycle_test.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go index 15d5e250f981..c95f705aff22 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ b/deploy/operator/internal/controller/lpx/lifecycle.go @@ -595,10 +595,8 @@ func ownsLPXRequest( return metav1.IsControlledBy(pcs, deployment) && metav1.IsControlledBy(request, pcs) } - // The deployment-UID label keeps owner-GC observation level-driven after the - // PCS disappears. The request's controller kind still prevents adopting an - // unrelated labeled object while the exact UID remains in its owner reference. - return request.Labels[lpxOwnerUIDLabel] == string(deployment.UID) + // Keep observing requests from the missing PCS, not another owner with a copied label. + return owner.Name == dynamo.PCSNameForLPX(deployment) && request.Labels[lpxOwnerUIDLabel] == string(deployment.UID) } // listOwnedLPXRequests filters the deployment-scoped list against the caller's diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index e124c9366296..1b135070cb5d 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -708,6 +708,15 @@ func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodClique pcs := &grovev1alpha1.PodCliqueSet{} require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) + t.Log("An unrelated live PCS also has a request with this deployment's copied UID label") + foreignPCS := pcs.DeepCopy() + foreignPCS.Name, foreignPCS.UID, foreignPCS.ResourceVersion = lpxTestOtherName, "foreign-pcs", "" + foreignPCS.OwnerReferences = nil + require.NoError(t, reconciler.Create(ctx, foreignPCS)) + foreign := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) + foreign.Name, foreign.UID, foreign.ResourceVersion = "foreign-request", "foreign-request-uid", "" + foreign.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(foreignPCS, grovev1alpha1.SchemeGroupVersion.WithKind(lpxPodCliqueSetKind))} + require.NoError(t, reconciler.Create(ctx, foreign)) require.NoError(t, reconciler.Delete(ctx, pcs)) t.Log("Keep the replacement PCS fenced while its predecessor's request remains persisted") @@ -723,6 +732,8 @@ func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodClique require.NoError(t, err) require.Empty(t, currents) require.Nil(t, classification) + require.Equal(t, foreign, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, foreign.Name)) + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(foreignPCS), &grovev1alpha1.PodCliqueSet{})) } func TestLPXScaleDownDeletesOnlyStaleRequests(t *testing.T) { @@ -1527,8 +1538,8 @@ func TestLPXPublicationFenceRejectsForeignExactNameCollisionFromAuthoritativeLis UID: "foreign-collision-uid", Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: nvidiacomv1beta1.GroupVersion.String(), - Kind: nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind, + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), + Kind: lpxPodCliqueSetKind, Name: lpxTestOtherName, UID: "other-uid", Controller: ptr.To(true), @@ -1537,6 +1548,8 @@ func TestLPXPublicationFenceRejectsForeignExactNameCollisionFromAuthoritativeLis } second := foreign.DeepCopy() second.Name, second.UID = "a-foreign-request", "second-foreign-collision-uid" + second.OwnerReferences[0].APIVersion = nvidiacomv1beta1.GroupVersion.String() + second.OwnerReferences[0].Kind = nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind desired := &lpxMaterializing{requests: []lpxModelMaterializing{ {requestName: foreign.Name}, {requestName: second.Name}, }} From 9bd7d9a1fa167df81f373d9ccfd0306933c873a6 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:37:37 +0300 Subject: [PATCH 091/196] fix(operator): bound LPX engine replicas (#528) Signed-off-by: Andrew Paprotsky (cherry picked from commit 60e6f6d0085a72ed8a7cd79d1d4a6c40e926526a) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/lifecycle_test.go | 28 +++++++++++++++++++ .../internal/dynamo/lpx/cyborg_render_test.go | 7 ++--- .../internal/dynamo/lpx/materialization.go | 7 +++-- .../dynamo/lpx/materialization_test.go | 11 ++++++-- .../dynamo-component-deployment.mdx | 2 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 1b135070cb5d..ff7b8308f525 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -323,6 +323,34 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( require.Contains(t, lpxResult(classification).Message, "TargetUnsupported") } +func TestLPXMaterializationRejectsExcessiveReplicas(t *testing.T) { + for _, nativeScale := range []bool{false, true} { + t.Run(fmt.Sprintf("nativeScale=%t", nativeScale), func(t *testing.T) { + t.Log("Prepare a valid workload and its owned Grove scaling group") + ctx := t.Context() + child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) + objects := lpxMaterializedObjects(t, r, child, source, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) + + t.Log("Exceed the engine limit through either the DGD or native Grove scale") + lpx.ServingComponent(source).Replicas = ptr.To(int32(2497)) + if nativeScale { + lpx.ServingComponent(source).Replicas = nil + group.Spec.Replicas = 2497 + } + createLPXTestObjects(t, ctx, r.Client, pcs, group) + + t.Log("Reject the count before constructing per-engine request state") + selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) + require.NoError(t, err) + require.Nil(t, selected) + require.Equal(t, &lpxRejected{reason: "LPX replica count must be between 0 and 2496"}, classification) + }) + } +} + func TestLPXImplicitReplicaObservationWaitsForMatchingPodCliqueSet(t *testing.T) { t.Log("Render the stable Grove identities for an externally managed replica count") ctx := t.Context() diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 45ba6ad2e7be..5e8011015737 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -7,7 +7,6 @@ package lpx import ( "encoding/json" - "math" "strings" "testing" @@ -32,7 +31,7 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { projection.configuredBuild.IOFanoutFactor = 1 workload := &SelectedWorkload{ modelProjections: []*ModelProjection{projection}, - scalingGroupReplicas: math.MaxInt32, + scalingGroupReplicas: 2496, } pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) plan, err := workload.PlanNodeLocalMaterialization(pcsName) @@ -45,8 +44,8 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { wantError bool }{ {name: "one GPU per engine", replicas: 1}, - {name: "hostname at DNS limit", replicas: 100}, - {name: "hostname over DNS limit", replicas: 101, wantError: true}, + {name: "hostname at DNS limit", replicas: 100_000_000}, + {name: "hostname over DNS limit", replicas: 100_000_001, wantError: true}, } { t.Run(test.name, func(t *testing.T) { t.Log("Render the GPU width beneath the same readable PCS name") diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 808db9ec90bd..3646274e0431 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -18,6 +18,7 @@ import ( const ( lpxScalingGroupTemplateName = "lpx" conductorTemplateName = "cond" + maxEngineReplicas = 2496 ) // MaxPodCliqueSetNameLength reserves Grove's combined name budget for the fixed @@ -101,10 +102,10 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi return plan, plan.ValidateReplicaCount() } -// ValidateReplicaCount checks conductor and Agent hostnames. +// ValidateReplicaCount bounds per-engine allocations and checks Pod hostnames. func (p *MaterializationPlan) ValidateReplicaCount() error { - if p.Replicas < 0 { - return fmt.Errorf("LPX replica count must not be negative") + if p.Replicas < 0 || p.Replicas > maxEngineReplicas { + return fmt.Errorf("LPX replica count must be between 0 and %d", maxEngineReplicas) } if p.ConductorTemplate != "" { if err := p.validatePodHostname("conductor", p.ConductorTemplate, 0); err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index acc913a7b1bb..2ef60466846e 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -17,7 +17,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) -func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { +func TestPlanMaterializationBounds(t *testing.T) { t.Parallel() t.Log("Project hybrid and LPU-only materialization fixtures") @@ -51,7 +51,7 @@ func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { } workload := &SelectedWorkload{ modelProjections: projections, - scalingGroupReplicas: math.MaxInt32, + scalingGroupReplicas: 2496, } pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) plan, err := workload.PlanNodeLocalMaterialization(pcsName) @@ -90,6 +90,13 @@ func TestPlanMaterializationBoundsGeneratedPodHostnames(t *testing.T) { t.Log("Reject a PCS name one character beyond Grove's combined name budget") _, err = workload.PlanNodeLocalMaterialization(pcsName + "a") require.ErrorContains(t, err, "exceeds the LPX maximum of 38 characters") + + t.Log("Bound engine replicas before allocating per-engine request state") + for _, replicas := range []int32{-1, 0, 1, 2496, 2497, math.MaxInt32} { + workload.scalingGroupReplicas = replicas + _, err := workload.PlanNodeLocalMaterialization(pcsName) + require.Equal(t, replicas < 0 || replicas > 2496, err != nil, "replicas=%d: %v", replicas, err) + } }) } } diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index c4000dc03950..577d7e82d7b6 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -67,7 +67,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Desired number of complete component instances. For a single-node component, each instance is one pod. For a multinode component, each instance contains all of its roles. Minimum `0`. When `scalingAdapter` is set, this field is owned by the [DynamoGraphDeploymentScalingAdapter](full-api-reference.mdx#dynamographdeploymentscalingadapter) and should not be modified directly. - For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid and LPU-only engines allow up to `2496` replicas within the 512 KiB scheduling-status budget; generated Pod hostname limits may lower that maximum. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. + For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid and LPU-only engines allow up to `2496` replicas; generated Pod hostname limits may lower that maximum. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. From ba4240bc4eab072f271fe6f8d095930c6ba17973 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:37:37 +0300 Subject: [PATCH 092/196] docs(operator): correct LPX security context ownership (#529) Signed-off-by: Andrew Paprotsky (cherry picked from commit 46921fe42060872457d9f1a4cace125dfa524821) Signed-off-by: Andrew Paprotsky --- docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md | 2 +- .../reference/kubernetes-api/dynamo-component-deployment.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index efb1e782ad9c..1c0549d329ea 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -281,7 +281,7 @@ spec: image: ``` -For LP20 and LP30, supply compatible builds and images, static arguments and environment settings, init containers, health probes, volumes, and mounts in the role templates. The operator uses the image's entrypoint unless the template supplies `command`; it does not generate runtime launchers, SSH-key preparation, or probes. Hardware policy still replaces the main-container `securityContext` for XT agents and direct-hybrid agents with privileged root settings, discarding other authored container security fields. Direct-hybrid Pods also force root user and group settings. +For LP20 and LP30, supply compatible builds and images, static arguments and environment settings, init containers, health probes, volumes, and mounts in the role templates. The operator uses the image's entrypoint unless the template supplies `command`; it does not generate runtime launchers, SSH-key preparation, or probes. Supply the Pod and container security contexts required for device access in the role templates; the operator preserves these settings. The operator supplies generated configuration, model and hardware bindings, and the LPU conductor's `LPX_ALLOCATION` value. A Nova conductor must include `--allocation` and `$(LPX_ALLOCATION)` in its authored arguments, along with its other runtime options. Mount the generated `config` volume and model storage where the selected runtime expects them. SSH-based startup must declare its own Secret volume, key preparation, and mounts. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 577d7e82d7b6..4f35a4175f50 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -106,7 +106,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Native Pod template for this role. LPX-only; required for every declared role. Each template requires a `main` container with an explicit image. Agent and conductor templates are independent. - For LP20 and LP30, the template owns runtime commands, static arguments and environment settings, init containers, health probes, and startup storage. It may use the image's entrypoint or supply `command`. The operator retains device-access security requirements and supplies generated configuration, model and hardware bindings, and `LPX_ALLOCATION` for the LPU conductor. A Nova template consumes that value through its authored `--allocation` argument. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for security-context overrides, role structure, and runtime examples. + For LP20 and LP30, the template owns runtime commands, static arguments and environment settings, init containers, health probes, startup storage, and Pod and container security contexts. It may use the image's entrypoint or supply `command`. The operator supplies generated configuration, model and hardware bindings, and `LPX_ALLOCATION` for the LPU conductor. A Nova template consumes that value through its authored `--allocation` argument. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for role structure and runtime examples. core/v1.PodTemplateSpec From 31b677c80e78d85063eb95254d041fda24c7ad76 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 18 Sep 2026 07:30:31 +0000 Subject: [PATCH 093/196] fix(codeowners): include LPX in generated artifact owners Signed-off-by: Andrew Paprotsky --- .github/codeowners/areas.yaml | 1 + CODEOWNERS | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 15abd20e5a08..5a48f5f03fce 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -580,6 +580,7 @@ shared: - frontend - gms - kv-memory + - lpx - multimodal - observability - operator diff --git a/CODEOWNERS b/CODEOWNERS index a1e699cb20e8..f3e3673bbff2 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -580,7 +580,7 @@ requirements.txt /.codex/ @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners /agents/ @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners /.agents/ @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners -/CODEOWNERS @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners @ai-dynamo/dynamo-backend-sglang-codeowners @ai-dynamo/dynamo-backend-trtllm-codeowners @ai-dynamo/dynamo-backend-vllm-codeowners @ai-dynamo/dynamo-diffusion-codeowners @ai-dynamo/dynamo-docs-codeowners @ai-dynamo/dynamo-epp-codeowners @ai-dynamo/dynamo-fault-tolerance-codeowners @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-kv-memory-codeowners @ai-dynamo/dynamo-multimodal-codeowners @ai-dynamo/dynamo-observability-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-performance-codeowners @ai-dynamo/dynamo-planner-codeowners @ai-dynamo/dynamo-rl-codeowners @ai-dynamo/dynamo-router-codeowners @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-tokenspeed-codeowners @ai-dynamo/dynamo-xpu-codeowners +/CODEOWNERS @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners @ai-dynamo/dynamo-backend-sglang-codeowners @ai-dynamo/dynamo-backend-trtllm-codeowners @ai-dynamo/dynamo-backend-vllm-codeowners @ai-dynamo/dynamo-diffusion-codeowners @ai-dynamo/dynamo-docs-codeowners @ai-dynamo/dynamo-epp-codeowners @ai-dynamo/dynamo-fault-tolerance-codeowners @ai-dynamo/dynamo-frontend-codeowners @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-kv-memory-codeowners @ai-dynamo/dynamo-lpx-codeowners @ai-dynamo/dynamo-multimodal-codeowners @ai-dynamo/dynamo-observability-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-performance-codeowners @ai-dynamo/dynamo-planner-codeowners @ai-dynamo/dynamo-rl-codeowners @ai-dynamo/dynamo-router-codeowners @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-tokenspeed-codeowners @ai-dynamo/dynamo-xpu-codeowners /agent-docs/ @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners /.lycheeignore @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-docs-codeowners /tests/deploy/ @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners From fd1d692a841f3c9045d2ffaa09002d5b6de5f4de Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:55:58 +0300 Subject: [PATCH 094/196] refactor(operator): let LPX runtimes consume build manifests (#531) Signed-off-by: Andrew Paprotsky (cherry picked from commit 2a9b1469ef00ace298ea07b24667a46974036c00) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 54 ++++++------ deploy/operator/internal/dynamo/graph_lpx.go | 35 +------- .../internal/dynamo/lpx/cyborg_render.go | 9 +- .../internal/dynamo/lpx/cyborg_render_test.go | 40 ++++----- .../internal/dynamo/lpx/cyborg_runtime.go | 70 ++++------------ .../dynamo/lpx/cyborg_runtime_test.go | 39 ++++++--- .../dynamo/lpx/hybrid_cyborg_bindings.go | 82 +------------------ .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 38 ++------- .../internal/dynamo/lpx/render_test.go | 2 +- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 22 +---- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 22 +---- .../node-local-v3-hx-hybrid.yaml | 8 -- 12 files changed, 107 insertions(+), 314 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 38928e42489a..79dc49d2bcd7 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1043,14 +1043,14 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test lpxComponent.LPX.BuildID = "node-local-v2-cpu-embeddings" lpxComponent.ModelRef = &v1beta1.ModelReference{Name: "test/model"} - t.Log("Customize the independent Cyborg batch while keeping its sidecar separate") + t.Log("Author the Cyborg batch while keeping its sidecar separate") cyborgRole := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXConductor) require.NotNil(t, cyborgRole) authoredCyborgMain := &cyborgRole.PodTemplate.Spec.Containers[0] require.Equal(t, commonconsts.MainContainerName, authoredCyborgMain.Name) authoredCyborgMain.Env = append( authoredCyborgMain.Env, - corev1.EnvVar{Name: lpx.CyborgBatchSizeEnv, Value: "2"}, + corev1.EnvVar{Name: "CYBORG_BATCH_SIZE", Value: "2"}, corev1.EnvVar{Name: "USER_BATCH_REF", Value: "$(CYBORG_BATCH_SIZE)"}, ) @@ -1219,15 +1219,14 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, "/dev/infiniband", infinibandVolume.HostPath.Path) require.Nil(t, infinibandVolume.PersistentVolumeClaim) - t.Log("Verify Cyborg receives every runtime value consumed by its startup command") + t.Log("Preserve authored Cyborg runtime values and their expansion order") cyborgEnv := make(map[string]string, len(cyborgMain.Env)) for _, variable := range cyborgMain.Env { cyborgEnv[variable.Name] = variable.Value } require.Equal(t, "19877", cyborgEnv["RDMA_PORT"]) require.Equal(t, "/tmp/lpu_servers", cyborgEnv["SERVER_HOSTS_FILE"]) - require.Equal(t, "1", cyborgEnv["TOTAL_REPLICAS"]) - require.Equal(t, "2", cyborgEnv[lpx.CyborgBatchSizeEnv]) + require.Equal(t, "2", cyborgEnv["CYBORG_BATCH_SIZE"]) require.NotContains(t, cyborgEnv, "CYBORG_SWA_CACHE_IDS") batchIndex := slices.IndexFunc(cyborgMain.Env, func(variable corev1.EnvVar) bool { return variable.Name == "CYBORG_BATCH_SIZE" @@ -1238,18 +1237,6 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.NotEqual(t, -1, batchIndex) require.NotEqual(t, -1, userIndex) require.Less(t, batchIndex, userIndex) - var tokenizerEnv *corev1.EnvVar - for index := range cyborgMain.Env { - if cyborgMain.Env[index].Name == "TOKENIZER_DIR" { - tokenizerEnv = &cyborgMain.Env[index] - break - } - } - require.NotNil(t, tokenizerEnv) - require.NotNil(t, tokenizerEnv.ValueFrom) - require.NotNil(t, tokenizerEnv.ValueFrom.ConfigMapKeyRef) - require.Equal(t, decodeConfigName, tokenizerEnv.ValueFrom.ConfigMapKeyRef.Name) - require.Equal(t, "tokenizer_dir", tokenizerEnv.ValueFrom.ConfigMapKeyRef.Key) t.Log("Keep partition zero as the collapsed runtime root for all three physical partitions") lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, lpuConfigName) @@ -1262,9 +1249,8 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, decodeConfigName, extraResources[0].GetName()) require.Equal(t, lpuConfigName, extraResources[1].GetName()) require.Equal(t, dgd.Namespace, decodeConfig.Namespace) - require.Len(t, decodeConfig.Data, 2) + require.Len(t, decodeConfig.Data, 1) require.Equal(t, []string{"lpx-${GROVE_PCSG_INDEX}-agt-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) - require.NotEmpty(t, decodeConfig.Data["tokenizer_dir"]) decodeHash := lpx.LPUConfigMapHash(decodeConfig) require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) } @@ -1274,6 +1260,7 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. registry := newTestDataModelRegistry(t, t.TempDir()) staticEnvNames := []string{ "POD_IP", "POD_NAME", "POD_NAMESPACE", "POD_UID", "LPU_MODEL_NAME", "CYBORG_FPGA_GPI_REPLICA_INDEX", + "CYBORG_BATCH_SIZE", "CYBORG_FPGA_GPI_IO_FPGA_COUNT", "CYBORG_SWA_CACHE_IDS", "TOKENIZER_DIR", "TOTAL_REPLICAS", "DYN_SYSTEM_ENABLED", "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", "DYN_SYSTEM_PORT", "DYN_HEALTH_CHECK_ENABLED", "NIXL_TELEMETRY_ENABLE", "NIXL_TELEMETRY_EXPORTER", "NIXL_TELEMETRY_PROMETHEUS_PORT", "DYN_FORWARDPASS_METRIC_PORT", } @@ -1327,12 +1314,18 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. {name: "allocation environment expansion", args: []string{"--allocation", "$(LPX_ALLOCATION)"}}, {name: "argument terminator", command: []string{"/opt/custom-runtime"}, args: []string{"--", "literal argument"}}, {name: "shell command", command: []string{"/bin/sh", "-c"}, args: []string{"exec /opt/custom-runtime --allocation $LPX_ALLOCATION"}}, + { + name: "Cyborg manifest argument remains user-owned", + cyborgCommand: []string{"/opt/cyborg-runtime"}, + cyborgArgs: []string{"--gbuild-manifest", "$(GBUILD_MANIFEST_PATH)"}, + hybridOnly: true, + }, { name: "explicit Cyborg shell remains user-owned", command: []string{"/opt/custom-runtime"}, args: []string{"serve"}, cyborgCommand: []string{"/bin/sh", "-c"}, - cyborgArgs: []string{"exec /opt/cyborg-runtime serve"}, + cyborgArgs: []string{`exec /opt/cyborg-runtime --gbuild-manifest "$GBUILD_MANIFEST_PATH"`}, hybridOnly: true, }, { @@ -1565,6 +1558,9 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. require.NotContains(t, main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) case lpxv1alpha1.PodRoleCyborgWorker: cyborgs++ + require.Contains(t, main.Env, corev1.EnvVar{ + Name: "GBUILD_MANIFEST_PATH", Value: filepath.Join("/nfs", component.LPX.BuildID, "manifest.v2.capnp.bin"), + }) } } require.Positive(t, agents) @@ -1613,7 +1609,13 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, {Name: "TOTAL_REPLICAS", Value: "9"}, {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, - {Name: lpx.CyborgBatchSizeEnv, Value: "2"}, + } + for _, name := range []string{"CYBORG_BATCH_SIZE", "CYBORG_FPGA_GPI_IO_FPGA_COUNT", "CYBORG_SWA_CACHE_IDS"} { + leader.Spec.Containers[0].Env = append(leader.Spec.Containers[0].Env, corev1.EnvVar{ + Name: name, ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "runtime-settings"}, Key: name, + }}, + }) } authoredConfig := corev1.Volume{ Name: "config", @@ -1646,18 +1648,14 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { } require.Equal(t, []corev1.Volume{authoredConfig}, configVolumes) - t.Log("Preserve authored mounts and runtime environment over the generated Cyborg defaults") + t.Log("Preserve authored mounts and runtime environment without interpreting literal or valueFrom settings") main := podSpec.Containers[0] require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom-config", ReadOnly: true}) require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}) require.False(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" })) - env := make(map[string]string, len(main.Env)) - for _, variable := range main.Env { - env[variable.Name] = variable.Value + for _, variable := range leader.Spec.Containers[0].Env { + require.Contains(t, main.Env, variable) } - require.Equal(t, "/custom-tokenizer", env["TOKENIZER_DIR"]) - require.Equal(t, "9", env["TOTAL_REPLICAS"]) - require.Equal(t, "/custom-servers", env["SERVER_HOSTS_FILE"]) t.Log("Preserve the authored command and host-file override without adding a launcher") require.Equal(t, leader.Spec.Containers[0].Command, main.Command) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 1c465b0d288d..a1ce2feb5614 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -312,16 +312,12 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p role.ComponentType = v1beta1.ComponentTypeDecode role.Replicas = ptr.To(replicas) role.MinAvailable = nil - var defaults ComponentDefaults = &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} + defaults := &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} if workload.BuildFamily() == dynamolpx.BuildFamilyXT { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan, lpuTemplate.Spec) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan) if err != nil { return nil, nil, err } - defaults = &selectedCyborgComponentDefaults{ - ComponentDefaults: defaults, workload: workload, configMapName: input.CyborgConfigMap.Name, - replicas: *role.Replicas, lpxPodSpec: lpuTemplate.Spec, - } } gpu := p gpu.component = role @@ -390,33 +386,6 @@ func (d *podTemplateRuntimeDefaults) GetBaseContainer(context ComponentContext) return container, nil } -// selectedCyborgComponentDefaults installs LPX-owned Cyborg bindings before -// the user's PodTemplate is merged, preserving the normal merge strategy. -type selectedCyborgComponentDefaults struct { - ComponentDefaults - workload *dynamolpx.SelectedWorkload - configMapName string - replicas int32 - lpxPodSpec corev1.PodSpec -} - -func (d *selectedCyborgComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) { - container, err := d.ComponentDefaults.GetBaseContainer(context) - if err != nil { - return corev1.Container{}, err - } - if err := dynamolpx.ApplySelectedCyborgContainerDefaults( - &container, - d.workload, - d.configMapName, - d.replicas, - d.lpxPodSpec, - ); err != nil { - return corev1.Container{}, err - } - return container, nil -} - // renderSelectedLPXRole consumes a private component copy; other inputs are read-only. // component and its PodTemplate must be non-nil. // alphaComponent may be nil. diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 2c683cdde0d3..b0a9732e16d2 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -35,16 +35,9 @@ func configureHybridCyborg( if cyborgStorage.mount.MountPath != modelStorage.mount.MountPath { return fmt.Errorf("selected Cyborg podTemplate conflicts with model storage mount %q", modelStorage.mount.MountPath) } - cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, cyborg.Spec.Replicas) - if err != nil { - return err - } - cyborgBatchSize, err = configuredCyborgBatchSize(container, cyborgBatchSize) - if err != nil { + if err := validateCyborgReplicas(&projection.configuredBuild, cyborg.Spec.Replicas); err != nil { return err } - applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) - applyCyborgSWACacheIDs(container, cyborgBatchSize) if err := applyCyborgManifestPath(container, projection, modelStorage.mount.MountPath); err != nil { return err } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 5e8011015737..ee83238f7edc 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -72,7 +72,7 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { } } -func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { +func TestRenderHybridPreservesRuntimeEnvironment(t *testing.T) { t.Parallel() t.Log("Project a split-I/O selected workload") @@ -107,11 +107,17 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { } decode.Spec.PodSpec.Containers[0].Command = []string{"/custom-cyborg", "--wrapper-option"} decode.Spec.PodSpec.Containers[0].Args = []string{"argument with spaces", "literal $HOME", ""} - decode.Spec.PodSpec.Containers[0].Env = append( - decode.Spec.PodSpec.Containers[0].Env, - corev1.EnvVar{Name: "RDMA_PORT", Value: "12345"}, - corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, - ) + authoredEnv := []corev1.EnvVar{ + {Name: "RDMA_PORT", Value: "12345"}, + {Name: "CYBORG_BATCH_SIZE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels['runtime-batch']", + }}}, + {Name: "CYBORG_FPGA_GPI_IO_FPGA_COUNT", Value: "9"}, + {Name: "CYBORG_SWA_CACHE_IDS", Value: "8,9"}, + {Name: "TOKENIZER_DIR", Value: "$(GBUILD_MANIFEST_PATH)/../tokenizer"}, + {Name: "TOTAL_REPLICAS", Value: "9"}, + } + decode.Spec.PodSpec.Containers[0].Env = authoredEnv t.Log("Use independently provisioned Cyborg model storage at the shared runtime path") decode.Spec.PodSpec.Volumes[0].PersistentVolumeClaim.ClaimName = "cyborg-models" @@ -125,19 +131,11 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.NoError(t, err) cyborg := namedClique(t, rendered, "cond") - t.Log("Verify the rendered Cyborg runtime I/O contract") - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: "CYBORG_FPGA_GPI_IO_FPGA_COUNT", Value: "2", - }) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: "CYBORG_BATCH_SIZE", Value: "3", - }) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + t.Log("Keep runtime environment opaque while supplying its manifest before authored references") + manifestEnv := corev1.EnvVar{ Name: "GBUILD_MANIFEST_PATH", Value: "/models/model-build/manifest.v2.capnp.bin", - }) - require.Contains(t, cyborg.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ - Name: "RDMA_PORT", Value: "12345", - }) + } + require.Equal(t, append([]corev1.EnvVar{manifestEnv}, authoredEnv...), cyborg.Spec.PodSpec.Containers[0].Env) require.Equal(t, &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "cyborg-models", ReadOnly: true}, cyborg.Spec.PodSpec.Volumes[0].PersistentVolumeClaim) require.Contains(t, cyborg.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{ Name: "model-storage", MountPath: "/models", SubPath: "cyborg", ReadOnly: true, @@ -151,10 +149,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { imageEntrypointCyborg.Spec.Replicas = 4 imageEntrypointCyborg.Spec.MinAvailable = ptr.To[int32](4) imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args = []string{"serve"} - imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env = append( - imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env, - corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: "3"}, - ) + imageEntrypointEnv := append([]corev1.EnvVar{manifestEnv}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env...) t.Log("Leave the image ENTRYPOINT selected when command is omitted") input.Stages = map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}} @@ -162,6 +157,7 @@ func TestRenderHybridProjectsManifestRuntimeIO(t *testing.T) { require.NoError(t, err) require.Nil(t, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"serve"}, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Args) + require.Equal(t, imageEntrypointEnv, imageEntrypointCyborg.Spec.PodSpec.Containers[0].Env) t.Log("Reject invalid Cyborg runtime bindings") diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index e183089613aa..aa7dd9e57aba 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -8,20 +8,11 @@ package lpx import ( "fmt" "path/filepath" - "strconv" - "strings" corev1 "k8s.io/api/core/v1" ) -const ( - // CyborgBatchSizeEnv configures the batch handled by each Cyborg replica. - CyborgBatchSizeEnv = "CYBORG_BATCH_SIZE" - cyborgFpgaGpiIOFPGACountEnv = "CYBORG_FPGA_GPI_IO_FPGA_COUNT" - cyborgSwaCacheIDsEnv = "CYBORG_SWA_CACHE_IDS" - gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" - grovePodCliquePodIndexPath = "metadata.labels['grove.io/podclique-pod-index']" -) +const gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" // applyCyborgManifestPath projects an authoritative manifest location into one Cyborg container. func applyCyborgManifestPath(container *corev1.Container, projection *ModelProjection, modelStoragePath string) error { @@ -29,43 +20,32 @@ func applyCyborgManifestPath(container *corev1.Container, projection *ModelProje if err != nil { return fmt.Errorf("resolve GBuild manifest path: %w", err) } - setContainerEnv(container, corev1.EnvVar{ + + // Kubernetes expands environment references in order; publish the manifest before authored bindings. + env := make([]corev1.EnvVar, 0, len(container.Env)+1) + env = append(env, corev1.EnvVar{ Name: gbuildManifestPathEnv, Value: filepath.Join(buildRoot, gbuildManifestV2CapnpFile), }) - return nil -} - -// applyCyborgRuntimeIO projects the resolved split-I/O fanout into one Cyborg container. -func applyCyborgRuntimeIO(container *corev1.Container, cyborgBatchSize int, ioFPGACount int32) { - setContainerEnv(container, - corev1.EnvVar{Name: cyborgFpgaGpiIOFPGACountEnv, Value: strconv.FormatInt(int64(ioFPGACount), 10)}, - corev1.EnvVar{Name: CyborgBatchSizeEnv, Value: strconv.Itoa(cyborgBatchSize)}, - ) -} - -// applyCyborgSWACacheIDs binds the replica index for an unbatched Cyborg container. -func applyCyborgSWACacheIDs(container *corev1.Container, cyborgBatchSize int) { - if cyborgBatchSize == 1 { - setContainerEnv(container, corev1.EnvVar{ - Name: cyborgSwaCacheIDsEnv, - ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: grovePodCliquePodIndexPath, - }}, - }) + for _, variable := range container.Env { + if variable.Name != gbuildManifestPathEnv { + env = append(env, variable) + } } + container.Env = env + return nil } -// cyborgRuntimeIO requires a nonnil normalized build and validates its Cyborg replica domain. -func cyborgRuntimeIO(build *Build, replicas int32) (int, int32, error) { +// validateCyborgReplicas requires a nonnil normalized build and validates its Cyborg replica domain. +func validateCyborgReplicas(build *Build, replicas int32) error { // Require every physical endpoint and client-owned transaction in this replica domain. ioFPGACount := build.IOFPGACount ioFanoutFactor := build.IOFanoutFactor if replicas%ioFPGACount != 0 { - return 0, 0, fmt.Errorf("decode service Cyborg replicas %d must be divisible by ioFpgaCount %d", replicas, ioFPGACount) + return fmt.Errorf("decode service Cyborg replicas %d must be divisible by ioFpgaCount %d", replicas, ioFPGACount) } if replicas/ioFPGACount%ioFanoutFactor != 0 { - return 0, 0, fmt.Errorf( + return fmt.Errorf( "decode service Cyborg replicas %d must provide fanoutFactor %d clients for each of %d I/O FPGA endpoints", replicas, ioFanoutFactor, @@ -73,25 +53,7 @@ func cyborgRuntimeIO(build *Build, replicas int32) (int, int32, error) { ) } - // Derive the normalized batch carried by one fanout transaction on one endpoint. - return build.BatchSize / int(ioFPGACount) / int(ioFanoutFactor), ioFPGACount, nil -} - -func configuredCyborgBatchSize(container *corev1.Container, derived int) (int, error) { - for _, variable := range container.Env { - if variable.Name != CyborgBatchSizeEnv { - continue - } - if variable.ValueFrom != nil { - return 0, fmt.Errorf("Cyborg %s must use a literal value", CyborgBatchSizeEnv) - } - configured, err := strconv.Atoi(strings.TrimSpace(variable.Value)) - if err != nil || configured < 1 { - return 0, fmt.Errorf("Cyborg %s must be an integer greater than zero, got %q", CyborgBatchSizeEnv, variable.Value) - } - return configured, nil - } - return derived, nil + return nil } // setContainerEnv updates the first matching variable or appends a new one. diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go index e02514f8d6bc..0dc4d8d363b5 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime_test.go @@ -12,16 +12,35 @@ import ( corev1 "k8s.io/api/core/v1" ) -func TestConfiguredCyborgBatchSizeRejectsInvalidEnvironment(t *testing.T) { - t.Log("Define invalid literal and field-sourced Cyborg batch-size variables") - tests := []corev1.EnvVar{ - {Name: CyborgBatchSizeEnv, Value: "0"}, - {Name: CyborgBatchSizeEnv, Value: "invalid"}, - {Name: CyborgBatchSizeEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, +func TestApplyCyborgManifestPathPrecedesAuthoredReferences(t *testing.T) { + t.Parallel() + + t.Log("Define authored bindings that depend on the generated manifest location") + projection := &ModelProjection{configuredBuild: Build{Path: "file:///models/build"}} + authored := []corev1.EnvVar{ + {Name: "MODEL_PATH", Value: "$(GBUILD_MANIFEST_PATH)"}, + {Name: "OTHER", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, } - for _, variable := range tests { - t.Logf("Reject invalid Cyborg batch-size environment %+v", variable) - _, err := configuredCyborgBatchSize(&corev1.Container{Env: []corev1.EnvVar{variable}}, 1) - require.Error(t, err) + want := append([]corev1.EnvVar{{Name: gbuildManifestPathEnv, Value: "/models/build/manifest.v2.capnp.bin"}}, authored...) + for _, test := range []struct { + name string + env []corev1.EnvVar + }{ + {name: "missing binding", env: authored}, + {name: "stale binding after reference", env: append(append([]corev1.EnvVar(nil), authored...), corev1.EnvVar{ + Name: gbuildManifestPathEnv, Value: "/stale/manifest", + })}, + {name: "existing binding before reference", env: want}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Publish the authoritative path before references while preserving other bindings") + container := corev1.Container{Env: test.env} + require.NoError(t, applyCyborgManifestPath(&container, projection, "/models")) + require.Equal(t, want, container.Env) + + t.Log("Repeat rendering without duplicating or reordering environment bindings") + require.NoError(t, applyCyborgManifestPath(&container, projection, "/models")) + require.Equal(t, want, container.Env) + }) } } diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 3d2e6c9f209c..b5193e453b8e 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -6,95 +6,20 @@ package lpx import ( - "fmt" - "path/filepath" - "slices" "strconv" "strings" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" ) -const ( - selectedCyborgTokenizerDirEnv = "TOKENIZER_DIR" - selectedCyborgTotalReplicasEnv = "TOTAL_REPLICAS" -) - -// ApplySelectedCyborgContainerDefaults installs the operator-owned V2 Cyborg -// bindings before the user's main-container override is merged. container and -// workload must be non-nil, and workload must contain a projection. The -// function mutates container and reads workload and lpxPodSpec without mutating them. -func ApplySelectedCyborgContainerDefaults( - container *corev1.Container, - workload *SelectedWorkload, - configMapName string, - totalReplicas int32, - lpxPodSpec corev1.PodSpec, -) error { - projection := workload.modelProjections[0] - - container.Env = append( - container.Env, - corev1.EnvVar{ - Name: selectedCyborgTokenizerDirEnv, - ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, - Key: "tokenizer_dir", - }}, - }, - corev1.EnvVar{Name: selectedCyborgTotalReplicasEnv, Value: strconv.FormatInt(int64(totalReplicas), 10)}, - ) - cyborgBatchSize, ioFPGACount, err := cyborgRuntimeIO(&projection.configuredBuild, totalReplicas) - if err != nil { - return err - } - // SWA cache IDs depend on the final, user-overridable batch size and are - // bound only after the main-container override is merged. - applyCyborgRuntimeIO(container, cyborgBatchSize, ioFPGACount) - lpxContainer := common.FindContainerByName(lpxPodSpec.Containers, commonconsts.MainContainerName) - mountIndex := slices.IndexFunc(lpxContainer.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == commonconsts.ModelStorageVolumeName - }) - if mountIndex < 0 { - return fmt.Errorf("selected LPX main container requires model storage volume mount %q", commonconsts.ModelStorageVolumeName) - } - modelStoragePath := lpxContainer.VolumeMounts[mountIndex].MountPath - if strings.TrimSpace(modelStoragePath) == "" { - return fmt.Errorf("model storage volume %q has no mount path", commonconsts.ModelStorageVolumeName) - } - if err := applyCyborgManifestPath(container, projection, modelStoragePath); err != nil { - return err - } - return nil -} - -// RenderCyborgConfigMap renders the XT hybrid configuration before Cyborg defaults are merged. +// RenderCyborgConfigMap renders the XT hybrid Agent endpoints. // The workload and plan must be non-nil, validated, and describe the selected hybrid model. -// agentPodSpec supplies its merged storage. The workload and plan are not mutated. +// The workload and plan are not mutated. func (w *SelectedWorkload) RenderCyborgConfigMap( namespace string, plan *MaterializationPlan, - agentPodSpec corev1.PodSpec, ) (*corev1.ConfigMap, error) { - storage, err := lpuModelStorageBinding(agentPodSpec) - if err != nil { - return nil, err - } build := &w.modelProjections[0].configuredBuild - tokenizerDir := "" - if build.Path != "" { - tokenizerPath := build.RuntimeTokenizerPath - if strings.TrimSpace(tokenizerPath) == "" { - return nil, fmt.Errorf("capnp manifest build is missing model.tokenizer.path") - } - runtimePath, runtimeErr := buildRuntimePath(build.Path, storage.mount.MountPath) - if runtimeErr != nil { - return nil, fmt.Errorf("failed to determine tokenizer_dir: %w", runtimeErr) - } - tokenizerDir = filepath.Join(runtimePath, tokenizerPath) - } // Cyborg supplies the PCS prefix; startup supplies this engine's Grove index. serverPrefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" @@ -106,7 +31,6 @@ func (w *SelectedWorkload) RenderCyborgConfigMap( } return renderRuntimeConfigMap(namespace, plan.PodCliqueSetName+"-decode", map[string]string{ - "tokenizer_dir": tokenizerDir, - "lpu_servers": strings.Join(servers, "\n"), + "lpu_servers": strings.Join(servers, "\n"), }) } diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 8fe1ee0f6b8d..7889925f89bd 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -7,14 +7,12 @@ package lpx import ( "encoding/json" - "path/filepath" "strconv" "strings" "testing" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" ) func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { @@ -40,13 +38,10 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { scalingGroupReplicas: 1, } - t.Log("Resolve the projected runtime path") + t.Log("Render the generated Agent endpoints") plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - runtimePath, err := buildRuntimePath(projection.configuredBuild.Path, "/models") - require.NoError(t, err) - podSpec := renderTestPodSpec() - initial, err := workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) + initial, err := workload.RenderCyborgConfigMap("test-namespace", plan) require.NoError(t, err) t.Log("Verify every engine replica addresses only its own Agents") @@ -55,13 +50,13 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { workload.scalingGroupReplicas = replicas scaledPlan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - configMap, err := workload.RenderCyborgConfigMap("test-namespace", scaledPlan, podSpec) + configMap, err := workload.RenderCyborgConfigMap("test-namespace", scaledPlan) require.NoError(t, err) require.Equal(t, initial, configMap) require.True(t, *configMap.Immutable) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" require.Equal(t, prefix+"agt-0\n"+prefix+"agt-2", configMap.Data["lpu_servers"]) - require.Equal(t, filepath.Join(runtimePath, "tokenizer"), configMap.Data["tokenizer_dir"]) + require.Len(t, configMap.Data, 1) t.Log("Resolve Cyborg server addresses to the last engine replica's actual Agent hostnames") lastReplica := scaledPlan.ForReplica(replicas - 1) @@ -71,29 +66,6 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { } }) } - - t.Log("Render a Cap'n Proto build with a nested tokenizer path") - projection.configuredBuild.RuntimeTokenizerPath = "metadata/tokenizer" - configMap, err := workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) - - t.Log("Verify the nested tokenizer path is rooted in the runtime build") - require.NoError(t, err) - require.Equal(t, filepath.Join(runtimePath, "metadata/tokenizer"), configMap.Data["tokenizer_dir"]) - require.NotEqual(t, initial.Name, configMap.Name) - - t.Log("Reject oversized rendered Cyborg configuration before publication") - projection.configuredBuild.RuntimeTokenizerPath = strings.Repeat("x", corev1.MaxSecretSize) - _, err = workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) - require.ErrorContains(t, err, "rendered LPX ConfigMap") - require.ErrorContains(t, err, "maximum is 1048576") - - t.Log("Render a Cap'n Proto build without tokenizer metadata") - projection.configuredBuild.RuntimeTokenizerPath = "" - projection.configuredBuild.Path = "relative/build" - _, err = workload.RenderCyborgConfigMap("test-namespace", plan, podSpec) - - t.Log("Verify missing tokenizer metadata is rejected") - require.EqualError(t, err, "capnp manifest build is missing model.tokenizer.path") } func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { @@ -134,7 +106,7 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { require.NoError(t, err) require.Len(t, projection.RequestSpec(plan, "agents").Partitions, test.partitions) require.Equal(t, 2*test.partitions, projection.agentReplicas) - configMap, err := workload.RenderCyborgConfigMap("test", plan, renderTestPodSpec()) + configMap, err := workload.RenderCyborgConfigMap("test", plan) require.NoError(t, err) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(test.wantOffsets)) diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 361da8e6279f..582cb0dc99e7 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -48,7 +48,7 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model return nil, err } if workload.BuildFamily() == BuildFamilyXT && workload.Pipeline() == PipelineLPX { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, plan, input.Stages[testRenderComponentName].Spec) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, plan) if err != nil { return nil, err } diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index f51d4026ba03..a5dc46bf550d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -20,7 +20,6 @@ data: lpx-${GROVE_PCSG_INDEX}-agt-14 lpx-${GROVE_PCSG_INDEX}-agt-15 lpx-${GROVE_PCSG_INDEX}-agt-16 - tokenizer_dir: /testdata/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer immutable: true metadata: annotations: @@ -29,7 +28,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a + name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 namespace: inference-engine --- data: @@ -428,7 +427,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 3aba9d08778da54a7d7c53f0d1f5bdeb06f931a8f6390287d34c5f039dc58075 + nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -471,18 +470,10 @@ spec: - -- - /usr/local/bin/cyborg env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - name: DYN_COMPONENT value: decode - name: DYN_DISCOVERY_BACKEND @@ -537,13 +528,6 @@ spec: value: "19877" - name: SERVER_HOSTS_FILE value: /tmp/lpu_servers - - name: TOKENIZER_DIR - valueFrom: - configMapKeyRef: - key: tokenizer_dir - name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a - - name: TOTAL_REPLICAS - value: "1" image: example.invalid/gpu-runtime:test livenessProbe: failureThreshold: 1 @@ -615,7 +599,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-decode-3aba9d08778da54a + name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 01cfdeddc15d..037818f58eeb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -3,7 +3,6 @@ data: lpu_servers: lpx-${GROVE_PCSG_INDEX}-agt-0 - tokenizer_dir: /testdata/node-local-v2-connected-lpx/tokenizer immutable: true metadata: annotations: @@ -12,7 +11,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 + name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f namespace: test --- data: @@ -243,7 +242,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: f8838533ed7ff8246dfdef9db2185d041489d1dc07addbf57f5893ae77eb5139 + nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -269,18 +268,10 @@ spec: - -- - /opt/gpu-runtime env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - name: DYN_COMPONENT value: decode - name: DYN_DISCOVERY_BACKEND @@ -333,13 +324,6 @@ spec: value: "19877" - name: SERVER_HOSTS_FILE value: /tmp/lpu_servers - - name: TOKENIZER_DIR - valueFrom: - configMapKeyRef: - key: tokenizer_dir - name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 - - name: TOTAL_REPLICAS - value: "1" image: example.invalid/gpu-runtime:test livenessProbe: failureThreshold: 1 @@ -404,7 +388,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-decode-f8838533ed7ff824 + name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 6f3684356ebb..517115feba65 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -260,18 +260,10 @@ spec: command: - /opt/gpu-runtime env: - - name: CYBORG_BATCH_SIZE - value: "1" - - name: CYBORG_FPGA_GPI_IO_FPGA_COUNT - value: "1" - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: fieldPath: metadata.labels['grove.io/podclique-pod-index'] - - name: CYBORG_SWA_CACHE_IDS - valueFrom: - fieldRef: - fieldPath: metadata.labels['grove.io/podclique-pod-index'] - name: DYN_COMPONENT value: decode - name: DYN_DISCOVERY_BACKEND From d42d252e0dadf56f0fecc8051e37248a8d139d57 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:02:38 +0300 Subject: [PATCH 095/196] docs(helm): correct LPX startup prerequisites (#532) Signed-off-by: Andrew Paprotsky (cherry picked from commit d3a9e11bc75d6d25919c0948f17e4fcceaef11a4) Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 2 +- deploy/helm/charts/platform/components/operator/values.yaml | 3 +-- deploy/helm/charts/platform/values.yaml | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index e2d6d39329c3..8d8ebe6278d8 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -170,7 +170,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | -| dynamo-operator.lpx.enabled | bool | `false` | Enable the Dynamo operator's LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest and scheduler.grove.io/v1alpha1 PodGang APIs. | +| dynamo-operator.lpx.enabled | bool | `false` | Enable the Dynamo operator's LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. | | dynamo-operator.lpx.modelRegistryURL | string | `""` | Configure the LPU model registry used by the LPX integration. | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. | diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 4e2abe80c203..8926ec8fa564 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -25,8 +25,7 @@ upgradeCRD: true # The LPX scheduler and its CRDs must be installed separately. lpx: # -- Enable the Dynamo operator's LPX integration. Startup requires the - # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest - # and scheduler.grove.io/v1alpha1 PodGang APIs. + # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. enabled: false # -- Configure the LPU model registry used by the LPX integration. modelRegistryURL: "" diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index d844e901e28f..1a71a30e3aee 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -91,8 +91,7 @@ dynamo-operator: # The LPX scheduler and its CRDs must be installed separately. lpx: # -- Enable the Dynamo operator's LPX integration. Startup requires the - # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest - # and scheduler.grove.io/v1alpha1 PodGang APIs. + # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. enabled: false # -- Configure the LPU model registry used by the LPX integration. modelRegistryURL: "" From 299f798a602e011793ea7f4866aea44911ad0171 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:29:11 +0300 Subject: [PATCH 096/196] fix(operator): require Cyborg to consume declared resource claims (#533) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7bea2aac16e15a3dae21176b6d9cf5328affa339) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/lifecycle_test.go | 1 + .../dynamo/lpx/resolve_selected_workload.go | 15 ++++-- .../lpx/resolve_selected_workload_test.go | 52 ++++++++++++++++++- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index ff7b8308f525..8951e6ca15f8 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -1796,6 +1796,7 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ ResourceClaims: []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("gpu")}}, Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "cyborg-runtime", + Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "gpu"}}}, VolumeMounts: []corev1.VolumeMount{ {Name: "config", MountPath: "/configs"}, {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index 25f1d088d2a5..d2b8fef5f1c5 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -154,10 +154,19 @@ func validateSelectedConductor( if err != nil { return fmt.Errorf("component %q conductor resources: %w", component.ComponentName, err) } - if len(template.Spec.ResourceClaims) == 0 && count <= 0 { - return fmt.Errorf("component %q conductor requires resourceClaims or a positive %s request", component.ComponentName, commonconsts.KubeResourceGPUNvidia) + if count > 0 { + return nil } - return nil + + // A Pod claim supplies devices only to containers that reference its local name. + for _, claim := range container.Resources.Claims { + for _, podClaim := range template.Spec.ResourceClaims { + if claim.Name == podClaim.Name { + return nil + } + } + } + return fmt.Errorf("component %q conductor main container requires a declared resourceClaim or a positive %s request", component.ComponentName, commonconsts.KubeResourceGPUNvidia) } // Only the LPU-only serving template materializes a renamed conductor container. diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index 5036e84f151a..f69a46e0c76e 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -64,11 +64,59 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) source := staticBuildSnapshotSource{"build": snapshot} _, err = ResolveSelectedWorkload(t.Context(), dgd, source) - require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") + require.ErrorContains(t, err, "requires a declared resourceClaim or a positive nvidia.com/gpu request") conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) conductor.PodTemplate = testLPXPodTemplate("cyborg-runtime") _, err = ResolveSelectedWorkload(t.Context(), dgd, source) - require.ErrorContains(t, err, "requires resourceClaims or a positive nvidia.com/gpu request") + require.ErrorContains(t, err, "requires a declared resourceClaim or a positive nvidia.com/gpu request") + + t.Log("Validate claim consumption by the Cyborg main container") + for _, test := range []struct { + name string + claims []corev1.ResourceClaim + declared, sidecar, scalarGPU bool + wantErr bool + }{ + {name: "unused claim", declared: true, wantErr: true}, + {name: "sidecar-only claim", declared: true, sidecar: true, wantErr: true}, + {name: "undeclared claim", claims: []corev1.ResourceClaim{{Name: "gpu"}}, wantErr: true}, + {name: "mismatched claim", declared: true, claims: []corev1.ResourceClaim{{Name: "missing"}}, wantErr: true}, + {name: "external name is not the alias", declared: true, claims: []corev1.ResourceClaim{{Name: "external-gpu"}}, wantErr: true}, + {name: "consumed claim", declared: true, claims: []corev1.ResourceClaim{{Name: "gpu"}}}, + {name: "scalar GPU with unused claim", declared: true, scalarGPU: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Author independent Pod and main-container claim references") + candidate := dgd.DeepCopy() + pod := &candidate.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec + pod.Containers[0].Resources.Claims = test.claims + if test.declared { + pod.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "other", ResourceClaimName: ptr.To("external-other")}, + {Name: "gpu", ResourceClaimName: ptr.To("external-gpu")}, + } + } + if test.sidecar { + pod.Containers = append(pod.Containers, corev1.Container{ + Name: "sidecar", Image: "sidecar", Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "gpu"}}}, + }) + } + if test.scalarGPU { + pod.Containers[0].Resources.Limits = corev1.ResourceList{corev1.ResourceName(commonconsts.KubeResourceGPUNvidia): resource.MustParse("1")} + } + before := candidate.DeepCopy() + + t.Log("Require scalar GPUs or a claim consumed by main without rewriting the template") + _, err := ResolveSelectedWorkload(t.Context(), candidate, source) + if test.wantErr { + require.ErrorContains(t, err, "conductor main container requires a declared resourceClaim") + } else { + require.NoError(t, err) + } + require.Equal(t, before, candidate) + }) + } + conductor.PodTemplate.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ corev1.ResourceName(commonconsts.KubeResourceGPUNvidia): resource.MustParse("1"), } From 817e61d5ecbd18499c71c5045311b39f62bb5dd6 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:31:51 +0300 Subject: [PATCH 097/196] refactor(lpx): defer model configuration to runtimes (#534) Signed-off-by: Andrew Paprotsky (cherry picked from commit e30f33b46456a337b81d68932759bca460b8ee12) Signed-off-by: Andrew Paprotsky --- .../controller/lpx/controller_test.go | 10 +- .../internal/controller/lpx/render_test.go | 51 ++---- deploy/operator/internal/dynamo/lpx/build.go | 12 +- .../dynamo/lpx/build_snapshot_gcs_test.go | 1 - .../dynamo/lpx/build_snapshot_test.go | 5 +- .../internal/dynamo/lpx/cyborg_render_test.go | 1 - .../internal/dynamo/lpx/lpu_configmap.go | 107 +++--------- .../internal/dynamo/lpx/lpu_configmap_test.go | 59 +++---- .../dynamo/lpx/model_manifest_common.go | 44 ----- .../internal/dynamo/lpx/model_manifest_v2.go | 165 +++++------------- .../dynamo/lpx/model_manifest_v2_test.go | 92 ++++++++-- deploy/operator/internal/dynamo/lpx/render.go | 2 +- .../internal/dynamo/lpx/render_test.go | 24 ++- .../lpx/resolve_selected_workload_test.go | 4 +- .../internal/dynamo/lpx/runtime_agent.go | 7 - .../internal/dynamo/lpx/runtime_settings.go | 59 +------ .../dynamo/lpx/runtime_settings_test.go | 94 +++------- .../dynamo/lpx/runtime_startup_test.go | 2 +- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 34 +--- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 34 +--- .../node-local-v2-lpu-only.input.yaml | 6 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 38 +--- .../node-local-v2-specdecode.input.yaml | 6 +- .../node-local-v2-specdecode.yaml | 73 ++------ .../node-local-v3-hx-hybrid.yaml | 43 +---- .../node-local-v3-hx-lpu-only.input.yaml | 6 +- .../node-local-v3-hx-lpu-only.yaml | 38 +--- .../node-local-v3-hx-specdecode.input.yaml | 6 +- .../node-local-v3-hx-specdecode.yaml | 73 ++------ .../from_dgd_yaml/single_v2.input.yaml | 6 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 32 +--- .../dynamo/lpx/workload_projection_v2.go | 13 +- .../dynamo/lpx/workload_projection_v2_test.go | 10 +- .../dynamo/lpx/workload_projection_v3.go | 14 +- .../dynamo/lpx/workload_projection_v3_test.go | 10 +- 35 files changed, 354 insertions(+), 827 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index e41d6622ed24..fe1eae942890 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1198,13 +1198,17 @@ func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { } require.ElementsMatch(t, []string{"cond", "agt0", "agt1"}, cliqueNames) - t.Log("Keep runtime Agent addresses aligned with the clique names") + t.Log("Keep runtime allocation aligned with the clique names") config := &corev1.ConfigMap{} configKey := client.ObjectKey{Namespace: child.Namespace, Name: lpx.LPUConfigMapName(root, configHash)} require.NoError(t, r.Get(t.Context(), configKey, config)) require.True(t, metav1.IsControlledBy(config, child)) - require.Contains(t, config.Data["datacenter.toml"], "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}") - require.Contains(t, config.Data["datacenter.toml"], "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}") + require.NotContains(t, config.Data, "datacenter.toml") + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name == "cond" { + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "agt0:agt1"}) + } + } } } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 79dc49d2bcd7..2186eea04e93 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -403,7 +403,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate template.Spec.Containers[0].Image = "independent-conductor-runtime" template.Spec.Containers[0].Command = []string{"/bin/nova"} - template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "CONDUCTOR_ONLY", Value: "kept"}} + template.Spec.Containers[0].Env = append(template.Spec.Containers[0].Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) template.Labels = map[string]string{"owner": "explicit-conductor"} template.Spec.NodeSelector = map[string]string{"runtime-role": "conductor"} template.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ @@ -437,9 +437,10 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { conductors++ require.Equal(t, "independent-conductor-runtime", container.Image) require.Equal(t, []string{"/bin/nova"}, container.Command) - configFlag := slices.Index(container.Args, "--datacenter-config-filepath") - require.GreaterOrEqual(t, configFlag, 0) - require.Equal(t, "/configs/datacenter.toml", container.Args[configFlag+1]) + require.NotContains(t, container.Args, "--datacenter-config-filepath") + require.Contains(t, container.Env, corev1.EnvVar{ + Name: "NOVA_NODE_NAME_TEMPLATE", Value: "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE}", + }) require.Equal(t, "explicit-conductor", clique.Labels["owner"]) require.Equal(t, template.Spec.NodeSelector, clique.Spec.PodSpec.NodeSelector) require.Equal(t, template.Spec.Affinity, clique.Spec.PodSpec.Affinity) @@ -729,7 +730,7 @@ func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphM return data } -func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { +func TestSingleV2ConfigContainsOnlyDeploymentOverrides(t *testing.T) { controllerConfig := &configv1alpha1.OperatorConfiguration{ Orchestrators: configv1alpha1.OrchestratorConfiguration{ Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, @@ -745,7 +746,7 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { modelConfig string hash string } - t.Log("Render manifest-derived, equivalent explicit, and overridden runtime settings") + t.Log("Render omitted defaults and explicit deployment overrides") configs := make([]renderedConfig, 0, 3) for _, settingsJSON := range []string{"", `{"batch_size":1,"swa":{"chunked":false}}`, `{"sequence_length":65536}`} { payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml") @@ -788,44 +789,22 @@ func TestSingleV2ManifestDefaultsDriveConfigAndHash(t *testing.T) { configs = append(configs, renderedConfig{modelConfig: configMap.Data["model_config.toml"], hash: hash}) } - t.Log("Preserve derived config and hash until an explicit value changes the output") - derived, explicitButEqual, overridden := configs[0], configs[1], configs[2] + t.Log("Hash only emitted settings while preserving explicit deployment intent") + derived, explicit, overridden := configs[0], configs[1], configs[2] buildPath := filepath.Join(registryRoot, buildID) require.Equal(t, fmt.Sprintf(`type = 'Single' [iop] -batch_size = 1 -cpu_embeddings = true -dkvc = true -input_size = 1 model_path = '%s' -num_dkvc_blocks = 256 -num_kv_caches = 1 -num_layers = 24 -output_size = 1 -prop_sync = false -sequence_length = 131072 -stop_tokens = [200002, 199999, 200012] -tokenizer_path = '%s' -vocab_size = 201088 - -[iop.swa] -chunked = false -num_swa_dkvc_blocks = 1 -swa_ctx_len = 128 -swa_num_users = 8 -swa_padding_len = 0 - -[scheduler] [setup] resolved_partitions_dir = '/configs' -setup_ops_format = 'agent_v2' -`, buildPath, filepath.Join(buildPath, "tokenizer")), derived.modelConfig) - require.Equal(t, derived.modelConfig, explicitButEqual.modelConfig) - require.Equal(t, derived.hash, explicitButEqual.hash) - require.NotEqual(t, derived.modelConfig, overridden.modelConfig) +`, buildPath), derived.modelConfig) + require.Contains(t, explicit.modelConfig, "batch_size = 1") + require.Contains(t, explicit.modelConfig, "chunked = false") + require.NotEqual(t, derived.hash, explicit.hash) + require.Contains(t, overridden.modelConfig, "sequence_length = 65536") require.NotEqual(t, derived.hash, overridden.hash) } @@ -1171,7 +1150,7 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test lpuConfigName := lpx.LPUConfigMapName(plan.PodCliqueSetName, agent.Annotations[commonconsts.AnnotationExtraResourcesHash]) require.Equal(t, lpuConfigName, gasDirSource.ConfigMapKeyRef.Name) require.Equal(t, "gas_dir", gasDirSource.ConfigMapKeyRef.Key) - require.Equal(t, "topologies", envValueSource(main.Env, "TOPOLOGIES").ConfigMapKeyRef.Key) + require.NotContains(t, runtimeEnv, "TOPOLOGIES") podIPSource := envValueSource(main.Env, "POD_IP") require.NotNil(t, podIPSource) require.NotNil(t, podIPSource.FieldRef) diff --git a/deploy/operator/internal/dynamo/lpx/build.go b/deploy/operator/internal/dynamo/lpx/build.go index 442628073d31..95401790e7a7 100644 --- a/deploy/operator/internal/dynamo/lpx/build.go +++ b/deploy/operator/internal/dynamo/lpx/build.go @@ -36,7 +36,7 @@ const ( // Build is the registry's source-independent view of an LPU build. // // The version 2 Cap'n Proto manifest populates this shape before deployment code -// derives container args, replica counts, prop-sync settings, and tokenizer defaults. +// derives placement, replica counts, and runtime overrides. type Build struct { // Path is the absolute file or GCS reference of the build payload. Path string @@ -46,24 +46,18 @@ type Build struct { CompilationMode BuildCompilationMode // Partitions contains the normalized physical compiler partitions. Partitions []BuildPartition - // BatchSize is the normalized batch size for the build. - BatchSize int // SelectedPropSyncChains contains source partition IDs grouped into selected prop-sync chains. SelectedPropSyncChains [][]int // StandaloneTokenEmbeddings reports whether token embeddings occupy a standalone partition. StandaloneTokenEmbeddings bool // SupportsCPUEmbeddings reports whether standalone token embeddings may run on the CPU. SupportsCPUEmbeddings bool - // RuntimeTokenizerPath is the tokenizer path relative to the build payload. - RuntimeTokenizerPath string - // RuntimeTokenEmbeddingsPath is the token-embeddings path relative to the build payload. - RuntimeTokenEmbeddingsPath string // IOFPGACount is the number of I/O FPGA endpoints described by the build. IOFPGACount int32 // IOFanoutFactor is the number of clients assigned to each I/O FPGA transaction. IOFanoutFactor int32 - // runtimeSettings holds immutable normalized defaults on a snapshot and - // configured runtime settings shared by immutable component projections. + // runtimeSettings holds manifest overrides on a snapshot and merged + // authored settings shared by immutable component projections. runtimeSettings map[string]any } diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go index 84f25828f909..bb45982788c9 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go @@ -118,7 +118,6 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { build, err := normalizeRegistryFixtureBuild(testCase.ctx, registry, testCase.ref) require.NoError(t, err) require.Equal(t, "gs://bucket/registry/model/build", build.Path) - require.Equal(t, 8, build.BatchSize) require.Len(t, build.Partitions, 1) require.Equal(t, "part-0", build.Partitions[0].PartPath) require.Len(t, client.listRequests, 2) diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go index d54b64f91dcd..12540c9525dc 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go @@ -59,8 +59,7 @@ func TestAcquireBuildSnapshotTracksLocalContent(t *testing.T) { normalized, err := normalizeBuildSnapshot(first) require.NoError(t, err) build := normalized.build - require.Equal(t, 8, build.BatchSize) - require.Equal(t, int64(8), build.runtimeSettings["batch_size"]) + require.NotContains(t, build.runtimeSettings, "batch_size") require.EqualValues(t, 4, build.IOFPGACount) require.EqualValues(t, 2, build.IOFanoutFactor) @@ -69,7 +68,7 @@ func TestAcquireBuildSnapshotTracksLocalContent(t *testing.T) { require.NoError(t, err) relativeBuild, err := normalizeRegistryFixtureBuild(t.Context(), relativeRegistry, "build-id") require.NoError(t, err) - require.Equal(t, 8, relativeBuild.BatchSize) + require.Equal(t, build, relativeBuild) t.Log("Changing only a payload path changes snapshot identity") require.NoError(t, os.Rename(filepath.Join(buildDir, "part-0/chip-0.gas"), filepath.Join(buildDir, "part-0/chip-1.gas"))) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index ee83238f7edc..e417a86b924f 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -84,7 +84,6 @@ func TestRenderHybridPreservesRuntimeEnvironment(t *testing.T) { fixture.partitions[0].devicesPerNode = 8 normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) build := normalized.build - build.BatchSize = 4 build.CompilationMode = BuildCompilationModeHybrid build.IOFPGACount = 2 build.IOFanoutFactor = 2 diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 4a739cd589e9..aefe4218ae54 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -45,47 +45,27 @@ func renderLPUConfigMap( materializationName string, modelStoragePath string, projections []*ModelProjection, - agents []ExpectedAgent, ) (*corev1.ConfigMap, error) { - modelConfig, err := lpuModelConfig(projections, modelStoragePath) - if err != nil { - return nil, fmt.Errorf("render model_config.toml: %w", err) - } - var modelTOML strings.Builder - if err := toml.NewEncoder(&modelTOML).Encode(modelConfig); err != nil { - return nil, fmt.Errorf("render model_config.toml: %w", err) - } - - datacenterRacks := make(map[string]any, len(agents)) - for _, agent := range agents { - availableNodes := make([]int, agent.Replicas) - for index := range availableNodes { - availableNodes[index] = index - } - datacenterRacks[agent.TemplateName] = map[string]any{ - "available_nodes": availableNodes, - "preferred_start": 0, - "node_name_template": fmt.Sprintf( - "%s-%s-{node}.${GROVE_HEADLESS_SERVICE}", - "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}", - agent.TemplateName, - ), - } - } - var datacenterTOML strings.Builder - if err := toml.NewEncoder(&datacenterTOML).Encode(map[string]any{ - "model_base_paths": []string{modelStoragePath}, - "datacenters": map[string]any{"racks": datacenterRacks}, - }); err != nil { - return nil, fmt.Errorf("render datacenter.toml: %w", err) - } - data := resolvedPartitionData(projections) if projections[0].pipeline == PipelineLPX { - data["gas_dir"] = modelConfig["iop"].(map[string]any)["model_path"].(string) + // Direct agents consume the build directory and partition table. + modelPath, err := buildRuntimePath(lpuRuntimeBuildRef(projections[0], modelStoragePath), modelStoragePath) + if err != nil { + return nil, fmt.Errorf("resolve gas_dir: %w", err) + } + data["gas_dir"] = modelPath + } else { + // Conductors consume deployment overrides alongside compile-owned manifest defaults. + modelConfig, err := lpuModelConfig(projections, modelStoragePath) + if err != nil { + return nil, fmt.Errorf("render model_config.toml: %w", err) + } + var modelTOML strings.Builder + if err := toml.NewEncoder(&modelTOML).Encode(modelConfig); err != nil { + return nil, fmt.Errorf("render model_config.toml: %w", err) + } + data["model_config.toml"] = modelTOML.String() } - data["model_config.toml"] = modelTOML.String() - data["datacenter.toml"] = datacenterTOML.String() return renderRuntimeConfigMap(namespace, materializationName+"-lpu", data) } @@ -167,7 +147,6 @@ func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (ma if err != nil { return nil, err } - delete(setup, "setup_ops_format") delete(setup, "resolved_partitions_dir") if len(setup) > 0 { config["setup"] = setup @@ -187,7 +166,6 @@ func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (ma // limited to the operator-owned LPU runtime bindings so Nova can inherit the // parent timeouts exactly as it did with the legacy pipeline field. draft["setup"] = map[string]any{ - "setup_ops_format": agentSetupOpsFormat, "resolved_partitions_dir": lpuConfigMountPath, } target, err := nestedLPUModelConfig(projections[draftCount], modelStoragePath) @@ -197,10 +175,6 @@ func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (ma config["type"] = "SpecDecode" config["draft"] = draft config["target"] = target - config["num_drafts"] = draftCount - setConfigDefault(config, "draft_to_target_port", 23456) - setConfigDefault(config, "target_to_draft_port", 23457) - setConfigDefault(config, "head_to_head_port", 23458) return config, nil } @@ -212,9 +186,6 @@ func nestedLPUModelConfig(projection *ModelProjection, modelStoragePath string) if err != nil { return nil, err } - if scheduler == nil { - scheduler = map[string]any{} - } delete(settings, "scheduler") setup, err := modelObjectSetting(projection, "setup") if err != nil { @@ -226,7 +197,6 @@ func nestedLPUModelConfig(projection *ModelProjection, modelStoragePath string) delete(settings, "setup") // Keep operator-owned LPU runtime setup bindings authoritative. - setup["setup_ops_format"] = agentSetupOpsFormat setup["resolved_partitions_dir"] = lpuConfigMountPath delete(settings, v3MaxSWADKVCBlocksDraft) @@ -241,48 +211,29 @@ func nestedLPUModelConfig(projection *ModelProjection, modelStoragePath string) delete(settings, "extra_programs") if projection.configuredBuild.Family == BuildFamilyXT && projection.pipeline == PipelineSingle && len(extraPrograms) > 0 { - return nil, fmt.Errorf("model %q settings.extra_programs is not supported with setup_ops_format %q", projection.model, agentSetupOpsFormat) + return nil, fmt.Errorf("model %q settings.extra_programs is not supported for XT Single", projection.model) } } build := &projection.configuredBuild buildRef := build.Path - if projection.pipeline == PipelineLPX || - projection.configuredBuild.Family == BuildFamilyHX { + if projection.configuredBuild.Family == BuildFamilyHX { buildRef = lpuRuntimeBuildRef(projection, modelStoragePath) } modelPath, err := buildRuntimePath(buildRef, modelStoragePath) if err != nil { return nil, fmt.Errorf("model %q model_path: %w", projection.model, err) } - if _, overridden := settings["tokenizer_path"]; !overridden && - strings.TrimSpace(build.RuntimeTokenizerPath) != "" { - settings["tokenizer_path"] = filepath.Join(modelPath, build.RuntimeTokenizerPath) - } - if projection.configuredBuild.Family == BuildFamilyHX || - projection.pipeline == PipelineSingle { - if err := validateRuntimeTokenizerSettings(settings); err != nil { - return nil, fmt.Errorf("model %q settings: %w", projection.model, err) - } - } - if build.SupportsCPUEmbeddings { - if enabled, _ := settings["cpu_embeddings"].(bool); enabled { - if _, overridden := settings["embedding_path"]; !overridden && - strings.TrimSpace(build.RuntimeTokenEmbeddingsPath) != "" { - settings["embedding_path"] = filepath.Join(modelPath, build.RuntimeTokenEmbeddingsPath) - } - } - } - if projection.configuredBuild.Family == BuildFamilyXT && - projection.pipeline == PipelineSpecDecode { - setConfigDefault(settings, "model_path", modelPath) - } else { + if _, overridden := settings["model_path"]; !overridden || + projection.configuredBuild.Family != BuildFamilyXT || projection.pipeline != PipelineSpecDecode { settings["model_path"] = modelPath } config := map[string]any{ - "scheduler": scheduler, - "setup": setup, - "iop": settings, + "setup": setup, + "iop": settings, + } + if scheduler != nil { + config["scheduler"] = scheduler } if extraPrograms != nil { config["extra_programs"] = extraPrograms @@ -380,12 +331,6 @@ func resolvedPartitionData(projections []*ModelProjection) map[string]string { return data } -func setConfigDefault(values map[string]any, key string, value any) { - if _, present := values[key]; !present { - values[key] = value - } -} - func withLPUConfigVolume(spec *corev1.PodSpec, configMapName string, allowOverrides bool) error { found := false for _, volume := range spec.Volumes { diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 741063e87b5e..2b30a1f69b48 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -109,28 +109,22 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { "test-dgd", "/models", []*ModelProjection{projection}, - nil, ) t.Log("Keep gas_dir rooted at the selected image-owned build") require.NoError(t, err) - require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") + require.NotContains(t, configMap.Data, "model_config.toml") + require.NotContains(t, configMap.Data, "datacenter.toml") require.Equal(t, "/models/model-build", configMap.Data["gas_dir"]) require.True(t, *configMap.Immutable) require.Equal(t, LPUConfigMapName("test-dgd", LPUConfigMapHash(configMap)), configMap.Name) - t.Log("Reject an LPX setting that makes the rendered ConfigMap exceed the API limit") + t.Log("Ignore model settings that are not consumed by the direct runtime") projection.configuredBuild.runtimeSettings["oversized"] = strings.Repeat("x", corev1.MaxSecretSize) - _, err = renderLPUConfigMap( - "test-namespace", - "test-dgd", - "/models", - []*ModelProjection{projection}, - nil, - ) - require.ErrorContains(t, err, "rendered LPX ConfigMap") - require.ErrorContains(t, err, "maximum is 1048576") - delete(projection.configuredBuild.runtimeSettings, "oversized") + projection.configuredBuild.runtimeSettings["scheduler"] = "not-a-Nova-scheduler" + unchanged, err := renderLPUConfigMap("test-namespace", "test-dgd", "/models", []*ModelProjection{projection}) + require.NoError(t, err) + require.Equal(t, configMap, unchanged) t.Log("Render the preserved runtime with an invalid snapshot build reference") projection.configuredBuild.Path = "relative-build" @@ -139,11 +133,10 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { "test-dgd", "/models", []*ModelProjection{projection}, - nil, ) t.Log("Return the model-path error before projecting gas_dir") - require.ErrorContains(t, err, "render model_config.toml: model \"default\" model_path: parse build path \"relative-build\": ref \"relative-build\" must be an absolute path or URL") + require.ErrorContains(t, err, "resolve gas_dir: parse build path \"relative-build\": ref \"relative-build\" must be an absolute path or URL") } func TestResolvedPartitionDataOmitsXTModelColumnsBeforeMaterialization(t *testing.T) { @@ -199,14 +192,14 @@ func TestV3SingleConfigUsesManifestTopologyAndArbitraryModelSettings(t *testing. "test-dgd", "/models", []*ModelProjection{projection}, - []ExpectedAgent{{TemplateName: "rack", Replicas: 1}}, ) require.NoError(t, err) t.Log("Verify the generated files need no manual topology or model-setting patches") require.Equal(t, "part-1", configMap.Data["partition_paths"]) require.Equal(t, v3OpaqueTopology, configMap.Data["topologies"]) - require.Contains(t, configMap.Data["model_config.toml"], "sequence_length = 8192") + require.NotContains(t, configMap.Data["model_config.toml"], "sequence_length") + require.NotContains(t, configMap.Data, "datacenter.toml") require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") require.Contains(t, configMap.Data["model_config.toml"], "batch_size = 4") require.Contains(t, configMap.Data["model_config.toml"], "custom_runtime_knob = 'enabled'") @@ -215,34 +208,24 @@ func TestV3SingleConfigUsesManifestTopologyAndArbitraryModelSettings(t *testing. require.Contains(t, configMap.Data["model_config.toml"], "stop_tokens = [1]") } -func TestNestedLPUModelConfigJoinsRuntimeAssetsToRemappedRoot(t *testing.T) { - t.Log("Construct a projection with build-relative runtime assets") +func TestNestedLPUModelConfigPreservesExplicitOverrides(t *testing.T) { + t.Log("Construct a model with deployment overrides and a remapped runtime path") projection := &ModelProjection{ - model: "default", pipeline: PipelineLPX, runtimeBuildRef: "model-build", + model: "default", pipeline: PipelineSingle, runtimeBuildRef: "model-build", configuredBuild: Build{ - Path: "file:///snapshot-build", SupportsCPUEmbeddings: true, - RuntimeTokenizerPath: ".", RuntimeTokenEmbeddingsPath: "runtime/text_embeddings.npz", - runtimeSettings: map[string]any{"cpu_embeddings": true}, + Path: "file:///snapshot-build", Family: BuildFamilyHX, + runtimeSettings: map[string]any{"tokenizer_path": "/custom/tokenizer", "batch_size": 4}, }, } - t.Log("Render model configuration under the remapped runtime root") + t.Log("Leave manifest defaults to the runtime while retaining authored overrides") config, err := nestedLPUModelConfig(projection, "/models") - - t.Log("Verify default runtime asset paths are joined to the remapped build") - require.NoError(t, err) - iop := config["iop"].(map[string]any) - require.Equal(t, "/models/model-build", iop["model_path"]) - require.Equal(t, "/models/model-build", iop["tokenizer_path"]) - require.Equal(t, "/models/model-build/runtime/text_embeddings.npz", iop["embedding_path"]) - - t.Log("Render an explicit absolute embedding-path override") - projection.configuredBuild.runtimeSettings["embedding_path"] = "/custom/embeddings" - config, err = nestedLPUModelConfig(projection, "/models") - - t.Log("Verify the absolute override remains unchanged") require.NoError(t, err) - require.Equal(t, "/custom/embeddings", config["iop"].(map[string]any)["embedding_path"]) + require.Equal(t, map[string]any{ + "model_path": "/models/model-build", "tokenizer_path": "/custom/tokenizer", "batch_size": 4, + }, config["iop"]) + require.Equal(t, map[string]any{"resolved_partitions_dir": "/configs"}, config["setup"]) + require.NotContains(t, config, "scheduler") } func TestV2SingleConfigRejectsExtraPrograms(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go index abe19bd5ed0d..b22d2dbbe89c 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go @@ -17,50 +17,6 @@ import ( const hxTopologyFamily = "16x8x2x3" -func addManifestTextDefault( - defaults map[string]any, - key string, - subject string, - has bool, - read func() (string, error), -) error { - if !has { - return nil - } - value, err := read() - if err != nil { - return fmt.Errorf("reading %s: %w", subject, err) - } - value = strings.TrimSpace(value) - if value != "" { - defaults[key] = value - } - return nil -} - -func childSwaDefaults(defaults map[string]any) map[string]any { - child, _ := defaults["swa"].(map[string]any) - if child == nil { - child = make(map[string]any) - defaults["swa"] = child - } - return child -} - -func capnpUInt32List(values capnp.UInt32List) []uint32 { - seen := make(map[uint32]struct{}, values.Len()) - out := make([]uint32, 0, values.Len()) - for i := 0; i < values.Len(); i++ { - value := values.At(i) - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - out = append(out, value) - } - return out -} - func classifyManifestPartitions(filename string, partitions []BuildPartition, partSelect bool) (BuildFamily, int, int, error) { family := BuildFamilyXT packagedNodes, partitionZeroNodes := 0, 0 diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go index b831714f5804..20307d63f664 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -97,9 +97,12 @@ func buildFromGbuildManifestV2(buildRef string, manifest manifestcapnpv2.Manifes ioFanoutFactor, ) } - defaults, tokenizerPath, err := defaultsFromManifestV2(manifest, program, compilationMode, chains) - if err != nil { - return nil, err + var settings map[string]any + if compilationMode == BuildCompilationModeLPUOnly { + settings, err = runtimeSettingsFromManifestV2(manifest, program) + if err != nil { + return nil, err + } } if !manifest.HasArtifacts() { return nil, fmt.Errorf("%s is missing artifacts", gbuildManifestV2CapnpFile) @@ -121,17 +124,14 @@ func buildFromGbuildManifestV2(buildRef string, manifest manifestcapnpv2.Manifes } build := &Build{ - Path: buildRef, - CompilationMode: compilationMode, - BatchSize: batchSize, - SelectedPropSyncChains: chains, - StandaloneTokenEmbeddings: program.StandaloneTokenEmbeddings(), - SupportsCPUEmbeddings: program.SupportsCpuEmbeddings(), - RuntimeTokenEmbeddingsPath: runtimeTokenEmbeddingsPath, - IOFPGACount: ioFPGACount, - IOFanoutFactor: ioFanoutFactor, - runtimeSettings: defaults, - RuntimeTokenizerPath: tokenizerPath, + Path: buildRef, + CompilationMode: compilationMode, + SelectedPropSyncChains: chains, + StandaloneTokenEmbeddings: program.StandaloneTokenEmbeddings(), + SupportsCPUEmbeddings: program.SupportsCpuEmbeddings(), + IOFPGACount: ioFPGACount, + IOFanoutFactor: ioFanoutFactor, + runtimeSettings: settings, } // Complete the normalized build with scheduler-facing LPU artifacts. @@ -203,123 +203,48 @@ func selectedPropSyncChainsFromManifestV2(deployment manifestcapnpv2.DeploymentI return chains, nil } -func defaultsFromManifestV2( - manifest manifestcapnpv2.Manifest, - program manifestcapnpv2.ProgramConfig, - compilationMode BuildCompilationMode, - chains [][]int, -) (map[string]any, string, error) { - defaults := make(map[string]any) - tokenizerPath, err := addModelDefaultsFromManifestV2(defaults, manifest) - if err != nil { - return nil, "", err - } - defaults["batch_size"] = int64(program.BatchSize()) - if err := addManifestTextDefault(defaults, "program_name", gbuildManifestV2CapnpFile+" deployment.program.programName", program.HasProgramName(), program.ProgramName); err != nil { - return nil, "", err - } - if err := addManifestTextDefault(defaults, "package_name", gbuildManifestV2CapnpFile+" deployment.program.packageName", program.HasPackageName(), program.PackageName); err != nil { - return nil, "", err - } - defaults["sequence_length"] = int64(program.SequenceLength()) - defaults["input_size"] = int64(program.InputSize()) - defaults["output_size"] = int64(program.OutputSize()) - defaults["num_kv_caches"] = int64(program.NumKvCaches()) - defaults["num_batch_split_divisions"] = int64(program.NumBatchSplitDivisions()) - defaults["batch_folding"] = program.BatchFolding() - defaults["cpu_embeddings"] = program.SupportsCpuEmbeddings() - if program.HasTopk() { - topk, err := program.Topk() +func runtimeSettingsFromManifestV2(manifest manifestcapnpv2.Manifest, program manifestcapnpv2.ProgramConfig) (map[string]any, error) { + // Retain only settings whose omission would change the runtime's interpretation. + settings := map[string]any{ + "batch_folding": program.BatchFolding(), + "num_batch_split_divisions": int64(program.NumBatchSplitDivisions()), + } + var archVocabulary uint32 + hasSWA := false + if manifest.HasModel() { + model, err := manifest.Model() if err != nil { - return nil, "", fmt.Errorf("reading %s deployment.program.topk: %w", gbuildManifestV2CapnpFile, err) + return nil, fmt.Errorf("reading %s model: %w", gbuildManifestV2CapnpFile, err) } - defaults["topk"] = int64(topk.Value()) - } - if program.EnableLogitMasking() { - defaults["enable_logit_masking"] = true - } - defaults["dkvc"] = program.HasDkvc() - if program.HasDkvc() { - dkvc, err := program.Dkvc() - if err != nil { - return nil, "", fmt.Errorf("reading %s deployment.program.dkvc: %w", gbuildManifestV2CapnpFile, err) + if model.HasArch() { + arch, err := model.Arch() + if err != nil { + return nil, fmt.Errorf("reading %s model.arch: %w", gbuildManifestV2CapnpFile, err) + } + archVocabulary = arch.VocabSize() + hasSWA = arch.HasSwa() } - defaults["num_dkvc_blocks"] = int64(dkvc.NumBlocksPerKvCache()) - } - defaults["prop_sync"] = compilationMode == BuildCompilationModeLPUOnly && len(chains) > 0 - if _, hasSWA := defaults["swa"]; hasSWA || program.SwaChunked() || program.NumSwaDkvcBlocks() != 0 { - swaDefaults := childSwaDefaults(defaults) - swaDefaults["chunked"] = program.SwaChunked() - swaDefaults["num_swa_dkvc_blocks"] = int64(program.NumSwaDkvcBlocks()) - } - return defaults, tokenizerPath, nil -} -func addModelDefaultsFromManifestV2(defaults map[string]any, manifest manifestcapnpv2.Manifest) (string, error) { - if !manifest.HasModel() { - return "", nil - } - model, err := manifest.Model() - if err != nil { - return "", fmt.Errorf("reading %s model: %w", gbuildManifestV2CapnpFile, err) - } - if model.HasArch() { - arch, err := model.Arch() - if err != nil { - return "", fmt.Errorf("reading %s model.arch: %w", gbuildManifestV2CapnpFile, err) - } - defaults["num_layers"] = int64(arch.NLayers()) - defaults["vocab_size"] = int64(arch.VocabSize()) - if arch.HasSwa() { - swa, err := arch.Swa() + // Tokenizer vocabulary takes precedence over architecture padding in the deployment contract. + if model.HasTokenizer() { + tokenizer, err := model.Tokenizer() if err != nil { - return "", fmt.Errorf("reading %s model.arch.swa: %w", gbuildManifestV2CapnpFile, err) + return nil, fmt.Errorf("reading %s model.tokenizer: %w", gbuildManifestV2CapnpFile, err) } - swaDefaults := childSwaDefaults(defaults) - swaDefaults["swa_ctx_len"] = int64(swa.MaxSeqLen()) - swaDefaults["swa_padding_len"] = int64(swa.Padding()) - if swa.HasNumUsers() { - numUsers, err := swa.NumUsers() - if err != nil { - return "", fmt.Errorf("reading %s model.arch.swa.numUsers: %w", gbuildManifestV2CapnpFile, err) - } - swaDefaults["swa_num_users"] = int64(numUsers.Value()) + if vocabulary := tokenizer.VocabSize(); vocabulary != 0 && vocabulary != archVocabulary { + settings["vocab_size"] = int64(vocabulary) } } } - if !model.HasTokenizer() { - return "", nil - } - tokenizer, err := model.Tokenizer() - if err != nil { - return "", fmt.Errorf("reading %s model.tokenizer: %w", gbuildManifestV2CapnpFile, err) - } - if tokenizer.VocabSize() != 0 { - defaults["vocab_size"] = int64(tokenizer.VocabSize()) - } - tokenizerPath := "" - if tokenizer.HasPath() { - tokenizerPath, err = tokenizer.Path() - if err != nil { - return "", fmt.Errorf("reading %s model.tokenizer.path: %w", gbuildManifestV2CapnpFile, err) - } - tokenizerPath, err = cleanManifestRelativeBuildPath( - fmt.Sprintf("%s model.tokenizer.path", gbuildManifestV2CapnpFile), - tokenizerPath, - true, - ) - if err != nil { - return "", err - } - } - if tokenizer.HasStopTokens() { - stopTokens, err := tokenizer.StopTokens() - if err != nil { - return "", fmt.Errorf("reading %s model.tokenizer.stopTokens: %w", gbuildManifestV2CapnpFile, err) + + // Program-only SWA settings are not inferred when the architecture has no SWA descriptor. + if !hasSWA && (program.SwaChunked() || program.NumSwaDkvcBlocks() != 0) { + settings["swa"] = map[string]any{ + "chunked": program.SwaChunked(), + "num_swa_dkvc_blocks": int64(program.NumSwaDkvcBlocks()), } - defaults["stop_tokens"] = capnpUInt32List(stopTokens) } - return tokenizerPath, nil + return settings, nil } func addLPUArtifactsFromManifestV2( diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go index 308ef88ceb2c..97c32d8d3ae1 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -42,6 +42,80 @@ func TestBuildFromGbuildManifestV2ProjectsRuntimeIO(t *testing.T) { require.ErrorContains(t, err, "contractRevision") } +func TestBuildFromGbuildManifestV2RetainsRuntimeOverrides(t *testing.T) { + t.Parallel() + + t.Log("Keep only compiler metadata whose runtime default would differ") + for _, test := range []struct { + name string + architectureVocabulary uint32 + tokenizerVocabulary uint32 + architectureSWA bool + programSWA bool + want map[string]any + }{ + {name: "matching vocabulary", architectureVocabulary: 128, tokenizerVocabulary: 128}, + {name: "architecture vocabulary only", architectureVocabulary: 128}, + {name: "tokenizer vocabulary only", tokenizerVocabulary: 128, want: map[string]any{"vocab_size": int64(128)}}, + {name: "tokenizer overrides architecture padding", architectureVocabulary: 256, tokenizerVocabulary: 128, want: map[string]any{"vocab_size": int64(128)}}, + {name: "architecture SWA is runtime-owned", architectureSWA: true, programSWA: true}, + {name: "program-only SWA defaults", programSWA: true, want: map[string]any{"swa": map[string]any{ + "chunked": true, "num_swa_dkvc_blocks": int64(2), + }}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Encode compiler values and the selected vocabulary/SWA boundary") + manifest := newManifestV2ContractFixture(t) + deployment, err := manifest.Deployment() + require.NoError(t, err) + deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpuOnly) + program, err := deployment.Program() + require.NoError(t, err) + require.NoError(t, program.SetProgramName("program")) + require.NoError(t, program.SetPackageName("package")) + program.SetSequenceLength(4096) + program.SetInputSize(1) + program.SetOutputSize(1) + program.SetNumKvCaches(8) + program.SetSwaChunked(test.programSWA) + if test.programSWA { + program.SetNumSwaDkvcBlocks(2) + } + model, err := manifest.NewModel() + require.NoError(t, err) + if test.architectureVocabulary != 0 || test.architectureSWA { + arch, err := model.NewArch() + require.NoError(t, err) + arch.SetVocabSize(test.architectureVocabulary) + arch.SetNLayers(12) + if test.architectureSWA { + swa, err := arch.NewSwa() + require.NoError(t, err) + swa.SetMaxSeqLen(128) + swa.SetPadding(32) + } + } + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + tokenizer.SetVocabSize(test.tokenizerVocabulary) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stops, err := tokenizer.NewStopTokens(1) + require.NoError(t, err) + stops.Set(0, 2) + + t.Log("Preserve required overrides without copying settings the runtime reads itself") + build, err := buildFromGbuildManifestV2("gs://models/build", manifest) + require.NoError(t, err) + require.NoError(t, resolveBuildSettings(build, nil)) + if test.want == nil { + require.Empty(t, build.runtimeSettings) + } else { + require.Equal(t, test.want, build.runtimeSettings) + } + }) + } +} + func TestNormalizeBuildSnapshotRejectsInvalidHXArtifacts(t *testing.T) { t.Log("Define malformed HX artifact inventories") tests := []struct { @@ -199,12 +273,11 @@ func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { t.Log("Define revision-2 path, XT geometry, node-count, and embedding contracts") tests := []struct { - name, partitionPath, tokenizerPath, wantErr, wantAsset string - geometry *geometryFixture - embedding *embeddingFixture + name, partitionPath, wantErr string + geometry *geometryFixture + embedding *embeddingFixture }{ {name: "partition traversal", partitionPath: "../outside", wantErr: "LPU partition 0 path"}, - {name: "absolute tokenizer", tokenizerPath: "/outside", wantErr: "model.tokenizer.path"}, { name: "runtime asset line break", embedding: &embeddingFixture{path: "runtime\nasset", cpu: true}, wantErr: "artifacts.runtimeAssets.tokenEmbeddingsPath", @@ -229,7 +302,6 @@ func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { }, { name: "valid standalone asset", embedding: &embeddingFixture{path: "runtime/text_embeddings.npz", cpu: true, standalone: true}, - wantAsset: "runtime/text_embeddings.npz", }, } @@ -257,13 +329,6 @@ func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { deployment.SetNumLpuNodes(test.geometry.numNodes) } } - if test.tokenizerPath != "" { - model, err := manifest.NewModel() - require.NoError(t, err) - tokenizer, err := model.NewTokenizer() - require.NoError(t, err) - require.NoError(t, tokenizer.SetPath(test.tokenizerPath)) - } if test.embedding != nil { deployment, err := manifest.Deployment() require.NoError(t, err) @@ -286,13 +351,12 @@ func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { } t.Log("Accept only the safe and internally consistent revision-2 contract") - build, err := buildFromGbuildManifestV2("gs://models/build", manifest) + _, err := buildFromGbuildManifestV2("gs://models/build", manifest) if test.wantErr != "" { require.ErrorContains(t, err, test.wantErr) return } require.NoError(t, err) - require.Equal(t, test.wantAsset, build.RuntimeTokenEmbeddingsPath) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 9bbcd40c2bed..4f003d0e10b2 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -92,7 +92,7 @@ func RenderSelectedNodeLocal( } modelStorage.volume = *modelStorage.volume.DeepCopy() modelStorage.mount = *modelStorage.mount.DeepCopy() - configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStorage.mount.MountPath, projections, plan.Agents) + configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStorage.mount.MountPath, projections) if err != nil { return nil, err } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 582cb0dc99e7..9f1a360772e7 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -14,7 +14,6 @@ import ( manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -268,6 +267,9 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { conductorTemplate.Labels = map[string]string{"owner": "conductor"} conductorTemplate.Annotations = map[string]string{"owner": "conductor"} conductorTemplate.Spec.Containers[0].Image = "conductor-runtime" + conductorTemplate.Spec.Containers[0].Env = []corev1.EnvVar{{ + Name: "NOVA_NODE_NAME_TEMPLATE", Value: "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE}", + }} before := source.DeepCopy() t.Log("Resolve and render the authored speculative workload") @@ -313,7 +315,10 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { require.Contains(t, configMap.Data["model_config.toml"], "SpecDecode") require.Contains(t, configMap.Data["model_config.toml"], "[draft") require.Contains(t, configMap.Data["model_config.toml"], "[target") - require.Contains(t, configMap.Data["model_config.toml"], "num_drafts = 2") + require.NotContains(t, configMap.Data["model_config.toml"], "num_drafts") + require.NotContains(t, configMap.Data["model_config.toml"], "draft_to_target_port") + require.NotContains(t, configMap.Data["model_config.toml"], "target_to_draft_port") + require.NotContains(t, configMap.Data["model_config.toml"], "head_to_head_port") require.Contains(t, configMap.Data["model_config.toml"], "max_swa_dkvc_blocks_draft = 2") require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_connect_timeout = '30s'")) require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_setup_timeout = '180s'")) @@ -324,23 +329,16 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { t.Log("Bind conductor allocation and Nova hostnames to the renamed Agent cliques") require.Equal(t, "agt0:agt1:agt2", testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, allocationEnvVar)) require.Equal(t, []string{"agt0", "agt1", "agt2"}, conductor.Spec.StartsAfter) - var datacenter struct { - Datacenters struct { - Racks map[string]struct { - NodeNameTemplate string `toml:"node_name_template"` - } - } - } - require.NoError(t, toml.Unmarshal([]byte(configMap.Data["datacenter.toml"]), &datacenter)) - require.Len(t, datacenter.Datacenters.Racks, len(plan.Agents)) + require.NotContains(t, configMap.Data, "datacenter.toml") + nodeNameTemplate := testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, "NOVA_NODE_NAME_TEMPLATE") for _, agent := range plan.Agents { - rack := datacenter.Datacenters.Racks[agent.TemplateName] hostname := strings.NewReplacer( "${GROVE_PCSG_NAME}", plan.LPXScalingGroup, "${GROVE_PCSG_INDEX}", "0", + "{rack}", agent.TemplateName, "{node}", "0", "${GROVE_HEADLESS_SERVICE}", pcs.Name, - ).Replace(rack.NodeNameTemplate) + ).Replace(nodeNameTemplate) require.Equal(t, agent.CliqueName+"-0."+pcs.Name, hostname) } } diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index f69a46e0c76e..11d4fe63284c 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -258,11 +258,11 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { _, err = workloadSetDigest([]*ModelProjection{projections[0], &mixedFamily}) require.ErrorContains(t, err, "mixed target families") - t.Log("Preserve family-specific runtime settings in every logical projection") + t.Log("Preserve compiled placement without repeating runtime-derived model settings") for _, projection := range projections { require.EqualValues(t, compiledAgentCount, projection.agentReplicas) if test.family == BuildFamilyHX { - require.EqualValues(t, 8192, projection.configuredBuild.runtimeSettings["sequence_length"]) + require.NotContains(t, projection.configuredBuild.runtimeSettings, "sequence_length") } } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 7e762c548de8..614a85619fd3 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -21,13 +21,6 @@ func configureDirectHybridAgentRuntime( agent := configureAgentIdentity(agentPodSpec) agent.Env = append(agent.Env, - corev1.EnvVar{ - Name: "TOPOLOGIES", - ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: lpuConfigMapName}, - Key: "topologies", - }}, - }, corev1.EnvVar{ Name: "GAS_DIR", ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings.go b/deploy/operator/internal/dynamo/lpx/runtime_settings.go index 3b007238a871..571ef2d00382 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings.go @@ -18,9 +18,6 @@ import ( // maxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. const maxSpecDecodeNumDrafts = 8 -// Preserve the setup-ops discriminator consumed by the LPU runtime image. -const agentSetupOpsFormat = "agent_v2" - // resolveBuildSettings requires a nonnil build; it normalizes and consumes modelSettings, which callers must not reuse. func resolveBuildSettings(build *Build, modelSettings map[string]any) error { // Preserve normalized defaults before reusing the compiler-owned field for model overrides. @@ -41,9 +38,12 @@ func resolveBuildSettings(build *Build, modelSettings map[string]any) error { } // Preserve the selected prop-sync and embedding transformations on the merged settings. - cpuEmbeddings, err := boolSetting(settings, "cpu_embeddings") - if err != nil { - return fmt.Errorf("settings.%w", err) + cpuEmbeddings := build.SupportsCPUEmbeddings + if _, overridden := settings["cpu_embeddings"]; overridden { + cpuEmbeddings, err = boolSetting(settings, "cpu_embeddings") + if err != nil { + return fmt.Errorf("settings.%w", err) + } } if err := build.consumeRuntimeSelectedPropSyncChain(); err != nil { return err @@ -110,43 +110,6 @@ func mergeRuntimeSettingOverride(base, override any, path string) (any, error) { } } -// validateRuntimeTokenizerSettings validates and normalizes tokenizer settings shared by V2 and V3. -func validateRuntimeTokenizerSettings(iop map[string]any) error { - tokenizerPath, ok := iop["tokenizer_path"] - if !ok { - return fmt.Errorf("iop.tokenizer_path is required; set spec.components[].lpx.settings.tokenizer_path when the capnp manifest does not provide model.tokenizer.path") - } - path, ok := tokenizerPath.(string) - path = strings.TrimSpace(path) - if !ok || path == "" { - return fmt.Errorf("iop.tokenizer_path must be a non-empty string") - } - iop["tokenizer_path"] = path - - stopTokens, ok := iop["stop_tokens"] - if !ok { - return fmt.Errorf("iop.stop_tokens is required; set spec.components[].lpx.settings.stop_tokens when the capnp manifest does not provide model.tokenizer.stopTokens") - } - var normalized []uint32 - switch tokens := stopTokens.(type) { - case []uint32: - normalized = append([]uint32(nil), tokens...) - case []any: - normalized = make([]uint32, len(tokens)) - for i, value := range tokens { - integer, ok := value.(int64) - if !ok || integer < 0 || uint64(integer) > uint64(^uint32(0)) { - return fmt.Errorf("iop.stop_tokens[%d] must be an integer between 0 and %d", i, uint64(^uint32(0))) - } - normalized[i] = uint32(integer) - } - default: - return fmt.Errorf("iop.stop_tokens must be an array") - } - iop["stop_tokens"] = normalized - return nil -} - // normalizeLegacyNovaSettings validates legacy Agent V2 controls and removes // their accepted defaults. It consumes caller-owned settings without touching build geometry. func normalizeLegacyNovaSettings(iop map[string]any) error { @@ -156,10 +119,10 @@ func normalizeLegacyNovaSettings(iop map[string]any) error { return fmt.Errorf("iop.batch_folding must be a boolean") } if batchFolding { - return fmt.Errorf("iop.batch_folding=true is not supported with setup_ops_format %q", agentSetupOpsFormat) + return fmt.Errorf("iop.batch_folding=true is not supported") } - // Accept only zero or one, including JSON numbers from unresolved V2 LPX settings. + // Accept only zero or one, preserving exact authored JSON numbers. if value, ok := iop["num_batch_split_divisions"]; ok { var numBatchSplitDivisions *big.Rat switch typed := value.(type) { @@ -174,11 +137,7 @@ func normalizeLegacyNovaSettings(iop map[string]any) error { return fmt.Errorf("iop.num_batch_split_divisions must be an integer") } if numBatchSplitDivisions.Sign() < 0 || numBatchSplitDivisions.Cmp(big.NewRat(1, 1)) > 0 { - return fmt.Errorf( - "iop.num_batch_split_divisions=%v is not supported with setup_ops_format %q", - value, - agentSetupOpsFormat, - ) + return fmt.Errorf("iop.num_batch_split_divisions=%v is not supported", value) } } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go index 1234eff839dd..c70703705612 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go @@ -19,27 +19,32 @@ func TestResolveBuildSettingsSelectedPropSyncChainWithCPUEmbeddings(t *testing.T t.Parallel() for _, test := range []struct { - name string - cpuSupported, standalone bool - firstPartition int + name string + cpuSupported, standalone, cpuDisabled bool + firstPartition int }{ - {"standalone CPU embeddings", true, true, 1}, - {"CPU embeddings unsupported", false, true, 0}, - {"partition zero is runnable", true, false, 0}, + {"standalone CPU embeddings", true, true, false, 1}, + {"CPU embeddings unsupported", false, true, false, 0}, + {"partition zero is runnable", true, false, false, 0}, + {"explicitly disabled CPU embeddings", true, true, true, 0}, } { t.Run(test.name, func(t *testing.T) { t.Log("Construct a selected chain with independent embedding placement requirements") build := selectedPropSyncBuildWithTopologies(t, []string{registryTestTopology, registryTestTopology, registryTestTopology}, [][]int{{0, 1, 2}}) build.SupportsCPUEmbeddings, build.StandaloneTokenEmbeddings = test.cpuSupported, test.standalone - build.runtimeSettings = map[string]any{"cpu_embeddings": true} + settings := map[string]any{} + if test.cpuDisabled { + settings["cpu_embeddings"] = false + } partitions := slices.Clone(build.Partitions) configured := *build t.Log("Resolve the selected chain and omit partition zero only when both flags permit it") - require.NoError(t, resolveBuildSettings(&configured, nil)) + require.NoError(t, resolveBuildSettings(&configured, settings)) require.Equal(t, partitions[test.firstPartition:], configured.Partitions) require.Empty(t, configured.SelectedPropSyncChains) + require.Equal(t, settings, configured.runtimeSettings) t.Log("Keep the original partition IDs, paths and selected chain unchanged") require.Equal(t, partitions, build.Partitions) @@ -191,7 +196,7 @@ func TestResolveBuildSettingsDeepMergesDefaultsAndOverrides(t *testing.T) { func TestProjectModelLegacyNovaSettings(t *testing.T) { t.Parallel() - t.Log("Capture the merged HX and direct XT-hybrid settings paths without legacy overrides") + t.Log("Capture the Nova and hybrid settings paths before runtime-specific overrides") singleIntent := ModelProjectionInput{ Models: []string{"default"}, Pipeline: PipelineSingle, BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), @@ -220,19 +225,20 @@ func TestProjectModelLegacyNovaSettings(t *testing.T) { {`{"num_batch_split_divisions":1.0}`, "", false}, {`{"num_batch_split_divisions":10e-1}`, "", false}, {`{"num_batch_split_divisions":-0.0}`, "", false}, - {`{"batch_folding":true}`, `iop.batch_folding=true is not supported with setup_ops_format "agent_v2"`, false}, + {`{"batch_folding":true}`, `iop.batch_folding=true is not supported`, false}, {`{"batch_folding":"false"}`, "iop.batch_folding must be a boolean", false}, - {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported with setup_ops_format "agent_v2"`, false}, - {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported with setup_ops_format "agent_v2"`, false}, - {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported with setup_ops_format "agent_v2"`, false}, + {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported`, false}, + {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported`, false}, + {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported`, false}, {`{"num_batch_split_divisions":0.5}`, "iop.num_batch_split_divisions must be an integer", false}, {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer", false}, {`{"num_batch_split_divisions":1.00000000000000000001}`, "iop.num_batch_split_divisions must be an integer", false}, {`{"num_batch_split_divisions":1e-1000}`, "iop.num_batch_split_divisions must be an integer", false}, {`{"num_batch_split_divisions":"1"}`, "iop.num_batch_split_divisions must be an integer", false}, {`{"batch_folding":false,"num_batch_split_divisions":1.0}`, "", true}, - {`{"batch_folding":true}`, `iop.batch_folding=true is not supported with setup_ops_format "agent_v2"`, true}, - {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer", true}, + {`{"batch_folding":true}`, "", true}, + {`{"num_batch_split_divisions":0.99999999999999999999}`, "", true}, + {`{"cpu_embeddings":"runtime-owned"}`, "", true}, } for _, test := range tests { @@ -256,10 +262,12 @@ func TestProjectModelLegacyNovaSettings(t *testing.T) { require.Equal(t, baseline.configuredBuild.Partitions, projection.configuredBuild.Partitions) require.Equal(t, baseline.connectors, projection.connectors) - t.Log("Omit both accepted legacy keys from normalized settings and rendered TOML") - require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") - require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") - configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", []*ModelProjection{projection}, nil) + t.Log("Omit legacy Nova settings from runtime output without validating unused hybrid values") + if !test.hybrid { + require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") + require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") + } + configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", []*ModelProjection{projection}) require.NoError(t, err) require.NotContains(t, configMap.Data["model_config.toml"], "batch_folding") require.NotContains(t, configMap.Data["model_config.toml"], "num_batch_split_divisions") @@ -293,54 +301,6 @@ func TestResolveBuildSettingsRejectsNestedNullOverride(t *testing.T) { require.ErrorContains(t, err, "settings.a must not be null") } -func TestValidateRuntimeTokenizerSettings(t *testing.T) { - t.Parallel() - - t.Log("Define incomplete and invalid tokenizer runtime settings") - tests := []struct { - name string - settings map[string]any - wantErr string - }{ - { - name: "tokenizer path", - settings: map[string]any{"stop_tokens": []uint32{1}}, - wantErr: "iop.tokenizer_path is required; set spec.components[].lpx.settings.tokenizer_path", - }, - { - name: "stop tokens", - settings: map[string]any{"tokenizer_path": "/models/tokenizer"}, - wantErr: "iop.stop_tokens is required; set spec.components[].lpx.settings.stop_tokens", - }, - { - name: "negative stop token", - settings: map[string]any{ - "tokenizer_path": "/models/tokenizer", - "stop_tokens": []any{int64(-1)}, - }, - wantErr: "iop.stop_tokens[0] must be an integer between 0 and 4294967295", - }, - { - name: "non-integer stop token representation", - settings: map[string]any{ - "tokenizer_path": "/models/tokenizer", - "stop_tokens": []any{float64(1)}, - }, - wantErr: "iop.stop_tokens[0] must be an integer between 0 and 4294967295", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - t.Log("Validate and verify rejection of the tokenizer settings") - err := validateRuntimeTokenizerSettings(tt.settings) - require.ErrorContains(t, err, tt.wantErr) - }) - } -} - func selectedPropSyncBuild(t *testing.T, chains [][]int) *Build { t.Helper() diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index f52902111a2d..decac9679aa9 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -123,7 +123,7 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { t.Log("Keep static environment values template-owned, including intentional omission") dynamicEnv := map[string]bool{ "LPX_ALLOCATION": role.conductor, - "TOPOLOGIES": role.direct, "GAS_DIR": role.direct, + "GAS_DIR": role.direct, } container.Env = slices.DeleteFunc(slices.Clone(container.Env), func(variable corev1.EnvVar) bool { return dynamicEnv[variable.Name] }) if role.conductor { diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index a5dc46bf550d..2efc32d27460 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -32,28 +32,7 @@ metadata: namespace: inference-engine --- data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 gas_dir: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - model_config.toml: | - type = 'Single' - - [iop] - model_path = '/nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c' - tokenizer_path = '/nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/tokenizer' - - [scheduler] - - [setup] - resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: |- 1 1 @@ -188,7 +167,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 + name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 namespace: inference-engine --- metadata: @@ -210,7 +189,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: d73a69201a8e19d2013c560e1155903bacf71d3a2cbdaaf4f3aea4deafa8d033 + nvidia.com/extra-resources-hash: 5448f938d5d52c694963453bff4e63367aa296f1c0aa36ea0960da60cad289b6 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 @@ -300,7 +279,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 + name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -331,11 +310,6 @@ spec: value: "19877" - name: READINESS_PORT value: "19878" - - name: TOPOLOGIES - valueFrom: - configMapKeyRef: - key: topologies - name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -420,7 +394,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-lpu-d73a69201a8e19d2 + name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 name: config replicas: 17 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 037818f58eeb..6602623fbf0c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -15,28 +15,7 @@ metadata: namespace: test --- data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 gas_dir: /nfs/node-local-v2-connected-lpx - model_config.toml: | - type = 'Single' - - [iop] - model_path = '/nfs/node-local-v2-connected-lpx' - tokenizer_path = '/nfs/node-local-v2-connected-lpx/tokenizer' - - [scheduler] - - [setup] - resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: "4" partition_ids: "0" partition_indices: "0" @@ -52,7 +31,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 + name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e namespace: test --- metadata: @@ -74,7 +53,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 5a21b650237fd262f004a640a67c5ecac0781784fa057703ee95223ab6a30c59 + nvidia.com/extra-resources-hash: a2baf441cd41d87e6ae2b9457c6134d41e06134d1091efe63bb4c4a01a17ef5f scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 @@ -115,7 +94,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 + name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -146,11 +125,6 @@ spec: value: "19877" - name: READINESS_PORT value: "19878" - - name: TOPOLOGIES - valueFrom: - configMapKeyRef: - key: topologies - name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -235,7 +209,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-lpu-5a21b650237fd262 + name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e name: config replicas: 4 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 32c4ebe67ecd..2a92e81f4f9b 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -40,8 +40,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -65,6 +63,10 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: '1' - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index bf77074bb257..68f2db13338c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -2,36 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 model_config.toml: | type = 'Single' [iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/testdata/node-local-v2-connected' - num_kv_caches = 1 - output_size = 1 - prop_sync = true - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' - - [scheduler] [setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: |- 2 2 @@ -55,7 +33,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 + name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d namespace: test --- metadata: @@ -77,7 +55,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 6acbae87b5dfe4f0837a02ffddf18a9af54e8ce74b645d0267c059517b8c1e2b + nvidia.com/extra-resources-hash: e9ba15e4ac831b0d6607f50bb0af48bae2bc56e3948e68850f78def4f0bbef8e scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 @@ -206,14 +184,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 + name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 6acbae87b5dfe4f0837a02ffddf18a9af54e8ce74b645d0267c059517b8c1e2b + nvidia.com/extra-resources-hash: e9ba15e4ac831b0d6607f50bb0af48bae2bc56e3948e68850f78def4f0bbef8e scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/role: conductor @@ -246,8 +224,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -289,6 +265,10 @@ spec: value: agt - name: NATS_SERVER value: nats-address + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -372,7 +352,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-lpu-only-33b1-lpu-6acbae87b5dfe4f0 + name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index e4ca8741d652..ad0995320dee 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -121,8 +121,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -146,6 +144,10 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: '1' - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index f98965de81c5..7a7a20ae65f8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -2,71 +2,22 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt0] - available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 - - [datacenters.racks.agt1] - available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 - - [datacenters.racks.agt2] - available_nodes = [0, 1, 2, 3] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt2-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 model_config.toml: | - draft_to_target_port = 23456 - head_to_head_port = 23458 - num_drafts = 2 - target_to_draft_port = 23457 type = 'SpecDecode' [draft] [draft.iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/testdata/node-local-v2-connected' - num_kv_caches = 1 - output_size = 1 - prop_sync = true - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' - - [draft.scheduler] [draft.setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' [target] [target.iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/testdata/node-local-v2-connected' - num_kv_caches = 1 - output_size = 1 - prop_sync = true - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/testdata/node-local-v2-connected/tokenizer' - - [target.scheduler] [target.setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: |- 2 2 @@ -124,7 +75,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 + name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be namespace: test --- metadata: @@ -146,7 +97,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 @@ -279,14 +230,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 + name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be name: config replicas: 4 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b @@ -419,14 +370,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 + name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be name: config replicas: 4 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e @@ -559,14 +510,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 + name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be name: config replicas: 4 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 0b6c0b4b697c4048acd29fadadbc9592c833ad1360fd9a4eed6a4ebf2fe6ebd2 + nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 scheduling.lpu.nvidia.com/role: conductor @@ -598,8 +549,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -641,6 +590,10 @@ spec: value: agt0:agt1:agt2 - name: NATS_SERVER value: nats-address + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -724,7 +677,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-specdecode-63df-lpu-0b6c0b4b697c4048 + name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 517115feba65..ab683ac58a91 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -2,37 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 gas_dir: /nfs/node-local-v3-hx-connected-lpx - model_config.toml: | - type = 'Single' - - [iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 - model_path = '/nfs/node-local-v3-hx-connected-lpx' - num_kv_caches = 1 - output_size = 1 - prop_sync = false - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/nfs/node-local-v3-hx-connected-lpx/tokenizer' - - [scheduler] - - [setup] - resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: "1" partition_ids: "1" partition_indices: "0" @@ -48,7 +18,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 + name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 namespace: test --- metadata: @@ -70,7 +40,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 7443e572ed6c9124e2bfdacd89de30a0515735f36cfb677a99e1be70668ca8d0 + nvidia.com/extra-resources-hash: dc7a6b2e2078e8215ec068db5b51870b05382a0252a8e8c788c34b1173fe44ca scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 @@ -111,7 +81,7 @@ spec: valueFrom: configMapKeyRef: key: gas_dir - name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 + name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME @@ -142,11 +112,6 @@ spec: value: "19877" - name: READINESS_PORT value: "19878" - - name: TOPOLOGIES - valueFrom: - configMapKeyRef: - key: topologies - name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 - name: UCX_TLS value: tcp image: example.invalid/lpu-runtime:test @@ -231,7 +196,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-hybrid-be18-lpu-7443e572ed6c9124 + name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 name: config replicas: 1 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index 744ae7737749..9d7925788e89 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -39,8 +39,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -104,6 +102,10 @@ spec: command: - /bin/nova env: + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: '1' - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 45ffc6897ba2..835a742f366c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -2,36 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 model_config.toml: | type = 'Single' [iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/nfs/node-local-v3-hx-connected' - num_kv_caches = 1 - output_size = 1 - prop_sync = false - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/nfs/node-local-v3-hx-connected/tokenizer' - - [scheduler] [setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: "1" partition_ids: "1" partition_indices: "0" @@ -47,7 +25,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f + name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 namespace: test --- metadata: @@ -69,7 +47,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b388a698f7dcc08f8fd26e78b06feab8d6992ea55a77555aacb9c70d9262f77a + nvidia.com/extra-resources-hash: 396d1555d763422281cc6c27d516f81737c8671ab6f880ade54570435a474152 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 @@ -228,14 +206,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f + name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 name: config replicas: 1 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b388a698f7dcc08f8fd26e78b06feab8d6992ea55a77555aacb9c70d9262f77a + nvidia.com/extra-resources-hash: 396d1555d763422281cc6c27d516f81737c8671ab6f880ade54570435a474152 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/role: conductor @@ -269,8 +247,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -350,6 +326,10 @@ spec: value: agt - name: NATS_SERVER value: nats-address + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -433,7 +413,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-b388a698f7dcc08f + name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 6ff3c963efb5..2ce317a4e893 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -150,8 +150,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -215,6 +213,10 @@ spec: command: - /bin/nova env: + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: '1' - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 3983441dc247..01e5615df135 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -2,71 +2,22 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt0] - available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt0-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 - - [datacenters.racks.agt1] - available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt1-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 - - [datacenters.racks.agt2] - available_nodes = [0] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt2-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 model_config.toml: | - draft_to_target_port = 23456 - head_to_head_port = 23458 - num_drafts = 2 - target_to_draft_port = 23457 type = 'SpecDecode' [draft] [draft.iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/nfs/node-local-v3-hx-sd-draft' - num_kv_caches = 1 - output_size = 1 - prop_sync = false - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/nfs/node-local-v3-hx-sd-draft/tokenizer' - - [draft.scheduler] [draft.setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' [target] [target.iop] - batch_size = 1 - cpu_embeddings = false - dkvc = false - input_size = 1 model_path = '/nfs/node-local-v3-hx-sd-target' - num_kv_caches = 1 - output_size = 1 - prop_sync = false - sequence_length = 8192 - stop_tokens = [1] - tokenizer_path = '/nfs/node-local-v3-hx-sd-target/tokenizer' - - [target.scheduler] [target.setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: |- 1 1 @@ -103,7 +54,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad namespace: test --- metadata: @@ -125,7 +76,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea @@ -288,14 +239,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad name: config replicas: 1 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed @@ -458,14 +409,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad name: config replicas: 1 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a @@ -628,14 +579,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad name: config replicas: 1 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 4704125bb939e57bb6093d7b211bf1b1994f47a4c89e1e4d3958075aa50108b2 + nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 scheduling.lpu.nvidia.com/role: conductor @@ -669,8 +620,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -750,6 +699,10 @@ spec: value: agt0:agt1:agt2 - name: NATS_SERVER value: nats-address + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -833,7 +786,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-4704125bb939e57b + name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml index 6b7746e394d8..d8596c31e3b1 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml @@ -50,8 +50,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -75,6 +73,10 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: MPI_BTL_TCP_IF_INCLUDE value: 192.0.2.0/24 - name: OMPI_MCA_btl_tcp_if_include diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index c0203b9387ae..5fccd9b5a001 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -2,37 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 data: - datacenter.toml: | - model_base_paths = ['/nfs'] - - [datacenters] - [datacenters.racks] - [datacenters.racks.agt] - available_nodes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - node_name_template = '${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-agt-{node}.${GROVE_HEADLESS_SERVICE}' - preferred_start = 0 model_config.toml: | type = 'Single' [iop] batch_size = 1 - cpu_embeddings = false - dkvc = false input_size = 1 model_path = '/testdata/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c' num_kv_caches = 1 - output_size = 1 - prop_sync = false sequence_length = 8192 stop_tokens = [128001, 128008, 128009] tokenizer_path = 'llama3.1' vocab_size = 128256 - [scheduler] - [setup] resolved_partitions_dir = '/configs' - setup_ops_format = 'agent_v2' nodes_per_partition: |- 1 1 @@ -131,7 +115,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa + name: dgd-single-v2-3a98-lpu-bd962bcf06374376 namespace: inference-engine --- metadata: @@ -153,7 +137,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: cc95c2008c398eaa60b329a696fa0f2faeff6c094df93b3d88d73cde3e6966a4 + nvidia.com/extra-resources-hash: bd962bcf06374376e0a6e9e786629713da2a5ddb698e5f351f8d27707e2335e5 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 @@ -286,14 +270,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa + name: dgd-single-v2-3a98-lpu-bd962bcf06374376 name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: cc95c2008c398eaa60b329a696fa0f2faeff6c094df93b3d88d73cde3e6966a4 + nvidia.com/extra-resources-hash: bd962bcf06374376e0a6e9e786629713da2a5ddb698e5f351f8d27707e2335e5 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 scheduling.lpu.nvidia.com/role: conductor @@ -327,8 +311,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --datacenter-config-filepath - - /configs/datacenter.toml - --model-config-filepath - /configs/model_config.toml - --json-logs @@ -372,6 +354,10 @@ spec: value: 192.0.2.0/24 - name: NATS_SERVER value: nats-address + - name: NOVA_MODEL_BASE_PATHS + value: /nfs + - name: NOVA_NODE_NAME_TEMPLATE + value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -457,7 +443,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: dgd-single-v2-3a98-lpu-cc95c2008c398eaa + name: dgd-single-v2-3a98-lpu-bd962bcf06374376 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index a59cc0e43f99..dddda6d4083f 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -32,6 +32,8 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu if err := resolveBuildSettings(&configured, modelSettings); err != nil { return nil, fmt.Errorf("resolving configured V2 build: %w", err) } + } else { + configured.runtimeSettings = modelSettings } if value, present := modelSettings["prop_sync"]; present { enabled, ok := value.(bool) @@ -51,17 +53,6 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu } } - // Share legacy-key handling without consuming the LPX build's partitions or selected chains. - if !usesResolvedRuntime { - if err := normalizeLegacyNovaSettings(modelSettings); err != nil { - return nil, fmt.Errorf("resolving configured V2 build: %w", err) - } - // Validate the Nova setting without removing Cyborg's required partition zero. - if _, err := boolSetting(modelSettings, "cpu_embeddings"); err != nil { - return nil, fmt.Errorf("model settings.%w", err) - } - configured.runtimeSettings = modelSettings - } allocationMetadata := json.RawMessage(`{}`) // Bind the V2 workload and runtime contract into projection identity before partition validation. diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index b9dbc48deff5..8cfcd604e954 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -174,7 +174,7 @@ func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { build.SelectedPropSyncChains = nil t.Log("Keep Cyborg's partition zero regardless of the Nova CPU-embedding setting") - for _, settings := range []string{"", `{"cpu_embeddings":false}`, `{"cpu_embeddings":true}`} { + for _, settings := range []string{"", `{"cpu_embeddings":false}`, `{"cpu_embeddings":true}`, `{"cpu_embeddings":"runtime-owned","batch_folding":true}`} { projection := projectTestBuild(t, normalized, PipelineLPX, settings) spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) @@ -186,14 +186,6 @@ func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { require.Equal(t, "0\n2", data["partition_node_offsets"]) } - t.Log("Still reject a malformed setting") - _, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineLPX, - Models: []string{"default"}, - BuildSnapshot: normalized, - ModelSettings: json.RawMessage(`{"cpu_embeddings":"false"}`), - }) - require.EqualError(t, err, "model settings.cpu_embeddings must be a boolean") } func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 2f1a25739377..c926fcd3b006 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -32,12 +32,18 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu // intentionally contains no artifact partitions. runtimeBuild.Partitions = nil runtimeBuild.SelectedPropSyncChains = nil - if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { + if intent.Pipeline == PipelineLPX { + runtimeBuild.runtimeSettings = modelSettings + } else if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { return nil, fmt.Errorf("resolving configured V3 runtime build: %w", err) } - propSyncEnabled, ok := runtimeBuild.runtimeSettings["prop_sync"].(bool) - if !ok { - return nil, fmt.Errorf("configured V3 runtime settings.prop_sync must be a boolean") + propSyncEnabled := runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly && len(selectedPropSyncChains) > 0 + if _, overridden := runtimeBuild.runtimeSettings["prop_sync"]; overridden { + var err error + propSyncEnabled, err = boolSetting(runtimeBuild.runtimeSettings, "prop_sync") + if err != nil { + return nil, fmt.Errorf("configured V3 runtime settings.%w", err) + } } if runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly { if len(selectedPropSyncChains) > 0 && !propSyncEnabled { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index f80271f0d31c..f9eb5e607769 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -37,8 +37,7 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.NoError(t, err) projection := projectionBatch[0] require.Equal(t, BuildCompilationModeHybrid, projection.configuredBuild.CompilationMode) - require.Equal(t, int64(8192), projection.configuredBuild.runtimeSettings["sequence_length"]) - require.Equal(t, false, projection.configuredBuild.runtimeSettings["prop_sync"]) + require.Empty(t, projection.configuredBuild.runtimeSettings) digest, err := workloadSetDigest([]*ModelProjection{projection}) require.NoError(t, err) require.Equal(t, projection.Digest(), digest, "one model is the aggregate digest base case") @@ -76,6 +75,13 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.Equal(t, *before, projection.RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) require.Nil(t, projection.RequestSpec(&MaterializationPlan{}, "agents").CyborgPodCliqueRef) + t.Log("Leave unused Nova settings opaque on the hybrid path") + intent.ModelSettings = json.RawMessage(`{"cpu_embeddings":"runtime-owned","batch_folding":true}`) + authored, err := appendModelProjections(nil, intent) + require.NoError(t, err) + require.Equal(t, map[string]any{"cpu_embeddings": "runtime-owned", "batch_folding": true}, authored[0].configuredBuild.runtimeSettings) + require.Equal(t, *before, authored[0].RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) + t.Log("Remove the manifest's selected chain without synthesizing hybrid connectors") fixture.selectedPropSyncChains = nil writeTestV3CapnpManifest(t, buildDir, fixture) From b8ae764055e2e589f616ae8ba837d82dfd259d37 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:41:40 +0300 Subject: [PATCH 098/196] fix(operator): reject unused hybrid runtime overrides (#536) Signed-off-by: Andrew Paprotsky (cherry picked from commit 5400bf1bc9cdeee22e87a979cd95ae1f8b2f7e0c) Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/cyborg_render_test.go | 3 +- .../dynamo/lpx/runtime_settings_test.go | 60 +++++++------------ .../dynamo/lpx/workload_projection.go | 10 ++++ .../dynamo/lpx/workload_projection_test.go | 39 ++++++++++++ .../dynamo/lpx/workload_projection_v2.go | 2 - .../dynamo/lpx/workload_projection_v2_test.go | 23 ++++--- .../dynamo/lpx/workload_projection_v3.go | 8 +-- .../dynamo/lpx/workload_projection_v3_test.go | 7 --- 8 files changed, 84 insertions(+), 68 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index e417a86b924f..cbb5d53a5cfc 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "strings" "testing" @@ -90,7 +89,7 @@ func TestRenderHybridPreservesRuntimeEnvironment(t *testing.T) { projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ Pipeline: PipelineLPX, Models: []string{"default"}, BuildSnapshot: normalized, - RuntimeBuildRef: "model-build", ModelSettings: json.RawMessage(`{"prop_sync":false}`), + RuntimeBuildRef: "model-build", }) require.NoError(t, err) projection := projectionBatch[0] diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go index c70703705612..57e871f81f11 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go @@ -11,7 +11,6 @@ import ( "slices" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" ) @@ -196,56 +195,39 @@ func TestResolveBuildSettingsDeepMergesDefaultsAndOverrides(t *testing.T) { func TestProjectModelLegacyNovaSettings(t *testing.T) { t.Parallel() - t.Log("Capture the Nova and hybrid settings paths before runtime-specific overrides") + t.Log("Capture the Nova settings path before runtime-specific overrides") singleIntent := ModelProjectionInput{ Models: []string{"default"}, Pipeline: PipelineSingle, BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), } - hybridFixture := newV2CompilerFixture() - hybridFixture.compilationMode = manifestcapnp.CompilationMode_lpx - hybridIntent := ModelProjectionInput{ - Models: []string{"default"}, Pipeline: PipelineLPX, - BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, hybridFixture))), - } singleBaselineBatch, err := appendModelProjections(nil, singleIntent) require.NoError(t, err) singleBaseline := singleBaselineBatch[0] - hybridBaselineBatch, err := appendModelProjections(nil, hybridIntent) - require.NoError(t, err) - hybridBaseline := hybridBaselineBatch[0] t.Log("Define accepted legacy defaults and unsupported values") tests := []struct { settings, wantErr string - hybrid bool }{ - {`{"batch_folding":false}`, "", false}, - {`{"num_batch_split_divisions":0}`, "", false}, - {`{"num_batch_split_divisions":1}`, "", false}, - {`{"num_batch_split_divisions":1.0}`, "", false}, - {`{"num_batch_split_divisions":10e-1}`, "", false}, - {`{"num_batch_split_divisions":-0.0}`, "", false}, - {`{"batch_folding":true}`, `iop.batch_folding=true is not supported`, false}, - {`{"batch_folding":"false"}`, "iop.batch_folding must be a boolean", false}, - {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported`, false}, - {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported`, false}, - {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported`, false}, - {`{"num_batch_split_divisions":0.5}`, "iop.num_batch_split_divisions must be an integer", false}, - {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer", false}, - {`{"num_batch_split_divisions":1.00000000000000000001}`, "iop.num_batch_split_divisions must be an integer", false}, - {`{"num_batch_split_divisions":1e-1000}`, "iop.num_batch_split_divisions must be an integer", false}, - {`{"num_batch_split_divisions":"1"}`, "iop.num_batch_split_divisions must be an integer", false}, - {`{"batch_folding":false,"num_batch_split_divisions":1.0}`, "", true}, - {`{"batch_folding":true}`, "", true}, - {`{"num_batch_split_divisions":0.99999999999999999999}`, "", true}, - {`{"cpu_embeddings":"runtime-owned"}`, "", true}, + {`{"batch_folding":false}`, ""}, + {`{"num_batch_split_divisions":0}`, ""}, + {`{"num_batch_split_divisions":1}`, ""}, + {`{"num_batch_split_divisions":1.0}`, ""}, + {`{"num_batch_split_divisions":10e-1}`, ""}, + {`{"num_batch_split_divisions":-0.0}`, ""}, + {`{"batch_folding":true}`, `iop.batch_folding=true is not supported`}, + {`{"batch_folding":"false"}`, "iop.batch_folding must be a boolean"}, + {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported`}, + {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported`}, + {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported`}, + {`{"num_batch_split_divisions":0.5}`, "iop.num_batch_split_divisions must be an integer"}, + {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer"}, + {`{"num_batch_split_divisions":1.00000000000000000001}`, "iop.num_batch_split_divisions must be an integer"}, + {`{"num_batch_split_divisions":1e-1000}`, "iop.num_batch_split_divisions must be an integer"}, + {`{"num_batch_split_divisions":"1"}`, "iop.num_batch_split_divisions must be an integer"}, } for _, test := range tests { intent, baseline := singleIntent, singleBaseline - if test.hybrid { - intent, baseline = hybridIntent, hybridBaseline - } t.Run(string(intent.Pipeline)+"/"+test.settings, func(t *testing.T) { t.Log("Validate model-provided JSON settings before rendering") intent.ModelSettings = []byte(test.settings) @@ -262,11 +244,9 @@ func TestProjectModelLegacyNovaSettings(t *testing.T) { require.Equal(t, baseline.configuredBuild.Partitions, projection.configuredBuild.Partitions) require.Equal(t, baseline.connectors, projection.connectors) - t.Log("Omit legacy Nova settings from runtime output without validating unused hybrid values") - if !test.hybrid { - require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") - require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") - } + t.Log("Omit legacy Nova settings from runtime output") + require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") + require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", []*ModelProjection{projection}) require.NoError(t, err) require.NotContains(t, configMap.Data["model_config.toml"], "batch_folding") diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go index 49d01ba79991..0a862265d654 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -31,6 +31,16 @@ func appendModelProjections(dst []*ModelProjection, intent ModelProjectionInput) modelSettings, modelSettingsObject := canonicalModelSettings(intent.ModelSettings) intent.ModelSettings = modelSettings + // Hybrid runtimes read their manifests directly; only XT scheduler controls are consumed here. + if intent.Pipeline == PipelineLPX { + for key := range modelSettingsObject { + if intent.BuildSnapshot.build.Family != BuildFamilyXT || + (key != "prop_sync" && key != "prop_sync_multiple_load_sets") { + return nil, fmt.Errorf("hybrid %s workloads do not support runtime settings overrides", intent.BuildSnapshot.build.Family) + } + } + } + var projections []*ModelProjection var err error switch intent.BuildSnapshot.build.Family { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go index 8bdf61f71129..1f6a6a86e0be 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -9,10 +9,49 @@ import ( "encoding/json" "testing" + manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" ) +func TestHybridSettingsRejectRuntimeOverrides(t *testing.T) { + for _, fixture := range []testV3CapnpFixture{newV2CompilerFixture(), newV3CompilerFixture()} { + t.Log("Acquire a hybrid build for each supported family") + fixture.compilationMode = manifestcapnp.CompilationMode_lpx + snapshot := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + + t.Log("Allow empty settings and XT scheduler controls, but not runtime overrides") + for _, test := range []struct { + settings string + xtOnly, reject bool + }{ + {settings: ""}, + {settings: "null"}, + {settings: "{}"}, + {settings: `{"prop_sync":true}`, xtOnly: true}, + {settings: `{"prop_sync_multiple_load_sets":true}`, xtOnly: true}, + {settings: `{"batch_size":4}`, reject: true}, + {settings: `{"setup":{"agent_setup_timeout":"600s"}}`, reject: true}, + {settings: `{"cpu_embeddings":true}`, reject: true}, + {settings: `{"batch_folding":false,"num_batch_split_divisions":1}`, reject: true}, + {settings: `{"prop_sync":false,"batch_size":4}`, reject: true}, + } { + t.Run(string(snapshot.build.Family)+"/"+test.settings, func(t *testing.T) { + t.Log("Reject unsupported settings before producing a workload projection") + _, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineLPX, Models: []string{"default"}, BuildSnapshot: snapshot, + ModelSettings: json.RawMessage(test.settings), + }) + if test.reject || (test.xtOnly && snapshot.build.Family != BuildFamilyXT) { + require.ErrorContains(t, err, "do not support runtime settings overrides") + } else { + require.NoError(t, err) + } + }) + } + } +} + func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { t.Parallel() diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index dddda6d4083f..7e63f1cf29ea 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -32,8 +32,6 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu if err := resolveBuildSettings(&configured, modelSettings); err != nil { return nil, fmt.Errorf("resolving configured V2 build: %w", err) } - } else { - configured.runtimeSettings = modelSettings } if value, present := modelSettings["prop_sync"]; present { enabled, ok := value.(bool) diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index 8cfcd604e954..6ea8a7677631 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -173,19 +173,16 @@ func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { build.SupportsCPUEmbeddings = true build.SelectedPropSyncChains = nil - t.Log("Keep Cyborg's partition zero regardless of the Nova CPU-embedding setting") - for _, settings := range []string{"", `{"cpu_embeddings":false}`, `{"cpu_embeddings":true}`, `{"cpu_embeddings":"runtime-owned","batch_folding":true}`} { - projection := projectTestBuild(t, normalized, PipelineLPX, settings) - spec := projection.RequestSpec(&MaterializationPlan{}, "agents") - require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) - require.Len(t, spec.Partitions, 2) - require.Equal(t, int64(0), spec.Partitions[0].CompilerPartitionID) - require.Equal(t, 4, projection.agentReplicas) - data := resolvedPartitionData([]*ModelProjection{projection}) - require.Equal(t, "0\n1", data["partition_ids"]) - require.Equal(t, "0\n2", data["partition_node_offsets"]) - } - + t.Log("Keep Cyborg's partition zero independently of Nova's embedding placement") + projection := projectTestBuild(t, normalized, PipelineLPX, "") + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") + require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) + require.Len(t, spec.Partitions, 2) + require.Equal(t, int64(0), spec.Partitions[0].CompilerPartitionID) + require.Equal(t, 4, projection.agentReplicas) + data := resolvedPartitionData([]*ModelProjection{projection}) + require.Equal(t, "0\n1", data["partition_ids"]) + require.Equal(t, "0\n2", data["partition_node_offsets"]) } func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index c926fcd3b006..2015525c5d31 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -32,10 +32,10 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu // intentionally contains no artifact partitions. runtimeBuild.Partitions = nil runtimeBuild.SelectedPropSyncChains = nil - if intent.Pipeline == PipelineLPX { - runtimeBuild.runtimeSettings = modelSettings - } else if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { - return nil, fmt.Errorf("resolving configured V3 runtime build: %w", err) + if intent.Pipeline != PipelineLPX { + if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { + return nil, fmt.Errorf("resolving configured V3 runtime build: %w", err) + } } propSyncEnabled := runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly && len(selectedPropSyncChains) > 0 if _, overridden := runtimeBuild.runtimeSettings["prop_sync"]; overridden { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index f9eb5e607769..eef7c4f49e9d 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -75,13 +75,6 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.Equal(t, *before, projection.RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) require.Nil(t, projection.RequestSpec(&MaterializationPlan{}, "agents").CyborgPodCliqueRef) - t.Log("Leave unused Nova settings opaque on the hybrid path") - intent.ModelSettings = json.RawMessage(`{"cpu_embeddings":"runtime-owned","batch_folding":true}`) - authored, err := appendModelProjections(nil, intent) - require.NoError(t, err) - require.Equal(t, map[string]any{"cpu_embeddings": "runtime-owned", "batch_folding": true}, authored[0].configuredBuild.runtimeSettings) - require.Equal(t, *before, authored[0].RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) - t.Log("Remove the manifest's selected chain without synthesizing hybrid connectors") fixture.selectedPropSyncChains = nil writeTestV3CapnpManifest(t, buildDir, fixture) From 9e07b9f13a35f3a2c89d5ea76fc49e51f966d405 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:02:55 +0300 Subject: [PATCH 099/196] refactor(lpx): move model configuration into pod templates (#537) Signed-off-by: Andrew Paprotsky (cherry picked from commit 330ffe66539b72c390c8074f59b0dfe587839028) Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1beta1/lpx_types.go | 4 +- .../nvidia.com_dynamographdeployments.yaml | 4 +- deploy/operator/go.mod | 1 - deploy/operator/go.sum | 2 - .../internal/controller/lpx/render_test.go | 78 ++--- deploy/operator/internal/dynamo/lpx/build.go | 19 +- .../dynamo/lpx/build_snapshot_test.go | 1 - .../internal/dynamo/lpx/cyborg_render.go | 10 +- .../internal/dynamo/lpx/lpu_configmap.go | 152 +-------- .../internal/dynamo/lpx/lpu_configmap_test.go | 106 ++---- .../internal/dynamo/lpx/model_manifest_v2.go | 52 --- .../dynamo/lpx/model_manifest_v2_test.go | 74 ----- deploy/operator/internal/dynamo/lpx/render.go | 18 +- .../internal/dynamo/lpx/render_test.go | 62 +--- .../lpx/resolve_selected_workload_test.go | 3 - .../internal/dynamo/lpx/runtime_agent.go | 33 ++ .../internal/dynamo/lpx/runtime_settings.go | 160 --------- .../dynamo/lpx/runtime_settings_test.go | 308 ------------------ .../dynamo/lpx/runtime_startup_test.go | 92 ++++++ .../dynamo/lpx/selection_validation.go | 3 + .../node-local-v2-lpu-only.input.yaml | 6 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 26 +- .../node-local-v2-specdecode.input.yaml | 12 +- .../node-local-v2-specdecode.yaml | 50 ++- .../node-local-v3-hx-lpu-only.input.yaml | 6 +- .../node-local-v3-hx-lpu-only.yaml | 26 +- .../node-local-v3-hx-specdecode.input.yaml | 12 +- .../node-local-v3-hx-specdecode.yaml | 50 ++- .../from_dgd_yaml/single_v2.input.yaml | 31 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 55 ++-- .../dynamo/lpx/workload_projection.go | 21 +- .../dynamo/lpx/workload_projection_test.go | 52 ++- .../dynamo/lpx/workload_projection_v2.go | 14 +- .../dynamo/lpx/workload_projection_v2_test.go | 106 +++++- .../dynamo/lpx/workload_projection_v3.go | 21 +- .../dynamo/lpx/workload_projection_v3_test.go | 1 - .../model-deployment/deploy-with-dgd.md | 33 +- .../additional-resources/api-reference-k8s.md | 4 +- .../dynamo-component-deployment.mdx | 6 +- .../kubernetes-api/full-api-reference.mdx | 4 +- 40 files changed, 554 insertions(+), 1164 deletions(-) delete mode 100644 deploy/operator/internal/dynamo/lpx/runtime_settings.go delete mode 100644 deploy/operator/internal/dynamo/lpx/runtime_settings_test.go diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index 4ec912d37609..c3b5d455017e 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -8,12 +8,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// LPXConfig identifies the component's compiled model and runtime settings. +// LPXConfig identifies the component's compiled model and placement settings. type LPXConfig struct { // buildId references the immutable model build. // +kubebuilder:validation:MinLength=1 BuildID string `json:"buildId"` - // settings override the existing model configuration. + // settings configure model placement. Runtime options belong in role pod templates. // +optional // +kubebuilder:validation:Type=object // +nullable diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index d27244bb3a50..c7202de00fb1 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11530,7 +11530,7 @@ spec: minLength: 1 type: string settings: - description: settings override the existing model configuration. + description: settings configure model placement. Runtime options belong in role pod templates. nullable: true type: object x-kubernetes-preserve-unknown-fields: true @@ -21073,7 +21073,7 @@ spec: minLength: 1 type: string settings: - description: settings override the existing model configuration. + description: settings configure model placement. Runtime options belong in role pod templates. nullable: true type: object x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 05be1fb384b6..96c2d6893c1a 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -18,7 +18,6 @@ require ( github.com/onsi/ginkgo/v2 v2.30.0 github.com/onsi/gomega v1.41.0 github.com/open-policy-agent/cert-controller v0.16.0 - github.com/pelletier/go-toml/v2 v2.3.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.71.2 github.com/prometheus/client_golang v1.23.2 diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 2baeb82cfa45..d48d2e0e207c 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -165,8 +165,6 @@ github.com/open-policy-agent/cert-controller v0.16.0 h1:/w+wAM9dylWGwhPnvfCgepyB github.com/open-policy-agent/cert-controller v0.16.0/go.mod h1:w5qBWYbc8HwyHI9VYAZ6YjWOcZtQ39A30I9W4X7pVVk= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a/go.mod h1:tI7nc6H6os2UYZRvSm9Y7bq4oMoXqhwA0WfnqKpoAgc= -github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= -github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 2186eea04e93..dac4c5dddecf 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -30,7 +30,6 @@ import ( "github.com/stretchr/testify/require" "gotest.tools/v3/golden" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -730,7 +729,7 @@ func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphM return data } -func TestSingleV2ConfigContainsOnlyDeploymentOverrides(t *testing.T) { +func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { controllerConfig := &configv1alpha1.OperatorConfiguration{ Orchestrators: configv1alpha1.OrchestratorConfiguration{ Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, @@ -742,70 +741,49 @@ func TestSingleV2ConfigContainsOnlyDeploymentOverrides(t *testing.T) { registry, err := lpx.NewModelRegistry(registryRoot, nil) require.NoError(t, err) - type renderedConfig struct { - modelConfig string - hash string - } - t.Log("Render omitted defaults and explicit deployment overrides") - configs := make([]renderedConfig, 0, 3) - for _, settingsJSON := range []string{"", `{"batch_size":1,"swa":{"chunked":false}}`, `{"sequence_length":65536}`} { + t.Log("Render runtime overrides supplied directly by the conductor template") + var partitionData map[string]string + for _, env := range []corev1.EnvVar{ + {Name: "NOVA_BATCH_SIZE", Value: "1"}, + {Name: "NOVA_SEQUENCE_LENGTH", Value: "65536"}, + {Name: "A_CUSTOM_MODEL_PATH", Value: "$(LPX_MODEL_PATH)"}, + } { payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml") require.NoError(t, err) - var deployment v1beta1.DynamoGraphDeployment require.NoError(t, yaml.Unmarshal(payload, &deployment)) - require.Len(t, deployment.Spec.Components, 1) - require.NotNil(t, deployment.Spec.Components[0].LPX) - require.Len(t, deployment.Spec.Components[0].Roles, 2) - deployment.Spec.Components[0].LPX = &v1beta1.LPXConfig{BuildID: buildID} - if settingsJSON != "" { - deployment.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(settingsJSON)} - } + component := &deployment.Spec.Components[0] + component.LPX = &v1beta1.LPXConfig{BuildID: buildID} + template := component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + template.Spec.Containers[0].Env = []corev1.EnvVar{env} selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) require.NoError(t, err) child := newLPXRenderDeployment(t, &deployment) - podCliqueSet, resources, err := renderPodCliqueSet( - t.Context(), - &deployment, - controllerConfig, - &controller_common.RuntimeConfig{}, - nil, - selected, - mustPlanSelectedLPX(t, &deployment, selected), - child, - ) + plan := mustPlanSelectedLPX(t, &deployment, selected) + pcs, resources, err := renderPodCliqueSet(t.Context(), &deployment, controllerConfig, + &controller_common.RuntimeConfig{}, nil, selected, plan, child) require.NoError(t, err) + t.Log("Keep authored environment values without changing shared partition files") var hash string - for _, clique := range podCliqueSet.Spec.Template.Cliques { - if value := clique.Annotations[commonconsts.AnnotationExtraResourcesHash]; value != "" { - hash = value - break + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name == plan.ConductorTemplate { + require.Equal(t, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}, clique.Spec.PodSpec.Containers[0].Env[0]) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, env) + hash = clique.Annotations[commonconsts.AnnotationExtraResourcesHash] } } require.NotEmpty(t, hash) configMap := getResource[*corev1.ConfigMap](t, resources, lpx.LPUConfigMapName(dynamo.PCSNameForLPX(child), hash)) - configs = append(configs, renderedConfig{modelConfig: configMap.Data["model_config.toml"], hash: hash}) + require.NotContains(t, configMap.Data, "model_config.toml") + require.NotContains(t, configMap.Data, "datacenter.toml") + if partitionData == nil { + partitionData = configMap.Data + } else { + require.Equal(t, partitionData, configMap.Data) + } } - - t.Log("Hash only emitted settings while preserving explicit deployment intent") - derived, explicit, overridden := configs[0], configs[1], configs[2] - - buildPath := filepath.Join(registryRoot, buildID) - require.Equal(t, fmt.Sprintf(`type = 'Single' - -[iop] -model_path = '%s' - -[setup] -resolved_partitions_dir = '/configs' -`, buildPath), derived.modelConfig) - require.Contains(t, explicit.modelConfig, "batch_size = 1") - require.Contains(t, explicit.modelConfig, "chunked = false") - require.NotEqual(t, derived.hash, explicit.hash) - require.Contains(t, overridden.modelConfig, "sequence_length = 65536") - require.NotEqual(t, derived.hash, overridden.hash) } func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { diff --git a/deploy/operator/internal/dynamo/lpx/build.go b/deploy/operator/internal/dynamo/lpx/build.go index 95401790e7a7..0efeb9a9f77f 100644 --- a/deploy/operator/internal/dynamo/lpx/build.go +++ b/deploy/operator/internal/dynamo/lpx/build.go @@ -8,7 +8,6 @@ package lpx import ( "fmt" "path/filepath" - "slices" "strings" ) @@ -36,7 +35,7 @@ const ( // Build is the registry's source-independent view of an LPU build. // // The version 2 Cap'n Proto manifest populates this shape before deployment code -// derives placement, replica counts, and runtime overrides. +// derives placement and replica counts. type Build struct { // Path is the absolute file or GCS reference of the build payload. Path string @@ -56,9 +55,6 @@ type Build struct { IOFPGACount int32 // IOFanoutFactor is the number of clients assigned to each I/O FPGA transaction. IOFanoutFactor int32 - // runtimeSettings holds manifest overrides on a snapshot and merged - // authored settings shared by immutable component projections. - runtimeSettings map[string]any } // BuildPartition describes one normalized physical compiler partition. @@ -90,19 +86,6 @@ func (p BuildPartition) effectiveNodeCount() int { return p.Topology.Replicas() } -// omitStandaloneEmbeddingPartition removes host-only embedding work when the -// build manifest explicitly identifies it as safe to run outside the LPU. -func (b *Build) omitStandaloneEmbeddingPartition() { - if !b.SupportsCPUEmbeddings || !b.StandaloneTokenEmbeddings { - return - } - - // Normalized XT partitions are sorted by source ID; keep the retained interval independently owned. - if len(b.Partitions) > 1 && b.Partitions[0].SourcePartitionID == 0 { - b.Partitions = slices.Clone(b.Partitions[1:]) - } -} - func buildRuntimePath(buildPath string, modelStoragePath string) (string, error) { buildURL, err := parseBuildRef(buildPath) if err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go index 12540c9525dc..b90f0be39709 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go @@ -59,7 +59,6 @@ func TestAcquireBuildSnapshotTracksLocalContent(t *testing.T) { normalized, err := normalizeBuildSnapshot(first) require.NoError(t, err) build := normalized.build - require.NotContains(t, build.runtimeSettings, "batch_size") require.EqualValues(t, 4, build.IOFPGACount) require.EqualValues(t, 2, build.IOFanoutFactor) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index b0a9732e16d2..33fd5e404100 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -21,24 +21,24 @@ func configureHybridCyborg( cyborg *grovev1alpha1.PodCliqueTemplateSpec, projection *ModelProjection, workloadDigest string, - modelStorage lpuModelStorage, + modelStoragePath string, agentTemplateNames []string, cyborgConfigMap *corev1.ConfigMap, ) error { container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) // Apply manifest-aware runtime bindings to the selected Cyborg container. - cyborgStorage, err := lpuModelStorageBinding(cyborg.Spec.PodSpec) + cyborgStoragePath, err := lpuModelStoragePath(cyborg.Spec.PodSpec) if err != nil { return err } - if cyborgStorage.mount.MountPath != modelStorage.mount.MountPath { - return fmt.Errorf("selected Cyborg podTemplate conflicts with model storage mount %q", modelStorage.mount.MountPath) + if cyborgStoragePath != modelStoragePath { + return fmt.Errorf("selected Cyborg podTemplate conflicts with model storage mount %q", modelStoragePath) } if err := validateCyborgReplicas(&projection.configuredBuild, cyborg.Spec.Replicas); err != nil { return err } - if err := applyCyborgManifestPath(container, projection, modelStorage.mount.MountPath); err != nil { + if err := applyCyborgManifestPath(container, projection, modelStoragePath); err != nil { return err } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index aefe4218ae54..e8db8d33538d 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -8,7 +8,6 @@ package lpx import ( "crypto/sha256" "fmt" - "maps" "net/url" "path/filepath" "slices" @@ -18,7 +17,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/pelletier/go-toml/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -29,11 +27,6 @@ const ( lpuConfigMountPath = "/configs" ) -type lpuModelStorage struct { - volume corev1.Volume - mount corev1.VolumeMount -} - // LPUConfigMapName names the immutable runtime table using its Pod-template content hash. // The root is the PCS identity carried by Grove's part-of Pod label. func LPUConfigMapName(root, configHash string) string { @@ -54,17 +47,6 @@ func renderLPUConfigMap( return nil, fmt.Errorf("resolve gas_dir: %w", err) } data["gas_dir"] = modelPath - } else { - // Conductors consume deployment overrides alongside compile-owned manifest defaults. - modelConfig, err := lpuModelConfig(projections, modelStoragePath) - if err != nil { - return nil, fmt.Errorf("render model_config.toml: %w", err) - } - var modelTOML strings.Builder - if err := toml.NewEncoder(&modelTOML).Encode(modelConfig); err != nil { - return nil, fmt.Errorf("render model_config.toml: %w", err) - } - data["model_config.toml"] = modelTOML.String() } return renderRuntimeConfigMap(namespace, materializationName+"-lpu", data) } @@ -107,150 +89,26 @@ func LPUConfigMapHash(configMap *corev1.ConfigMap) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(contentHash))) } -func lpuModelStorageBinding(spec corev1.PodSpec) (lpuModelStorage, error) { +func lpuModelStoragePath(spec corev1.PodSpec) (string, error) { container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == commonconsts.ModelStorageVolumeName }) if mountIndex < 0 { - return lpuModelStorage{}, fmt.Errorf( + return "", fmt.Errorf( "selected LPX main container requires model storage volume mount %q", commonconsts.ModelStorageVolumeName, ) } mount := container.VolumeMounts[mountIndex] if strings.TrimSpace(mount.MountPath) == "" { - return lpuModelStorage{}, fmt.Errorf("model storage volume %q has no mount path", mount.Name) + return "", fmt.Errorf("model storage volume %q has no mount path", mount.Name) } volumeIndex := slices.IndexFunc(spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == mount.Name }) if volumeIndex < 0 { - return lpuModelStorage{}, fmt.Errorf("selected LPX podTemplate has no model storage volume %q", mount.Name) - } - return lpuModelStorage{volume: spec.Volumes[volumeIndex], mount: mount}, nil -} - -func lpuModelConfig(projections []*ModelProjection, modelStoragePath string) (map[string]any, error) { - if projections[0].pipeline != PipelineSpecDecode { - config, err := nestedLPUModelConfig(projections[0], modelStoragePath) - if err != nil { - return nil, err - } - config["type"] = "Single" - return config, nil - } - - // Derive the pipeline-wide speculative-decoding controls from the draft model. - config := make(map[string]any) - - // Promote the draft's setup timeouts so Nova applies them to every model. - setup, err := modelObjectSetting(projections[0], "setup") - if err != nil { - return nil, err - } - delete(setup, "resolved_partitions_dir") - if len(setup) > 0 { - config["setup"] = setup - } - - // Promote the only model-provided speculative-decoding scalar currently consumed by the runtime. - if value, present := projections[0].configuredBuild.runtimeSettings[v3MaxSWADKVCBlocksDraft]; present { - config[v3MaxSWADKVCBlocksDraft] = value - } - - draftCount := len(projections) - 1 - draft, err := nestedLPUModelConfig(projections[0], modelStoragePath) - if err != nil { - return nil, err - } - // Draft setup values configure the SpecDecode parent. Keep the child setup - // limited to the operator-owned LPU runtime bindings so Nova can inherit the - // parent timeouts exactly as it did with the legacy pipeline field. - draft["setup"] = map[string]any{ - "resolved_partitions_dir": lpuConfigMountPath, - } - target, err := nestedLPUModelConfig(projections[draftCount], modelStoragePath) - if err != nil { - return nil, err - } - config["type"] = "SpecDecode" - config["draft"] = draft - config["target"] = target - return config, nil -} - -func nestedLPUModelConfig(projection *ModelProjection, modelStoragePath string) (map[string]any, error) { - settings := maps.Clone(projection.configuredBuild.runtimeSettings) - - // Lift Nova's model-level sections out of the IOP settings object. - scheduler, err := modelObjectSetting(projection, "scheduler") - if err != nil { - return nil, err - } - delete(settings, "scheduler") - setup, err := modelObjectSetting(projection, "setup") - if err != nil { - return nil, err - } - if setup == nil { - setup = map[string]any{} - } - delete(settings, "setup") - - // Keep operator-owned LPU runtime setup bindings authoritative. - setup["resolved_partitions_dir"] = lpuConfigMountPath - delete(settings, v3MaxSWADKVCBlocksDraft) - - // Lift additional programs into the Nova model object when configured. - var extraPrograms []any - if configured, present := settings["extra_programs"]; present { - var ok bool - extraPrograms, ok = configured.([]any) - if !ok { - return nil, fmt.Errorf("model %q settings.extra_programs must be an array", projection.model) - } - delete(settings, "extra_programs") - if projection.configuredBuild.Family == BuildFamilyXT && - projection.pipeline == PipelineSingle && len(extraPrograms) > 0 { - return nil, fmt.Errorf("model %q settings.extra_programs is not supported for XT Single", projection.model) - } - } - - build := &projection.configuredBuild - buildRef := build.Path - if projection.configuredBuild.Family == BuildFamilyHX { - buildRef = lpuRuntimeBuildRef(projection, modelStoragePath) - } - modelPath, err := buildRuntimePath(buildRef, modelStoragePath) - if err != nil { - return nil, fmt.Errorf("model %q model_path: %w", projection.model, err) - } - if _, overridden := settings["model_path"]; !overridden || - projection.configuredBuild.Family != BuildFamilyXT || projection.pipeline != PipelineSpecDecode { - settings["model_path"] = modelPath - } - config := map[string]any{ - "setup": setup, - "iop": settings, - } - if scheduler != nil { - config["scheduler"] = scheduler - } - if extraPrograms != nil { - config["extra_programs"] = extraPrograms - } - return config, nil -} - -func modelObjectSetting(projection *ModelProjection, name string) (map[string]any, error) { - configured, present := projection.configuredBuild.runtimeSettings[name] - if !present { - return nil, nil - } - object, ok := configured.(map[string]any) - if !ok { - return nil, fmt.Errorf("model %q settings.%s must be an object", projection.model, name) + return "", fmt.Errorf("selected LPX podTemplate has no model storage volume %q", mount.Name) } - return maps.Clone(object), nil + return mount.MountPath, nil } func lpuRuntimeBuildRef(projection *ModelProjection, modelStoragePath string) string { diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 2b30a1f69b48..420ada0012af 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "strings" "testing" @@ -97,9 +96,8 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { pipeline: PipelineLPX, runtimeBuildRef: "model-build", configuredBuild: Build{ - Path: "file:///snapshot-build", - Family: BuildFamilyXT, - runtimeSettings: map[string]any{}, + Path: "file:///snapshot-build", + Family: BuildFamilyXT, }, } @@ -119,13 +117,6 @@ func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { require.True(t, *configMap.Immutable) require.Equal(t, LPUConfigMapName("test-dgd", LPUConfigMapHash(configMap)), configMap.Name) - t.Log("Ignore model settings that are not consumed by the direct runtime") - projection.configuredBuild.runtimeSettings["oversized"] = strings.Repeat("x", corev1.MaxSecretSize) - projection.configuredBuild.runtimeSettings["scheduler"] = "not-a-Nova-scheduler" - unchanged, err := renderLPUConfigMap("test-namespace", "test-dgd", "/models", []*ModelProjection{projection}) - require.NoError(t, err) - require.Equal(t, configMap, unchanged) - t.Log("Render the preserved runtime with an invalid snapshot build reference") projection.configuredBuild.Path = "relative-build" _, err = renderLPUConfigMap( @@ -168,77 +159,32 @@ func TestResolvedPartitionDataOmitsXTModelColumnsBeforeMaterialization(t *testin }, data) } -func TestV3SingleConfigUsesManifestTopologyAndArbitraryModelSettings(t *testing.T) { +func TestConductorConfigMapContainsOnlyPartitions(t *testing.T) { t.Parallel() - t.Log("Project the V3 manifest with arbitrary deployment-time model settings") - snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) - projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineSingle, Models: []string{"default"}, RuntimeBuildRef: "model-build", BuildSnapshot: normalizeTestSnapshot(t, snapshot), - ModelSettings: json.RawMessage(`{ - "batch_size":4, - "custom_runtime_knob":"enabled", - "scheduler":{"max_inflight_tasks":3}, - "tokenizer_path":"/nfs/models/gemma4-31b", - "stop_tokens":[1] - }`), - }) - require.NoError(t, err) - projection := projectionBatch[0] - - t.Log("Render the complete LPU ConfigMap") - configMap, err := renderLPUConfigMap( - "test", - "test-dgd", - "/models", - []*ModelProjection{projection}, - ) - require.NoError(t, err) - - t.Log("Verify the generated files need no manual topology or model-setting patches") - require.Equal(t, "part-1", configMap.Data["partition_paths"]) - require.Equal(t, v3OpaqueTopology, configMap.Data["topologies"]) - require.NotContains(t, configMap.Data["model_config.toml"], "sequence_length") - require.NotContains(t, configMap.Data, "datacenter.toml") - require.Contains(t, configMap.Data["model_config.toml"], "model_path = '/models/model-build'") - require.Contains(t, configMap.Data["model_config.toml"], "batch_size = 4") - require.Contains(t, configMap.Data["model_config.toml"], "custom_runtime_knob = 'enabled'") - require.Contains(t, configMap.Data["model_config.toml"], "max_inflight_tasks = 3") - require.Contains(t, configMap.Data["model_config.toml"], "tokenizer_path = '/nfs/models/gemma4-31b'") - require.Contains(t, configMap.Data["model_config.toml"], "stop_tokens = [1]") -} + for _, family := range []string{"v2", "v3"} { + t.Run(family, func(t *testing.T) { + t.Log("Project a compiler build with a template-owned runtime configuration") + var path string + if family == "v2" { + path = writeV2CompilerFixture(t) + } else { + path = writeV3CompilerFixture(t) + } + projections, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: PipelineSingle, Models: []string{"default"}, + BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, path)), + }) + require.NoError(t, err) -func TestNestedLPUModelConfigPreservesExplicitOverrides(t *testing.T) { - t.Log("Construct a model with deployment overrides and a remapped runtime path") - projection := &ModelProjection{ - model: "default", pipeline: PipelineSingle, runtimeBuildRef: "model-build", - configuredBuild: Build{ - Path: "file:///snapshot-build", Family: BuildFamilyHX, - runtimeSettings: map[string]any{"tokenizer_path": "/custom/tokenizer", "batch_size": 4}, - }, + t.Log("Retain only the resolved partition files in the immutable ConfigMap") + configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", projections) + require.NoError(t, err) + require.Equal(t, resolvedPartitionData(projections), configMap.Data) + require.NotContains(t, configMap.Data, "model_config.toml") + require.NotContains(t, configMap.Data, "datacenter.toml") + require.NotEmpty(t, configMap.Data["partition_paths"]) + require.NotEmpty(t, configMap.Data["topologies"]) + }) } - - t.Log("Leave manifest defaults to the runtime while retaining authored overrides") - config, err := nestedLPUModelConfig(projection, "/models") - require.NoError(t, err) - require.Equal(t, map[string]any{ - "model_path": "/models/model-build", "tokenizer_path": "/custom/tokenizer", "batch_size": 4, - }, config["iop"]) - require.Equal(t, map[string]any{"resolved_partitions_dir": "/configs"}, config["setup"]) - require.NotContains(t, config, "scheduler") -} - -func TestV2SingleConfigRejectsExtraPrograms(t *testing.T) { - t.Log("Render a V2 single-model configuration with unsupported extra programs") - _, err := lpuModelConfig([]*ModelProjection{{ - model: "default", - pipeline: PipelineSingle, - configuredBuild: Build{ - Family: BuildFamilyXT, - runtimeSettings: map[string]any{"extra_programs": []any{"unsupported"}}, - }, - }}, "/models") - - t.Log("Verify the unsupported runtime setting is rejected") - require.ErrorContains(t, err, "settings.extra_programs is not supported") } diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go index 20307d63f664..de0986d9b319 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -97,13 +97,6 @@ func buildFromGbuildManifestV2(buildRef string, manifest manifestcapnpv2.Manifes ioFanoutFactor, ) } - var settings map[string]any - if compilationMode == BuildCompilationModeLPUOnly { - settings, err = runtimeSettingsFromManifestV2(manifest, program) - if err != nil { - return nil, err - } - } if !manifest.HasArtifacts() { return nil, fmt.Errorf("%s is missing artifacts", gbuildManifestV2CapnpFile) } @@ -131,7 +124,6 @@ func buildFromGbuildManifestV2(buildRef string, manifest manifestcapnpv2.Manifes SupportsCPUEmbeddings: program.SupportsCpuEmbeddings(), IOFPGACount: ioFPGACount, IOFanoutFactor: ioFanoutFactor, - runtimeSettings: settings, } // Complete the normalized build with scheduler-facing LPU artifacts. @@ -203,50 +195,6 @@ func selectedPropSyncChainsFromManifestV2(deployment manifestcapnpv2.DeploymentI return chains, nil } -func runtimeSettingsFromManifestV2(manifest manifestcapnpv2.Manifest, program manifestcapnpv2.ProgramConfig) (map[string]any, error) { - // Retain only settings whose omission would change the runtime's interpretation. - settings := map[string]any{ - "batch_folding": program.BatchFolding(), - "num_batch_split_divisions": int64(program.NumBatchSplitDivisions()), - } - var archVocabulary uint32 - hasSWA := false - if manifest.HasModel() { - model, err := manifest.Model() - if err != nil { - return nil, fmt.Errorf("reading %s model: %w", gbuildManifestV2CapnpFile, err) - } - if model.HasArch() { - arch, err := model.Arch() - if err != nil { - return nil, fmt.Errorf("reading %s model.arch: %w", gbuildManifestV2CapnpFile, err) - } - archVocabulary = arch.VocabSize() - hasSWA = arch.HasSwa() - } - - // Tokenizer vocabulary takes precedence over architecture padding in the deployment contract. - if model.HasTokenizer() { - tokenizer, err := model.Tokenizer() - if err != nil { - return nil, fmt.Errorf("reading %s model.tokenizer: %w", gbuildManifestV2CapnpFile, err) - } - if vocabulary := tokenizer.VocabSize(); vocabulary != 0 && vocabulary != archVocabulary { - settings["vocab_size"] = int64(vocabulary) - } - } - } - - // Program-only SWA settings are not inferred when the architecture has no SWA descriptor. - if !hasSWA && (program.SwaChunked() || program.NumSwaDkvcBlocks() != 0) { - settings["swa"] = map[string]any{ - "chunked": program.SwaChunked(), - "num_swa_dkvc_blocks": int64(program.NumSwaDkvcBlocks()), - } - } - return settings, nil -} - func addLPUArtifactsFromManifestV2( artifacts manifestcapnpv2.ArtifactInfo, deployment manifestcapnpv2.DeploymentInfo, diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go index 97c32d8d3ae1..e669f8e58117 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -42,80 +42,6 @@ func TestBuildFromGbuildManifestV2ProjectsRuntimeIO(t *testing.T) { require.ErrorContains(t, err, "contractRevision") } -func TestBuildFromGbuildManifestV2RetainsRuntimeOverrides(t *testing.T) { - t.Parallel() - - t.Log("Keep only compiler metadata whose runtime default would differ") - for _, test := range []struct { - name string - architectureVocabulary uint32 - tokenizerVocabulary uint32 - architectureSWA bool - programSWA bool - want map[string]any - }{ - {name: "matching vocabulary", architectureVocabulary: 128, tokenizerVocabulary: 128}, - {name: "architecture vocabulary only", architectureVocabulary: 128}, - {name: "tokenizer vocabulary only", tokenizerVocabulary: 128, want: map[string]any{"vocab_size": int64(128)}}, - {name: "tokenizer overrides architecture padding", architectureVocabulary: 256, tokenizerVocabulary: 128, want: map[string]any{"vocab_size": int64(128)}}, - {name: "architecture SWA is runtime-owned", architectureSWA: true, programSWA: true}, - {name: "program-only SWA defaults", programSWA: true, want: map[string]any{"swa": map[string]any{ - "chunked": true, "num_swa_dkvc_blocks": int64(2), - }}}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Encode compiler values and the selected vocabulary/SWA boundary") - manifest := newManifestV2ContractFixture(t) - deployment, err := manifest.Deployment() - require.NoError(t, err) - deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpuOnly) - program, err := deployment.Program() - require.NoError(t, err) - require.NoError(t, program.SetProgramName("program")) - require.NoError(t, program.SetPackageName("package")) - program.SetSequenceLength(4096) - program.SetInputSize(1) - program.SetOutputSize(1) - program.SetNumKvCaches(8) - program.SetSwaChunked(test.programSWA) - if test.programSWA { - program.SetNumSwaDkvcBlocks(2) - } - model, err := manifest.NewModel() - require.NoError(t, err) - if test.architectureVocabulary != 0 || test.architectureSWA { - arch, err := model.NewArch() - require.NoError(t, err) - arch.SetVocabSize(test.architectureVocabulary) - arch.SetNLayers(12) - if test.architectureSWA { - swa, err := arch.NewSwa() - require.NoError(t, err) - swa.SetMaxSeqLen(128) - swa.SetPadding(32) - } - } - tokenizer, err := model.NewTokenizer() - require.NoError(t, err) - tokenizer.SetVocabSize(test.tokenizerVocabulary) - require.NoError(t, tokenizer.SetPath("tokenizer")) - stops, err := tokenizer.NewStopTokens(1) - require.NoError(t, err) - stops.Set(0, 2) - - t.Log("Preserve required overrides without copying settings the runtime reads itself") - build, err := buildFromGbuildManifestV2("gs://models/build", manifest) - require.NoError(t, err) - require.NoError(t, resolveBuildSettings(build, nil)) - if test.want == nil { - require.Empty(t, build.runtimeSettings) - } else { - require.Equal(t, test.want, build.runtimeSettings) - } - }) - } -} - func TestNormalizeBuildSnapshotRejectsInvalidHXArtifacts(t *testing.T) { t.Log("Define malformed HX artifact inventories") tests := []struct { diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 4f003d0e10b2..7b6419e6a6ee 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -86,13 +86,11 @@ func RenderSelectedNodeLocal( } storageTemplate = *conductorTemplate } - modelStorage, err := lpuModelStorageBinding(storageTemplate.Spec) + modelStoragePath, err := lpuModelStoragePath(storageTemplate.Spec) if err != nil { return nil, err } - modelStorage.volume = *modelStorage.volume.DeepCopy() - modelStorage.mount = *modelStorage.mount.DeepCopy() - configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStorage.mount.MountPath, projections) + configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStoragePath, projections) if err != nil { return nil, err } @@ -117,6 +115,10 @@ func RenderSelectedNodeLocal( // Consume the independently rendered conductor without copying Agent startup or placement. var conductor *grovev1alpha1.PodCliqueTemplateSpec if conductorTemplateName != "" { + container := common.FindContainerByName(conductorTemplate.Spec.Containers, commonconsts.MainContainerName) + if err := applyConductorModelPaths(container, projections, modelStoragePath); err != nil { + return nil, err + } annotations := roleAnnotations(conductorTemplate.Annotations, lpxv1alpha1.PodRoleConductor, workloadDigest) annotations[commonconsts.AnnotationExtraResourcesHash] = configHash conductor = &grovev1alpha1.PodCliqueTemplateSpec{ @@ -140,12 +142,12 @@ func RenderSelectedNodeLocal( stage := projection.stage if index == 0 || stage != projections[index-1].stage { template = input.Stages[stage] - storage, err := lpuModelStorageBinding(template.Spec) + storagePath, err := lpuModelStoragePath(template.Spec) if err != nil { return nil, fmt.Errorf("stage %s: %w", stage, err) } - if storage.mount.MountPath != modelStorage.mount.MountPath { - return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStorage.mount.MountPath) + if storagePath != modelStoragePath { + return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStoragePath) } var conductorSpec *corev1.PodSpec if stage == conductorStage && conductor != nil { @@ -206,7 +208,7 @@ func RenderSelectedNodeLocal( cyborg, projections[0], workloadDigest, - modelStorage, + modelStoragePath, agentTemplateNames, cyborgConfigMap, ); err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 9f1a360772e7..afd298596697 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -16,7 +16,6 @@ import ( grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -156,30 +155,9 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { t.Parallel() - t.Log("Project an LPU-only model with nested runtime settings") + t.Log("Project an LPU-only model with template-owned runtime settings") snapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) projection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) - projection.configuredBuild.runtimeSettings["tokenizer_path"] = "tokenizer" - projection.configuredBuild.runtimeSettings["stop_tokens"] = []uint32{1} - projection.configuredBuild.runtimeSettings["swa"] = map[string]any{"chunked": true} - projection.configuredBuild.runtimeSettings["scheduler"] = map[string]any{"max_inflight_tasks": int64(3)} - retainedSetup := map[string]any{"custom": "kept"} - projection.configuredBuild.runtimeSettings["setup"] = retainedSetup - - t.Log("Render and verify isolated model configuration state") - firstConfig, err := lpuModelConfig([]*ModelProjection{projection}, "/models") - require.NoError(t, err) - require.EqualValues(t, 3, firstConfig["scheduler"].(map[string]any)["max_inflight_tasks"]) - require.NotContains(t, firstConfig["iop"], "scheduler") - require.NotContains(t, firstConfig["iop"], "setup") - - t.Log("Mutate one rendered config and verify the next render is isolated") - firstConfig["setup"].(map[string]any)["mutated"] = true - secondConfig, err := lpuModelConfig([]*ModelProjection{projection}, "/models") - require.NoError(t, err) - require.NotContains(t, secondConfig["setup"], "mutated") - require.Equal(t, map[string]any{"custom": "kept"}, retainedSetup) - require.Equal(t, map[string]any{"chunked": true}, projection.configuredBuild.runtimeSettings["swa"]) t.Log("Construct a base PodSpec with model binding and custom placement") modelAnnotationSource := &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ @@ -211,8 +189,9 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { require.Equal(t, conductorPodSpec.NodeSelector, conductor.Spec.PodSpec.NodeSelector) require.Equal(t, []string{"/bin/bash"}, conductor.Spec.PodSpec.Containers[0].Command) require.Equal(t, []string{"-c", "custom-agent"}, conductor.Spec.PodSpec.Containers[0].Args[:2]) - require.Equal(t, modelAnnotationSource, conductor.Spec.PodSpec.Containers[0].Env[0].ValueFrom) - require.Empty(t, conductor.Spec.PodSpec.Containers[0].Env[0].Value) + require.Contains(t, conductor.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{ + Name: "LPU_MODEL_NAME", ValueFrom: modelAnnotationSource, + }) require.Equal(t, "agt", testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, allocationEnvVar)) require.Equal(t, []string{"agt"}, conductor.Spec.StartsAfter) @@ -228,7 +207,7 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { require.Equal(t, projection.Model(), agent.Annotations[lpxv1alpha1.PodModelAnnotation]) } -func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { +func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { t.Parallel() t.Log("Create distinct draft and target compiler fixtures") @@ -244,10 +223,6 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) draft.Replicas = ptr.To(int32(2)) - draft.LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{ - "max_swa_dkvc_blocks_draft":2, - "setup":{"agent_connect_timeout":"30s","agent_setup_timeout":"180s"} - }`)} target := testLPXComponent(testTargetStageName, "target-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, @@ -267,9 +242,13 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { conductorTemplate.Labels = map[string]string{"owner": "conductor"} conductorTemplate.Annotations = map[string]string{"owner": "conductor"} conductorTemplate.Spec.Containers[0].Image = "conductor-runtime" - conductorTemplate.Spec.Containers[0].Env = []corev1.EnvVar{{ - Name: "NOVA_NODE_NAME_TEMPLATE", Value: "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE}", - }} + conductorTemplate.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "NOVA_NODE_NAME_TEMPLATE", Value: "${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE}"}, + {Name: "NOVA_PIPELINE_TYPE", Value: "SpecDecode"}, + {Name: "NOVA_MAX_SWA_DKVC_BLOCKS_DRAFT", Value: "2"}, + {Name: "NOVA_AGENT_CONNECT_TIMEOUT", Value: "30s"}, + {Name: "NOVA_AGENT_SETUP_TIMEOUT", Value: "180s"}, + } before := source.DeepCopy() t.Log("Resolve and render the authored speculative workload") @@ -309,20 +288,13 @@ func TestRenderSpecDecodeRoleOwnershipAndSharedSettings(t *testing.T) { require.Equal(t, secondAgentBefore, &secondAgent.Spec.PodSpec) require.Equal(t, conductorBefore, &conductor.Spec.PodSpec) - t.Log("Project draft settings once into the shared SpecDecode configuration") + t.Log("Preserve runtime settings in the conductor template and share only partition data") configMap, ok := extraResources[0].(*corev1.ConfigMap) require.True(t, ok) - require.Contains(t, configMap.Data["model_config.toml"], "SpecDecode") - require.Contains(t, configMap.Data["model_config.toml"], "[draft") - require.Contains(t, configMap.Data["model_config.toml"], "[target") - require.NotContains(t, configMap.Data["model_config.toml"], "num_drafts") - require.NotContains(t, configMap.Data["model_config.toml"], "draft_to_target_port") - require.NotContains(t, configMap.Data["model_config.toml"], "target_to_draft_port") - require.NotContains(t, configMap.Data["model_config.toml"], "head_to_head_port") - require.Contains(t, configMap.Data["model_config.toml"], "max_swa_dkvc_blocks_draft = 2") - require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_connect_timeout = '30s'")) - require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "agent_setup_timeout = '180s'")) - require.Equal(t, 1, strings.Count(configMap.Data["model_config.toml"], "[draft.iop]")) + require.NotContains(t, configMap.Data, "model_config.toml") + for _, env := range conductorTemplate.Spec.Containers[0].Env { + require.Contains(t, conductor.Spec.PodSpec.Containers[0].Env, env) + } require.Equal(t, "draft0\ndraft1\ntarget", configMap.Data["partition_models"]) require.Equal(t, before, source) diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index 11d4fe63284c..b64b64f54330 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -261,9 +261,6 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Preserve compiled placement without repeating runtime-derived model settings") for _, projection := range projections { require.EqualValues(t, compiledAgentCount, projection.agentReplicas) - if test.family == BuildFamilyHX { - require.NotContains(t, projection.configuredBuild.runtimeSettings, "sequence_length") - } } for _, expansion := range []struct { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 614a85619fd3..0184f47be7dc 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -6,6 +6,9 @@ package lpx import ( + "fmt" + "slices" + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" @@ -43,6 +46,36 @@ func configureNodeLocalConductorRuntime( retargetMainContainerReferences(conductorPodSpec, conductor) } +// applyConductorModelPaths binds nonempty canonical projections into the fresh conductor container. +func applyConductorModelPaths(container *corev1.Container, projections []*ModelProjection, modelStoragePath string) error { + type modelPathBinding struct { + name string + projection *ModelProjection + } + bindings := []modelPathBinding{{"LPX_MODEL_PATH", projections[0]}} + if projections[0].pipeline == PipelineSpecDecode { + bindings[0].name = "LPX_DRAFT_MODEL_PATH" + bindings = append(bindings, modelPathBinding{"LPX_TARGET_MODEL_PATH", projections[len(projections)-1]}) + } + + // Resolve all paths before publishing authoritative values ahead of authored references. + env := make([]corev1.EnvVar, 0, len(container.Env)+len(bindings)) + for _, binding := range bindings { + path, err := buildRuntimePath(lpuRuntimeBuildRef(binding.projection, modelStoragePath), modelStoragePath) + if err != nil { + return fmt.Errorf("resolve %s: %w", binding.name, err) + } + env = append(env, corev1.EnvVar{Name: binding.name, Value: path}) + } + for _, variable := range container.Env { + if !slices.ContainsFunc(bindings, func(binding modelPathBinding) bool { return binding.name == variable.Name }) { + env = append(env, variable) + } + } + container.Env = env + return nil +} + // configureAgentIdentity names the main runtime and retargets its container references. // agentPodSpec must be non-nil and contain the validated main container. func configureAgentIdentity(agentPodSpec *corev1.PodSpec) *corev1.Container { diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings.go b/deploy/operator/internal/dynamo/lpx/runtime_settings.go deleted file mode 100644 index 571ef2d00382..000000000000 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings.go +++ /dev/null @@ -1,160 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "encoding/json" - "fmt" - "maps" - "math/big" - "slices" - "strconv" - "strings" -) - -// maxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. -const maxSpecDecodeNumDrafts = 8 - -// resolveBuildSettings requires a nonnil build; it normalizes and consumes modelSettings, which callers must not reuse. -func resolveBuildSettings(build *Build, modelSettings map[string]any) error { - // Preserve normalized defaults before reusing the compiler-owned field for model overrides. - defaults := build.runtimeSettings - build.runtimeSettings = modelSettings - splitCount, overridesSplitCount := modelSettings["num_batch_split_divisions"] - - // Overlay model settings before consuming the selected prop-sync and embedding partitions. - merged, err := mergeRuntimeSettingOverride(defaults, build.runtimeSettings, "settings") - if err != nil { - return err - } - settings := merged.(map[string]any) - - // Restore the original split count for exact validation after merging settings. - if overridesSplitCount { - settings["num_batch_split_divisions"] = splitCount - } - - // Preserve the selected prop-sync and embedding transformations on the merged settings. - cpuEmbeddings := build.SupportsCPUEmbeddings - if _, overridden := settings["cpu_embeddings"]; overridden { - cpuEmbeddings, err = boolSetting(settings, "cpu_embeddings") - if err != nil { - return fmt.Errorf("settings.%w", err) - } - } - if err := build.consumeRuntimeSelectedPropSyncChain(); err != nil { - return err - } - if cpuEmbeddings { - build.omitStandaloneEmbeddingPartition() - } - build.runtimeSettings = settings - - return normalizeLegacyNovaSettings(build.runtimeSettings) -} - -func mergeRuntimeSettingOverride(base, override any, path string) (any, error) { - if override == nil { - return nil, fmt.Errorf("%s must not be null", path) - } - - switch typed := override.(type) { - case map[string]any: - baseMap, _ := base.(map[string]any) - merged := make(map[string]any, len(baseMap)+len(typed)) - for key, value := range baseMap { - if _, overridden := typed[key]; !overridden { - merged[key] = value - } - } - for _, key := range slices.Sorted(maps.Keys(typed)) { - overrideValue := typed[key] - value, err := mergeRuntimeSettingOverride(baseMap[key], overrideValue, path+"."+key) - if err != nil { - return nil, err - } - if _, numeric := overrideValue.(json.Number); numeric { - typed[key] = value - } - merged[key] = value - } - return merged, nil - case []any: - if typed == nil { - return []any{}, nil - } - for i, element := range typed { - value, err := mergeRuntimeSettingOverride(nil, element, fmt.Sprintf("%s[%d]", path, i)) - if err != nil { - return nil, err - } - typed[i] = value - } - return typed, nil - case json.Number: - number := typed.String() - if !strings.ContainsAny(number, ".eE") { - if integer, err := strconv.ParseInt(number, 10, 64); err == nil { - return integer, nil - } - } - if float, err := strconv.ParseFloat(number, 64); err == nil { - return float, nil - } - return typed, nil - default: - return override, nil - } -} - -// normalizeLegacyNovaSettings validates legacy Agent V2 controls and removes -// their accepted defaults. It consumes caller-owned settings without touching build geometry. -func normalizeLegacyNovaSettings(iop map[string]any) error { - // Reject unsupported folding before omitting its accepted false value. - batchFolding, err := boolSetting(iop, "batch_folding") - if err != nil { - return fmt.Errorf("iop.batch_folding must be a boolean") - } - if batchFolding { - return fmt.Errorf("iop.batch_folding=true is not supported") - } - - // Accept only zero or one, preserving exact authored JSON numbers. - if value, ok := iop["num_batch_split_divisions"]; ok { - var numBatchSplitDivisions *big.Rat - switch typed := value.(type) { - case int64: - numBatchSplitDivisions = new(big.Rat).SetInt64(typed) - case float64: - numBatchSplitDivisions = new(big.Rat).SetFloat64(typed) - case json.Number: - numBatchSplitDivisions, _ = new(big.Rat).SetString(typed.String()) - } - if numBatchSplitDivisions == nil || !numBatchSplitDivisions.IsInt() { - return fmt.Errorf("iop.num_batch_split_divisions must be an integer") - } - if numBatchSplitDivisions.Sign() < 0 || numBatchSplitDivisions.Cmp(big.NewRat(1, 1)) > 0 { - return fmt.Errorf("iop.num_batch_split_divisions=%v is not supported", value) - } - } - - // Nova no longer accepts either key, even when the value matches its default. - delete(iop, "batch_folding") - delete(iop, "num_batch_split_divisions") - return nil -} - -func boolSetting(values map[string]any, key string) (bool, error) { - value, ok := values[key] - if !ok { - return false, nil - } - boolValue, ok := value.(bool) - if !ok { - return false, fmt.Errorf("%s must be a boolean", key) - } - return boolValue, nil -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go b/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go deleted file mode 100644 index 57e871f81f11..000000000000 --- a/deploy/operator/internal/dynamo/lpx/runtime_settings_test.go +++ /dev/null @@ -1,308 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - "maps" - "slices" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestResolveBuildSettingsSelectedPropSyncChainWithCPUEmbeddings(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - cpuSupported, standalone, cpuDisabled bool - firstPartition int - }{ - {"standalone CPU embeddings", true, true, false, 1}, - {"CPU embeddings unsupported", false, true, false, 0}, - {"partition zero is runnable", true, false, false, 0}, - {"explicitly disabled CPU embeddings", true, true, true, 0}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Construct a selected chain with independent embedding placement requirements") - build := selectedPropSyncBuildWithTopologies(t, - []string{registryTestTopology, registryTestTopology, registryTestTopology}, [][]int{{0, 1, 2}}) - build.SupportsCPUEmbeddings, build.StandaloneTokenEmbeddings = test.cpuSupported, test.standalone - settings := map[string]any{} - if test.cpuDisabled { - settings["cpu_embeddings"] = false - } - partitions := slices.Clone(build.Partitions) - configured := *build - - t.Log("Resolve the selected chain and omit partition zero only when both flags permit it") - require.NoError(t, resolveBuildSettings(&configured, settings)) - require.Equal(t, partitions[test.firstPartition:], configured.Partitions) - require.Empty(t, configured.SelectedPropSyncChains) - require.Equal(t, settings, configured.runtimeSettings) - - t.Log("Keep the original partition IDs, paths and selected chain unchanged") - require.Equal(t, partitions, build.Partitions) - require.Equal(t, [][]int{{0, 1, 2}}, build.SelectedPropSyncChains) - }) - } -} - -func TestResolveBuildSettingsRejectsInvalidSelectedPropSyncChain(t *testing.T) { - t.Parallel() - - t.Log("Define malformed or incompatible selected prop-sync chains") - tests := []struct { - name string - chains [][]int - wantErr string - }{ - { - name: "multiple chains", - chains: [][]int{{0, 1}, {2, 3}}, - wantErr: "LPU-only runtime requires exactly one selected prop-sync chain, got 2", - }, - { - name: "missing partition", - chains: [][]int{{3, 4}}, - wantErr: "references missing partition id 4", - }, - { - name: "duplicate partition", - chains: [][]int{{1, 2, 1}}, - wantErr: "contains duplicate partition id 1", - }, - { - name: "noncontiguous partitions", - chains: [][]int{{1, 3}}, - wantErr: "is not contiguous at partition id 3", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - t.Log("Configure malformed or incompatible LPU-only selected-chain metadata") - build := selectedPropSyncBuild(t, tt.chains) - build.runtimeSettings = map[string]any{"cpu_embeddings": false} - configured := *build - - t.Log("Reject the build without consuming its selected-chain marker") - err := resolveBuildSettings(&configured, nil) - require.ErrorContains(t, err, tt.wantErr) - require.Equal(t, tt.chains, configured.SelectedPropSyncChains) - }) - } -} - -func TestResolveBuildSettingsDeepMergesDefaultsAndOverrides(t *testing.T) { - t.Parallel() - - t.Log("Construct paired runtime defaults and model overrides") - defaults, overrides := map[string]any{}, map[string]any{} - for name, values := range map[string][2]any{ - "batch_size": {int64(1), int64(2)}, - "sequence_length": {int64(131072), int64(4096)}, - "input_size": {int64(1), int64(2)}, - "num_kv_caches": {int64(1), int64(2)}, - "cpu_embeddings": {true, false}, - "dkvc": {true, false}, - "num_dkvc_blocks": {int64(256), int64(64)}, - "prop_sync": {true, false}, - "prop_sync_back_pressure": {false, true}, - "prop_sync_multiple_load_sets": {false, true}, - "has_init_tx": {false, true}, - "extended_topology": {true, false}, - "c2c_error_scan": {false, true}, - "num_batch_split_divisions": {int64(2), int64(0)}, - "batch_folding": {true, false}, - "num_layers": {int64(24), int64(25)}, - "vocab_size": {int64(201088), int64(201089)}, - "tokenizer_path": {"/build/tokenizer", "/dgd/tokenizer"}, - "stop_tokens": {[]uint32{200002, 199999, 200012}, []any{int64(42)}}, - "swa": { - map[string]any{ - "chunked": false, "num_swa_dkvc_blocks": int64(1), "swa_ctx_len": int64(128), - "swa_num_users": int64(8), "swa_padding_len": int64(0), - }, - map[string]any{ - "chunked": true, "num_swa_dkvc_blocks": int64(2), "swa_ctx_len": int64(256), - "swa_num_users": int64(4), "swa_padding_len": int64(16), - }, - }, - } { - defaults[name], overrides[name] = values[0], values[1] - } - overrides["custom_setting"] = "preserved" - overrides["custom_array"] = []any{map[string]any{"nested": []any{int64(1)}}} - build := &Build{CompilationMode: BuildCompilationModeLPUOnly, runtimeSettings: defaults} - - t.Log("Resolve overrides and verify every explicit model value wins") - require.NoError(t, resolveBuildSettings(build, overrides)) - expectedSettings := maps.Clone(overrides) - delete(expectedSettings, "batch_folding") - delete(expectedSettings, "num_batch_split_divisions") - require.Equal(t, expectedSettings, build.runtimeSettings, "every supported explicit DGD value must win") - - t.Log("Verify merge ownership and copy-on-write behavior") - overrideTokens := overrides["stop_tokens"].([]any) - configuredTokens := build.runtimeSettings["stop_tokens"].([]any) - require.Same(t, &overrideTokens[0], &configuredTokens[0], "caller-owned override slices are consumed") - empty, err := mergeRuntimeSettingOverride(nil, []any(nil), "settings.empty") - require.NoError(t, err) - require.NotNil(t, empty.([]any), "typed-nil arrays retain normalized empty-slice shape") - build.runtimeSettings["custom_setting"] = "changed" - require.Equal(t, "preserved", overrides["custom_setting"], "the settings map header remains independent") - build.runtimeSettings["swa"].(map[string]any)["swa_ctx_len"] = int64(512) - require.Equal(t, int64(128), defaults["swa"].(map[string]any)["swa_ctx_len"]) - require.Equal(t, uint32(200002), defaults["stop_tokens"].([]uint32)[0]) - require.Equal(t, int64(256), overrides["swa"].(map[string]any)["swa_ctx_len"], "overridden maps remain copy-on-write") - - t.Log("Complete a partial nested override from immutable defaults") - sharedDefault := []uint32{1} - partialDefaults := map[string]any{ - "shared": sharedDefault, - "swa": map[string]any{ - "chunked": false, - "num_swa_dkvc_blocks": int64(1), - "swa_ctx_len": int64(128), - "swa_num_users": int64(8), - "swa_padding_len": int64(0), - }, - } - partial := &Build{CompilationMode: BuildCompilationModeLPUOnly, runtimeSettings: partialDefaults} - require.NoError(t, resolveBuildSettings(partial, map[string]any{"swa": map[string]any{"chunked": true}})) - require.Equal(t, map[string]any{ - "chunked": true, - "num_swa_dkvc_blocks": int64(1), - "swa_ctx_len": int64(128), - "swa_num_users": int64(8), - "swa_padding_len": int64(0), - }, partial.runtimeSettings["swa"]) - - t.Log("Verify untouched defaults remain shared while nested maps remain isolated") - configuredDefault := partial.runtimeSettings["shared"].([]uint32) - require.Same(t, &sharedDefault[0], &configuredDefault[0], "untouched defaults remain immutable shared evidence") - partial.runtimeSettings["swa"].(map[string]any)["swa_ctx_len"] = int64(256) - require.Equal(t, int64(128), partialDefaults["swa"].(map[string]any)["swa_ctx_len"]) -} - -func TestProjectModelLegacyNovaSettings(t *testing.T) { - t.Parallel() - - t.Log("Capture the Nova settings path before runtime-specific overrides") - singleIntent := ModelProjectionInput{ - Models: []string{"default"}, Pipeline: PipelineSingle, - BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), - } - singleBaselineBatch, err := appendModelProjections(nil, singleIntent) - require.NoError(t, err) - singleBaseline := singleBaselineBatch[0] - - t.Log("Define accepted legacy defaults and unsupported values") - tests := []struct { - settings, wantErr string - }{ - {`{"batch_folding":false}`, ""}, - {`{"num_batch_split_divisions":0}`, ""}, - {`{"num_batch_split_divisions":1}`, ""}, - {`{"num_batch_split_divisions":1.0}`, ""}, - {`{"num_batch_split_divisions":10e-1}`, ""}, - {`{"num_batch_split_divisions":-0.0}`, ""}, - {`{"batch_folding":true}`, `iop.batch_folding=true is not supported`}, - {`{"batch_folding":"false"}`, "iop.batch_folding must be a boolean"}, - {`{"num_batch_split_divisions":-1}`, `iop.num_batch_split_divisions=-1 is not supported`}, - {`{"num_batch_split_divisions":2}`, `iop.num_batch_split_divisions=2 is not supported`}, - {`{"num_batch_split_divisions":-1.0}`, `iop.num_batch_split_divisions=-1.0 is not supported`}, - {`{"num_batch_split_divisions":0.5}`, "iop.num_batch_split_divisions must be an integer"}, - {`{"num_batch_split_divisions":0.99999999999999999999}`, "iop.num_batch_split_divisions must be an integer"}, - {`{"num_batch_split_divisions":1.00000000000000000001}`, "iop.num_batch_split_divisions must be an integer"}, - {`{"num_batch_split_divisions":1e-1000}`, "iop.num_batch_split_divisions must be an integer"}, - {`{"num_batch_split_divisions":"1"}`, "iop.num_batch_split_divisions must be an integer"}, - } - - for _, test := range tests { - intent, baseline := singleIntent, singleBaseline - t.Run(string(intent.Pipeline)+"/"+test.settings, func(t *testing.T) { - t.Log("Validate model-provided JSON settings before rendering") - intent.ModelSettings = []byte(test.settings) - projectionBatch, err := appendModelProjections(nil, intent) - if test.wantErr != "" { - require.ErrorContains(t, err, test.wantErr) - return - } - require.NoError(t, err) - projection := projectionBatch[0] - - t.Log("Preserve physical partitions, runtime partitions, and prop-sync connectors") - require.Equal(t, baseline.partitions, projection.partitions) - require.Equal(t, baseline.configuredBuild.Partitions, projection.configuredBuild.Partitions) - require.Equal(t, baseline.connectors, projection.connectors) - - t.Log("Omit legacy Nova settings from runtime output") - require.NotContains(t, projection.configuredBuild.runtimeSettings, "batch_folding") - require.NotContains(t, projection.configuredBuild.runtimeSettings, "num_batch_split_divisions") - configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", []*ModelProjection{projection}) - require.NoError(t, err) - require.NotContains(t, configMap.Data["model_config.toml"], "batch_folding") - require.NotContains(t, configMap.Data["model_config.toml"], "num_batch_split_divisions") - }) - } -} - -func TestResolveBuildSettingsRejectsNestedNullOverride(t *testing.T) { - t.Parallel() - - t.Log("Reject a null value in a nested known runtime setting") - build := &Build{runtimeSettings: map[string]any{ - "swa": map[string]any{"chunked": false}, - }} - - err := resolveBuildSettings(build, map[string]any{ - "swa": map[string]any{"chunked": nil}, - }) - require.ErrorContains(t, err, "settings.swa.chunked must not be null") - - t.Log("Reject a null value nested inside a custom array") - build = &Build{} - err = resolveBuildSettings(build, map[string]any{ - "custom": []any{map[string]any{"value": nil}}, - }) - require.ErrorContains(t, err, "settings.custom[0].value must not be null") - - t.Log("Report deterministic field order when multiple settings are null") - build = &Build{} - err = resolveBuildSettings(build, map[string]any{"z": nil, "a": nil}) - require.ErrorContains(t, err, "settings.a must not be null") -} - -func selectedPropSyncBuild(t *testing.T, chains [][]int) *Build { - t.Helper() - - topology := registryTestTopology - return selectedPropSyncBuildWithTopologies(t, []string{topology, topology, topology, topology}, chains) -} - -func selectedPropSyncBuildWithTopologies(t *testing.T, topologies []string, chains [][]int) *Build { - t.Helper() - - partitions := make([]BuildPartition, len(topologies)) - for i, raw := range topologies { - topology, err := parse(raw) - require.NoError(t, err) - partitions[i] = BuildPartition{SourcePartitionID: i, PartPath: fmt.Sprintf("part-%d", i), Topology: topology} - } - return &Build{ - Path: "file:///path/to/build-id", - CompilationMode: BuildCompilationModeLPUOnly, - Partitions: partitions, - StandaloneTokenEmbeddings: true, - SupportsCPUEmbeddings: true, - SelectedPropSyncChains: chains, - } -} diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index decac9679aa9..a1068319f3cd 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -6,6 +6,7 @@ package lpx import ( + "fmt" "slices" "testing" @@ -143,3 +144,94 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { } } } + +func TestConductorModelPathsPrecedeAuthoredReferences(t *testing.T) { + for _, test := range []struct { + name string + family BuildFamily + pipeline Pipeline + local bool + }{ + {name: "XT Single GCS", family: BuildFamilyXT, pipeline: PipelineSingle}, + {name: "HX Single local", family: BuildFamilyHX, pipeline: PipelineSingle, local: true}, + {name: "XT multiple drafts local", family: BuildFamilyXT, pipeline: PipelineSpecDecode, local: true}, + {name: "HX multiple drafts GCS", family: BuildFamilyHX, pipeline: PipelineSpecDecode}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Resolve each model from the registry reference and the conductor's custom mount") + count := 1 + names := []string{"LPX_MODEL_PATH"} + if test.pipeline == PipelineSpecDecode { + count = 3 + names = []string{"LPX_DRAFT_MODEL_PATH", "LPX_TARGET_MODEL_PATH"} + } + projections := make([]*ModelProjection, count) + for index := range projections { + buildID := fmt.Sprintf("model-%d", index) + path := "gs://registry/" + buildID + if test.local { + path = "file:///operator-cache/" + buildID + } + projections[index] = &ModelProjection{ + pipeline: test.pipeline, runtimeBuildRef: buildID, + configuredBuild: Build{Family: test.family, Path: path}, + } + } + want := []corev1.EnvVar{} + for index, name := range names { + path := fmt.Sprintf("/custom/models/gcs/registry/model-%d", index*(count-1)) + if test.local { + path = fmt.Sprintf("/custom/models/model-%d", index*(count-1)) + } + want = append(want, corev1.EnvVar{Name: name, Value: path}) + } + + t.Log("Author dependent values before duplicate forged bindings") + authored := []corev1.EnvVar{ + {Name: "A_RUNTIME_MODEL", Value: "$(" + names[0] + ")"}, + {Name: "OTHER", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + } + container := corev1.Container{Command: []string{"custom-runtime"}, Args: []string{"--unchanged"}, Env: slices.Clone(authored)} + for _, name := range names { + container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: "stale"}, corev1.EnvVar{Name: name, Value: "duplicate"}) + } + wantContainer := container.DeepCopy() + wantContainer.Env = append(want, authored...) + + t.Log("Publish only the first draft and final target once, before all authored references") + require.NoError(t, applyConductorModelPaths(&container, projections, "/custom/models")) + require.Equal(t, *wantContainer, container) + + t.Log("Repeated rendering keeps environment order, startup and authoritative values unchanged") + require.NoError(t, applyConductorModelPaths(&container, projections, "/custom/models")) + require.Equal(t, *wantContainer, container) + }) + } +} + +func TestConductorModelPathsRejectInvalidReferences(t *testing.T) { + for _, test := range []struct { + pipeline Pipeline + name string + }{ + {pipeline: PipelineSingle, name: "LPX_MODEL_PATH"}, + {pipeline: PipelineSpecDecode, name: "LPX_TARGET_MODEL_PATH"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Keep the target reference invalid after a valid speculative draft") + projections := []*ModelProjection{{pipeline: test.pipeline, configuredBuild: Build{Path: "gs://registry/model"}}} + if test.pipeline == PipelineSpecDecode { + projections = append(projections, &ModelProjection{pipeline: test.pipeline}) + } + projections[len(projections)-1].configuredBuild.Path = "gs://registry/../outside" + container := corev1.Container{Env: []corev1.EnvVar{{Name: "KEEP", Value: "unchanged"}}} + before := container.DeepCopy() + + t.Log("Report the failing binding without publishing a partial environment") + err := applyConductorModelPaths(&container, projections, "/custom/models") + require.ErrorContains(t, err, "resolve "+test.name) + require.ErrorContains(t, err, "bad path segment") + require.Equal(t, *before, container) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go index f84fb0ace1dc..8e356100983e 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ b/deploy/operator/internal/dynamo/lpx/selection_validation.go @@ -16,6 +16,9 @@ import ( "k8s.io/utils/ptr" ) +// maxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. +const maxSpecDecodeNumDrafts = 8 + // ValidateSelectedIntent validates the snapshot-independent selected-LPX // requirements beyond admission's structural checks before controller downloads // and selected-workload projection. diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 2a92e81f4f9b..44d93e6fbd6f 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -40,8 +40,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -63,6 +61,10 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_MODEL_PATH + value: '$(LPX_MODEL_PATH)' + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: '/configs' - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 68f2db13338c..8e6249a0c897 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -2,14 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - model_config.toml: | - type = 'Single' - - [iop] - model_path = '/testdata/node-local-v2-connected' - - [setup] - resolved_partitions_dir = '/configs' nodes_per_partition: |- 2 2 @@ -33,7 +25,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 namespace: test --- metadata: @@ -55,7 +47,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: e9ba15e4ac831b0d6607f50bb0af48bae2bc56e3948e68850f78def4f0bbef8e + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 @@ -184,14 +176,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: e9ba15e4ac831b0d6607f50bb0af48bae2bc56e3948e68850f78def4f0bbef8e + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 scheduling.lpu.nvidia.com/role: conductor @@ -224,8 +216,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -263,12 +253,18 @@ spec: value: etcd-address - name: LPX_ALLOCATION value: agt + - name: LPX_MODEL_PATH + value: /nfs/node-local-v2-connected - name: NATS_SERVER value: nats-address - name: NOVA_MODEL_BASE_PATHS value: /nfs + - name: NOVA_MODEL_PATH + value: $(LPX_MODEL_PATH) - name: NOVA_NODE_NAME_TEMPLATE value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: /configs - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -352,7 +348,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-lpu-only-33b1-lpu-e9ba15e4ac831b0d + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index ad0995320dee..001fea7c74e7 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -121,8 +121,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -144,6 +142,16 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_PIPELINE_TYPE + value: 'SpecDecode' + - name: NOVA_DRAFT_MODEL_PATH + value: '$(LPX_DRAFT_MODEL_PATH)' + - name: NOVA_DRAFT_RESOLVED_PARTITIONS_DIR + value: '/configs' + - name: NOVA_TARGET_MODEL_PATH + value: '$(LPX_TARGET_MODEL_PATH)' + - name: NOVA_TARGET_RESOLVED_PARTITIONS_DIR + value: '/configs' - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 7a7a20ae65f8..a8a0a31c2b06 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -2,22 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - model_config.toml: | - type = 'SpecDecode' - - [draft] - [draft.iop] - model_path = '/testdata/node-local-v2-connected' - - [draft.setup] - resolved_partitions_dir = '/configs' - - [target] - [target.iop] - model_path = '/testdata/node-local-v2-connected' - - [target.setup] - resolved_partitions_dir = '/configs' nodes_per_partition: |- 2 2 @@ -75,7 +59,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 namespace: test --- metadata: @@ -97,7 +81,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 @@ -230,14 +214,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b @@ -370,14 +354,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e @@ -510,14 +494,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b21dd05cabd990be4e08973107ec1fc5609ff3556d9ed4dcd445c0df9c8f8f7a + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 scheduling.lpu.nvidia.com/role: conductor @@ -549,8 +533,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -588,12 +570,26 @@ spec: value: etcd-address - name: LPX_ALLOCATION value: agt0:agt1:agt2 + - name: LPX_DRAFT_MODEL_PATH + value: /nfs/node-local-v2-connected + - name: LPX_TARGET_MODEL_PATH + value: /nfs/node-local-v2-connected - name: NATS_SERVER value: nats-address + - name: NOVA_DRAFT_MODEL_PATH + value: $(LPX_DRAFT_MODEL_PATH) + - name: NOVA_DRAFT_RESOLVED_PARTITIONS_DIR + value: /configs - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} + - name: NOVA_PIPELINE_TYPE + value: SpecDecode + - name: NOVA_TARGET_MODEL_PATH + value: $(LPX_TARGET_MODEL_PATH) + - name: NOVA_TARGET_RESOLVED_PARTITIONS_DIR + value: /configs - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -677,7 +673,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-specdecode-63df-lpu-b21dd05cabd990be + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index 9d7925788e89..50451eb85667 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -39,8 +39,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -102,6 +100,10 @@ spec: command: - /bin/nova env: + - name: NOVA_MODEL_PATH + value: '$(LPX_MODEL_PATH)' + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: '/configs' - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 835a742f366c..c11171bfd2b0 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -2,14 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - model_config.toml: | - type = 'Single' - - [iop] - model_path = '/nfs/node-local-v3-hx-connected' - - [setup] - resolved_partitions_dir = '/configs' nodes_per_partition: "1" partition_ids: "1" partition_indices: "0" @@ -25,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd namespace: test --- metadata: @@ -47,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 396d1555d763422281cc6c27d516f81737c8671ab6f880ade54570435a474152 + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 @@ -206,14 +198,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 396d1555d763422281cc6c27d516f81737c8671ab6f880ade54570435a474152 + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 scheduling.lpu.nvidia.com/role: conductor @@ -247,8 +239,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -324,12 +314,18 @@ spec: value: etcd-address - name: LPX_ALLOCATION value: agt + - name: LPX_MODEL_PATH + value: /nfs/node-local-v3-hx-connected - name: NATS_SERVER value: nats-address - name: NOVA_MODEL_BASE_PATHS value: /nfs + - name: NOVA_MODEL_PATH + value: $(LPX_MODEL_PATH) - name: NOVA_NODE_NAME_TEMPLATE value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: /configs - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -413,7 +409,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-396d1555d7634222 + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 2ce317a4e893..7b76189d0f48 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -150,8 +150,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -213,6 +211,16 @@ spec: command: - /bin/nova env: + - name: NOVA_PIPELINE_TYPE + value: 'SpecDecode' + - name: NOVA_DRAFT_MODEL_PATH + value: '$(LPX_DRAFT_MODEL_PATH)' + - name: NOVA_DRAFT_RESOLVED_PARTITIONS_DIR + value: '/configs' + - name: NOVA_TARGET_MODEL_PATH + value: '$(LPX_TARGET_MODEL_PATH)' + - name: NOVA_TARGET_RESOLVED_PARTITIONS_DIR + value: '/configs' - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 01e5615df135..3201171c3bb0 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -2,22 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - model_config.toml: | - type = 'SpecDecode' - - [draft] - [draft.iop] - model_path = '/nfs/node-local-v3-hx-sd-draft' - - [draft.setup] - resolved_partitions_dir = '/configs' - - [target] - [target.iop] - model_path = '/nfs/node-local-v3-hx-sd-target' - - [target.setup] - resolved_partitions_dir = '/configs' nodes_per_partition: |- 1 1 @@ -54,7 +38,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 namespace: test --- metadata: @@ -76,7 +60,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea @@ -239,14 +223,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed @@ -409,14 +393,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a @@ -579,14 +563,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 9ba1375a1e18c0ad8f9e55251644fe0aebbbc9b9a5183f1b40594beb12314b55 + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 scheduling.lpu.nvidia.com/role: conductor @@ -620,8 +604,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -697,12 +679,26 @@ spec: value: etcd-address - name: LPX_ALLOCATION value: agt0:agt1:agt2 + - name: LPX_DRAFT_MODEL_PATH + value: /nfs/node-local-v3-hx-sd-draft + - name: LPX_TARGET_MODEL_PATH + value: /nfs/node-local-v3-hx-sd-target - name: NATS_SERVER value: nats-address + - name: NOVA_DRAFT_MODEL_PATH + value: $(LPX_DRAFT_MODEL_PATH) + - name: NOVA_DRAFT_RESOLVED_PARTITIONS_DIR + value: /configs - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} + - name: NOVA_PIPELINE_TYPE + value: SpecDecode + - name: NOVA_TARGET_MODEL_PATH + value: $(LPX_TARGET_MODEL_PATH) + - name: NOVA_TARGET_RESOLVED_PARTITIONS_DIR + value: /configs - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -786,7 +782,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-9ba1375a1e18c0ad + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml index d8596c31e3b1..61df6a27fc03 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml @@ -17,17 +17,6 @@ spec: replicas: 1 lpx: buildId: llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - settings: - batch_size: 1 - input_size: 1 - num_kv_caches: 1 - sequence_length: 8192 - stop_tokens: - - 128001 - - 128008 - - 128009 - tokenizer_path: llama3.1 - vocab_size: 128256 roles: - name: conductor podTemplate: @@ -50,8 +39,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -73,6 +60,24 @@ spec: - --allocation - $(LPX_ALLOCATION) env: + - name: NOVA_BATCH_SIZE + value: '1' + - name: NOVA_INPUT_SIZE + value: '1' + - name: NOVA_MODEL_PATH + value: '$(LPX_MODEL_PATH)' + - name: NOVA_NUM_KV_CACHES + value: '1' + - name: NOVA_SEQUENCE_LENGTH + value: '8192' + - name: NOVA_STOP_TOKENS + value: '128001,128008,128009' + - name: NOVA_TOKENIZER_PATH + value: 'llama3.1' + - name: NOVA_VOCAB_SIZE + value: '128256' + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: '/configs' - name: NOVA_MODEL_BASE_PATHS value: /nfs - name: NOVA_NODE_NAME_TEMPLATE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 5fccd9b5a001..9fa6e9f4a93a 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -2,21 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - model_config.toml: | - type = 'Single' - - [iop] - batch_size = 1 - input_size = 1 - model_path = '/testdata/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c' - num_kv_caches = 1 - sequence_length = 8192 - stop_tokens = [128001, 128008, 128009] - tokenizer_path = 'llama3.1' - vocab_size = 128256 - - [setup] - resolved_partitions_dir = '/configs' nodes_per_partition: |- 1 1 @@ -115,7 +100,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-single-v2-3a98-lpu-bd962bcf06374376 + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 namespace: inference-engine --- metadata: @@ -123,7 +108,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 @@ -137,10 +122,10 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: bd962bcf06374376e0a6e9e786629713da2a5ddb698e5f351f8d27707e2335e5 + nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -270,16 +255,16 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-3a98-lpu-bd962bcf06374376 + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: bd962bcf06374376e0a6e9e786629713da2a5ddb698e5f351f8d27707e2335e5 + nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -311,8 +296,6 @@ spec: - $(POD_IP) - --control-server-addr - $(POD_IP):51045 - - --model-config-filepath - - /configs/model_config.toml - --json-logs - --disable-tracing - --mpi-use-tcp @@ -350,14 +333,34 @@ spec: value: etcd-address - name: LPX_ALLOCATION value: agt + - name: LPX_MODEL_PATH + value: /nfs/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: MPI_BTL_TCP_IF_INCLUDE value: 192.0.2.0/24 - name: NATS_SERVER value: nats-address + - name: NOVA_BATCH_SIZE + value: "1" + - name: NOVA_INPUT_SIZE + value: "1" - name: NOVA_MODEL_BASE_PATHS value: /nfs + - name: NOVA_MODEL_PATH + value: $(LPX_MODEL_PATH) - name: NOVA_NODE_NAME_TEMPLATE value: ${GROVE_PCSG_NAME}-${GROVE_PCSG_INDEX}-{rack}-{node}.${GROVE_HEADLESS_SERVICE} + - name: NOVA_NUM_KV_CACHES + value: "1" + - name: NOVA_RESOLVED_PARTITIONS_DIR + value: /configs + - name: NOVA_SEQUENCE_LENGTH + value: "8192" + - name: NOVA_STOP_TOKENS + value: 128001,128008,128009 + - name: NOVA_TOKENIZER_PATH + value: llama3.1 + - name: NOVA_VOCAB_SIZE + value: "128256" - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM @@ -443,7 +446,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: dgd-single-v2-3a98-lpu-bd962bcf06374376 + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 name: config replicas: 1 roleName: cond @@ -456,7 +459,7 @@ spec: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f41776b5eeefee5dab35cb81c9d4c052c88b10ac1025b0f408d690dec65d0ea1 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go index 0a862265d654..209973c8726a 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -12,6 +12,8 @@ import ( "encoding/json" "fmt" "hash" + "maps" + "slices" ) const ( @@ -28,6 +30,8 @@ const ( // and Pipeline is selected by the validated resolver. ModelSettings is API-validated // JSON: omitted, null or an object. Source inputs are not mutated; discard error results. func appendModelProjections(dst []*ModelProjection, intent ModelProjectionInput) ([]*ModelProjection, error) { + const propSyncSetting = "prop_sync" + modelSettings, modelSettingsObject := canonicalModelSettings(intent.ModelSettings) intent.ModelSettings = modelSettings @@ -35,10 +39,25 @@ func appendModelProjections(dst []*ModelProjection, intent ModelProjectionInput) if intent.Pipeline == PipelineLPX { for key := range modelSettingsObject { if intent.BuildSnapshot.build.Family != BuildFamilyXT || - (key != "prop_sync" && key != "prop_sync_multiple_load_sets") { + (key != propSyncSetting && key != "prop_sync_multiple_load_sets") { return nil, fmt.Errorf("hybrid %s workloads do not support runtime settings overrides", intent.BuildSnapshot.build.Family) } } + } else { + // Conductor templates own runtime configuration; model settings only affect placement. + for _, key := range slices.Sorted(maps.Keys(modelSettingsObject)) { + switch key { + case "cpu_embeddings", propSyncSetting, "prop_sync_multiple_load_sets": + if _, ok := modelSettingsObject[key].(bool); !ok { + return nil, fmt.Errorf("model settings.%s must be a boolean", key) + } + if intent.BuildSnapshot.build.Family == BuildFamilyHX && key != propSyncSetting { + return nil, fmt.Errorf("model settings.%s is not used for HX placement; configure runtime options in the conductor podTemplate", key) + } + default: + return nil, fmt.Errorf("model settings.%s is a runtime setting; configure it in the conductor podTemplate", key) + } + } } var projections []*ModelProjection diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go index 1f6a6a86e0be..2f67048a2a21 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -79,7 +79,7 @@ func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { require.NotEqual(t, firstProjection.Digest(), specDecodeProjection.Digest()) } -func TestCanonicalModelSettingsRetainRawAndNormalizeRuntimeNumbers(t *testing.T) { +func TestCanonicalModelSettingsRetainRaw(t *testing.T) { t.Log("Decode canonical model settings while retaining their exact raw bytes") raw, object := canonicalModelSettings(json.RawMessage( `{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`, @@ -87,16 +87,52 @@ func TestCanonicalModelSettingsRetainRawAndNormalizeRuntimeNumbers(t *testing.T) require.Equal(t, json.RawMessage(`{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`), raw) require.Equal(t, json.Number("8192"), object["integer"]) - t.Log("Normalize representable integers while retaining floating-point runtime values") - merged, err := mergeRuntimeSettingOverride(nil, object, "settings") - require.NoError(t, err) - normalized := merged.(map[string]any) - require.Equal(t, int64(8192), normalized["integer"]) - require.Equal(t, []any{int64(128001), float64(1.5), float64(9223372036854775808)}, normalized["nested"].(map[string]any)["array"]) - t.Log("Normalize explicit null settings to a present empty object") raw, object = canonicalModelSettings(json.RawMessage("null")) require.Nil(t, raw) require.NotNil(t, object) require.Empty(t, object) } + +func TestProjectModelSettingsPlacementBoundary(t *testing.T) { + t.Log("Prepare both physical families with conductor-owned runtime settings") + builds := []NormalizedBuildSnapshot{ + normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))), + normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), + } + for _, build := range builds { + for _, pipeline := range []Pipeline{PipelineSingle, PipelineSpecDecode} { + for _, test := range []struct { + settings, wantErr string + xtOnly bool + }{ + {settings: `{"sequence_length":4096}`, wantErr: "model settings.sequence_length is a runtime setting; configure it in the conductor podTemplate"}, + {settings: `{"setup":{"agent_setup_timeout":"30s"}}`, wantErr: "model settings.setup is a runtime setting"}, + {settings: `{"z":null,"a":true}`, wantErr: "model settings.a is a runtime setting"}, + {settings: `{"cpu_embeddings":"false"}`, wantErr: "model settings.cpu_embeddings must be a boolean"}, + {settings: `{"prop_sync":null}`, wantErr: "model settings.prop_sync must be a boolean"}, + {settings: `{"prop_sync_multiple_load_sets":1}`, wantErr: "model settings.prop_sync_multiple_load_sets must be a boolean"}, + {settings: `{"cpu_embeddings":true}`, xtOnly: true}, + {settings: `{"cpu_embeddings":false}`, xtOnly: true}, + {settings: `{"prop_sync_multiple_load_sets":true}`, xtOnly: true}, + {settings: `{"prop_sync_multiple_load_sets":false}`, xtOnly: true}, + {settings: `{"prop_sync":false}`}, + } { + t.Run(string(build.build.Family)+"/"+string(pipeline)+"/"+test.settings, func(t *testing.T) { + t.Log("Accept only placement controls consumed by the selected physical family") + _, err := appendModelProjections(nil, ModelProjectionInput{ + Pipeline: pipeline, Models: []string{"default"}, BuildSnapshot: build, + ModelSettings: json.RawMessage(test.settings), + }) + if test.xtOnly && build.build.Family == BuildFamilyHX { + require.ErrorContains(t, err, "is not used for HX placement; configure runtime options in the conductor podTemplate") + } else if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + } else { + require.NoError(t, err) + } + }) + } + } + } +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index 7e63f1cf29ea..d5b616e4c3c3 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -28,10 +28,22 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu ioFPGACount, ioFanoutFactor := build.IOFPGACount, build.IOFanoutFactor propSyncEnabled := build.CompilationMode == BuildCompilationModeLPUOnly && len(build.SelectedPropSyncChains) > 0 connectorBuild := intent.BuildSnapshot.build + + // Apply the selected chain and CPU embedding placement before deriving scheduler requests. if usesResolvedRuntime { - if err := resolveBuildSettings(&configured, modelSettings); err != nil { + cpuEmbeddings := configured.SupportsCPUEmbeddings + if value, overridden := modelSettings["cpu_embeddings"]; overridden { + cpuEmbeddings = value.(bool) + } + if err := configured.consumeRuntimeSelectedPropSyncChain(); err != nil { return nil, fmt.Errorf("resolving configured V2 build: %w", err) } + + // Omit host-only embeddings by retaining a view of the immutable source partitions. + if cpuEmbeddings && configured.SupportsCPUEmbeddings && configured.StandaloneTokenEmbeddings && + len(configured.Partitions) > 1 && configured.Partitions[0].SourcePartitionID == 0 { + configured.Partitions = configured.Partitions[1:] + } } if value, present := modelSettings["prop_sync"]; present { enabled, ok := value.(bool) diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index 6ea8a7677631..43d5db3fa9fd 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -143,23 +143,44 @@ func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { } func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { - t.Log("Prepare a V2 build with a standalone embedding partition in its selected chain") - normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) - build := normalized.build - build.Partitions[0].SourcePartitionID = 0 - build.Partitions[1].SourcePartitionID = 1 - build.StandaloneTokenEmbeddings = true - build.SupportsCPUEmbeddings = true - build.SelectedPropSyncChains = [][]int{{0, 1}} - - t.Log("Omit source partition zero when CPU embeddings are enabled") - offLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":true}`) - require.Equal(t, int64(1), offLPU.RequestSpec(&MaterializationPlan{}, "agents").Partitions[0].CompilerPartitionID) - - t.Log("Retain source partition zero when CPU embeddings are disabled") - onLPU := projectTestBuild(t, normalized, PipelineSingle, `{"cpu_embeddings":false}`) - require.Equal(t, 4, onLPU.agentReplicas) - require.Equal(t, int64(0), onLPU.RequestSpec(&MaterializationPlan{}, "agents").Partitions[0].CompilerPartitionID) + for _, test := range []struct { + name, settings string + cpuSupported, standalone bool + firstPartition int64 + }{ + {name: "default CPU embeddings", cpuSupported: true, standalone: true, firstPartition: 1}, + {name: "explicit CPU embeddings", settings: `{"cpu_embeddings":true}`, cpuSupported: true, standalone: true, firstPartition: 1}, + {name: "CPU embeddings disabled", settings: `{"cpu_embeddings":false}`, cpuSupported: true, standalone: true}, + {name: "CPU embeddings unsupported", standalone: true}, + {name: "CPU embeddings requested but unsupported", settings: `{"cpu_embeddings":true}`, standalone: true}, + {name: "partition zero is runnable", cpuSupported: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Prepare a selected V2 chain with explicit embedding placement capabilities") + normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + build := normalized.build + build.Partitions[0].SourcePartitionID = 0 + build.Partitions[1].SourcePartitionID = 1 + build.StandaloneTokenEmbeddings = test.standalone + build.SupportsCPUEmbeddings = test.cpuSupported + build.SelectedPropSyncChains = [][]int{{0, 1}} + + t.Log("Project retained source partitions into placement and runtime metadata") + projection := projectTestBuild(t, normalized, PipelineSingle, test.settings) + spec := projection.RequestSpec(&MaterializationPlan{}, "agents") + require.Equal(t, test.firstPartition, spec.Partitions[0].CompilerPartitionID) + require.EqualValues(t, 2*(2-test.firstPartition), projection.agentReplicas) + require.EqualValues(t, test.firstPartition, projection.configuredBuild.Partitions[0].SourcePartitionID) + require.Empty(t, projection.configuredBuild.SelectedPropSyncChains) + + t.Log("Keep scheduler output independently mutable from the projection and compiler snapshot") + spec.Partitions[0].CompilerPartitionID = -1 + require.EqualValues(t, test.firstPartition, projection.partitions[0].SourcePartitionID) + require.EqualValues(t, test.firstPartition, projection.RequestSpec(&MaterializationPlan{}, "agents").Partitions[0].CompilerPartitionID) + require.Equal(t, 0, build.Partitions[0].SourcePartitionID) + require.Equal(t, [][]int{{0, 1}}, build.SelectedPropSyncChains) + }) + } } func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { @@ -346,3 +367,54 @@ func TestProjectModelV2PreservesAgentReplicasWhenCollapsingSubHostPartitions(t * require.Equal(t, 2, projection.configuredBuild.Partitions[0].effectiveNodeCount()) require.Equal(t, 4, projection.configuredBuild.Partitions[0].Topology.ChipCount) } + +func TestBuildRejectsInvalidRuntimeSelectedPropSyncChain(t *testing.T) { + t.Parallel() + + t.Log("Define malformed or incompatible selected prop-sync chains") + tests := []struct { + name string + chains [][]int + wantErr string + }{ + { + name: "multiple chains", + chains: [][]int{{0, 1}, {2, 3}}, + wantErr: "LPU-only runtime requires exactly one selected prop-sync chain, got 2", + }, + { + name: "missing partition", + chains: [][]int{{3, 4}}, + wantErr: "references missing partition id 4", + }, + { + name: "duplicate partition", + chains: [][]int{{1, 2, 1}}, + wantErr: "contains duplicate partition id 1", + }, + { + name: "noncontiguous partitions", + chains: [][]int{{1, 3}}, + wantErr: "is not contiguous at partition id 3", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Log("Configure malformed or incompatible LPU-only selected-chain metadata") + configured := Build{ + Partitions: []BuildPartition{ + {SourcePartitionID: 0}, {SourcePartitionID: 1}, {SourcePartitionID: 2}, {SourcePartitionID: 3}, + }, + SelectedPropSyncChains: tt.chains, + } + + t.Log("Reject the build without consuming its selected-chain marker") + err := configured.consumeRuntimeSelectedPropSyncChain() + require.ErrorContains(t, err, tt.wantErr) + require.Equal(t, tt.chains, configured.SelectedPropSyncChains) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 2015525c5d31..5f370f40323f 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -16,7 +16,6 @@ import ( const ( v3CompilerEnvelopeSchema = "dynamo.lpx.v3-capnp/v1" v3ProjectionVersion = "v3-hx-capnp/v1" - v3MaxSWADKVCBlocksDraft = "max_swa_dkvc_blocks_draft" v3LPUDevice = "lpu" v3HXLogicalDeviceCount = 16 ) @@ -27,23 +26,17 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu selectedPropSyncChains := runtimeBuild.SelectedPropSyncChains ioFPGACount, ioFanoutFactor := runtimeBuild.IOFPGACount, runtimeBuild.IOFanoutFactor - // V3 topology and selected prop-sync chains are projected below. Do not feed - // them through the runtime-settings consumer: the runtime projection - // intentionally contains no artifact partitions. + + // Keep physical artifacts in the scheduler projection; the runtime uses resolved partition metadata. runtimeBuild.Partitions = nil runtimeBuild.SelectedPropSyncChains = nil - if intent.Pipeline != PipelineLPX { - if err := resolveBuildSettings(&runtimeBuild, modelSettings); err != nil { - return nil, fmt.Errorf("resolving configured V3 runtime build: %w", err) - } - } propSyncEnabled := runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly && len(selectedPropSyncChains) > 0 - if _, overridden := runtimeBuild.runtimeSettings["prop_sync"]; overridden { - var err error - propSyncEnabled, err = boolSetting(runtimeBuild.runtimeSettings, "prop_sync") - if err != nil { - return nil, fmt.Errorf("configured V3 runtime settings.%w", err) + if value, overridden := modelSettings["prop_sync"]; overridden { + enabled, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("model settings.prop_sync must be a boolean") } + propSyncEnabled = enabled } if runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly { if len(selectedPropSyncChains) > 0 && !propSyncEnabled { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index eef7c4f49e9d..388d2fcaa6eb 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -37,7 +37,6 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.NoError(t, err) projection := projectionBatch[0] require.Equal(t, BuildCompilationModeHybrid, projection.configuredBuild.CompilationMode) - require.Empty(t, projection.configuredBuild.runtimeSettings) digest, err := workloadSetDigest([]*ModelProjection{projection}) require.NoError(t, err) require.Equal(t, projection.Digest(), digest, "one model is the aggregate digest base case") diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index 1c0549d329ea..d07705479dab 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -257,35 +257,18 @@ vLLM and SGLang take tensor/pipeline/data parallelism as CLI flags (covered in t You normally do not need to set the top-level `spec.backendFramework` field — the operator infers the backend from the worker command. Set it explicitly (`vllm`, `sglang`, or `trtllm`) only when a feature needs the framework known up front, such as GMS failover or multinode TensorRT-LLM. -For LPX deployments, declare independent `agent` and `conductor` role templates. Every LPX graph requires exactly one explicit conductor template, including a graph with a single LPX component. This fragment shows the role structure only; the placeholder images use their own entrypoints: +For LPX, set `type: lpx`, `lpx.buildId`, and the required [agent and conductor roles](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference). For LP20 and LP30, role templates own images, commands or image entrypoints, runtime arguments and environment, init containers, probes, runtime storage, and security contexts. -```yaml -spec: - components: - - name: lpx - type: lpx - lpx: - buildId: my-model/build - roles: - - name: agent - podTemplate: - spec: - containers: - - name: main - image: - - name: conductor - podTemplate: - spec: - containers: - - name: main - image: -``` +Configure Nova in the conductor template: -For LP20 and LP30, supply compatible builds and images, static arguments and environment settings, init containers, health probes, volumes, and mounts in the role templates. The operator uses the image's entrypoint unless the template supplies `command`; it does not generate runtime launchers, SSH-key preparation, or probes. Supply the Pod and container security contexts required for device access in the role templates; the operator preserves these settings. +| Pipeline | Model environment bindings | Resolved-partition directory | +| --- | --- | --- | +| Single | `NOVA_MODEL_PATH=$(LPX_MODEL_PATH)` | `NOVA_RESOLVED_PARTITIONS_DIR` | +| `NOVA_PIPELINE_TYPE=SpecDecode` | `NOVA_DRAFT_MODEL_PATH=$(LPX_DRAFT_MODEL_PATH)`, `NOVA_TARGET_MODEL_PATH=$(LPX_TARGET_MODEL_PATH)` | `NOVA_DRAFT_RESOLVED_PARTITIONS_DIR`, `NOVA_TARGET_RESOLVED_PARTITIONS_DIR` | -The operator supplies generated configuration, model and hardware bindings, and the LPU conductor's `LPX_ALLOCATION` value. A Nova conductor must include `--allocation` and `$(LPX_ALLOCATION)` in its authored arguments, along with its other runtime options. Mount the generated `config` volume and model storage where the selected runtime expects them. SSH-based startup must declare its own Secret volume, key preparation, and mounts. +Dynamo resolves these model paths from each `buildId` and the model-storage mount; templates need no registry or cache-layout paths. Mount the generated `config` volume at `/configs` and set the resolved-partition variables above to `/configs`. Dynamo retains this ConfigMap for selected partition files; it generates no Nova model or datacenter TOML. Nova reads model defaults from the build manifests. Put runtime overrides, such as `NOVA_AGENT_CONNECT_TIMEOUT` and `NOVA_AGENT_SETUP_TIMEOUT`, in the template. -For LPU+LPU speculative decoding, declare separate draft and target LPX components: the draft has only an `agent` role; the target has `conductor` and `agent` roles. A hybrid engine places its GPU runtime in the conductor template. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. See [LPX component fields](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference) for replica counts and template requirements. +Include `--allocation` and `$(LPX_ALLOCATION)` in Nova's arguments to consume the operator's allocation. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. For per-backend setup and tuning, see [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/overview.md), [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/overview.md), and [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/overview.md). diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index e5e606341a96..cf4ea3a4d053 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2509,7 +2509,7 @@ _Appears in:_ -LPXConfig identifies the component's compiled model and runtime settings. +LPXConfig identifies the component's compiled model and placement settings. @@ -2520,7 +2520,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `buildId` _string_ | buildId references the immutable model build. | | MinLength: 1
| -| `settings` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | settings override the existing model configuration. | | Type: object
Optional: \{\}
| +| `settings` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | settings configure model placement. Runtime options belong in role pod templates. | | Type: object
Optional: \{\}
| #### MockerSpec diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 4f35a4175f50..11622062c48c 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -59,9 +59,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - DGD-only configuration for a `type: lpx` component; omitted from the standalone DCD schema. Set the required `buildId` and optional build `settings`. See [LPXConfig](full-api-reference.mdx#lpxconfig) for the full field set. - - Supported modes are LPU, hybrid, and LPU draft with LPU target. Every LPX graph requires exactly one explicit `conductor` role and template. A single LPX component declares both `conductor` and `agent` roles. In speculative decoding, the draft has only an `agent` role; the target declares both roles. Component names are arbitrary. All LPX components in a DGD share one LPX deployment and PodCliqueSet, separate from ordinary components. + DGD-only compiled-model configuration. Set `buildId`; see [LPXConfig](full-api-reference.mdx#lpxconfig) for the field definitions. LPU-only Single and SpecDecode accept only boolean placement `settings`: `prop_sync`, plus XT-only `cpu_embeddings` and `prop_sync_multiple_load_sets`. Runtime options belong in the conductor pod template. @@ -106,7 +104,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Native Pod template for this role. LPX-only; required for every declared role. Each template requires a `main` container with an explicit image. Agent and conductor templates are independent. - For LP20 and LP30, the template owns runtime commands, static arguments and environment settings, init containers, health probes, startup storage, and Pod and container security contexts. It may use the image's entrypoint or supply `command`. The operator supplies generated configuration, model and hardware bindings, and `LPX_ALLOCATION` for the LPU conductor. A Nova template consumes that value through its authored `--allocation` argument. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for role structure and runtime examples. + The template owns runtime startup and configuration for LP20 and LP30. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for Nova inputs and the operator's model-path, partition, and allocation bindings. core/v1.PodTemplateSpec diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 1d3a68a37fa3..d0ee1b70c02e 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3667,7 +3667,7 @@ preferredWeight is required and used only when enforcement is "preferred". Highe -LPXConfig identifies the component's compiled model and runtime settings. +LPXConfig identifies the component's compiled model and placement settings. **Kind:** `type` @@ -3679,7 +3679,7 @@ buildId references the immutable model build. -settings override the existing model configuration. +settings configure model placement. Runtime options belong in role pod templates. See [JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io). **Validation:** Type: object Optional: \{\} From 30ac06c36d463c1f2d9b8e458c77463b05ecf1f2 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:50:43 +0300 Subject: [PATCH 100/196] refactor(lpx): remove deployment settings (#538) Signed-off-by: Andrew Paprotsky (cherry picked from commit 154306eb9fe89fad1d265e6da6e7ecf911d05967) Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 2 - .../api/v1alpha1/lpx_conversion_test.go | 4 +- .../api/v1alpha1/zz_generated.deepcopy.go | 2 +- deploy/operator/api/v1beta1/lpx_types.go | 12 +- .../api/v1beta1/zz_generated.deepcopy.go | 8 +- .../nvidia.com_dynamographdeployments.yaml | 10 -- .../internal/controller/lpx/lifecycle_test.go | 6 +- .../internal/dynamo/graph_lpx_test.go | 3 - .../dynamo/lpx/compiler_fixture_test.go | 8 +- .../internal/dynamo/lpx/cyborg_render_test.go | 3 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 4 +- .../dynamo/lpx/materialization_test.go | 5 +- .../internal/dynamo/lpx/model_projection.go | 2 - .../internal/dynamo/lpx/render_test.go | 10 +- .../dynamo/lpx/resolve_selected_workload.go | 7 +- .../lpx/resolve_selected_workload_test.go | 5 - .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 8 +- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 8 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 8 +- .../node-local-v2-specdecode.yaml | 12 +- .../node-local-v3-hx-hybrid.yaml | 8 +- .../node-local-v3-hx-lpu-only.yaml | 8 +- .../node-local-v3-hx-specdecode.yaml | 12 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 8 +- .../dynamo/lpx/workload_projection.go | 64 +---------- .../dynamo/lpx/workload_projection_test.go | 104 +----------------- .../dynamo/lpx/workload_projection_v2.go | 53 ++------- .../dynamo/lpx/workload_projection_v2_test.go | 73 ++++-------- .../dynamo/lpx/workload_projection_v3.go | 18 +-- .../dynamo/lpx/workload_projection_v3_test.go | 14 +-- ...hdeployment_lpx_validation_envtest_test.go | 11 -- .../additional-resources/api-reference-k8s.md | 3 +- .../dynamo-component-deployment.mdx | 2 +- .../kubernetes-api/full-api-reference.mdx | 8 +- 34 files changed, 98 insertions(+), 415 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 13d6c673c577..784fed49b73a 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -62,7 +62,6 @@ DynamoComponentDeploymentSpec.experimental.grove.forceScalingGroup DynamoComponentDeploymentSpec.frontendSidecar DynamoComponentDeploymentSpec.globalDynamoNamespace DynamoComponentDeploymentSpec.lpx.buildId -DynamoComponentDeploymentSpec.lpx.settings DynamoComponentDeploymentSpec.minAvailable DynamoComponentDeploymentSpec.modelRef.name DynamoComponentDeploymentSpec.modelRef.revision @@ -136,7 +135,6 @@ DynamoGraphDeploymentSpec.components.experimental.grove.forceScalingGroup DynamoGraphDeploymentSpec.components.frontendSidecar DynamoGraphDeploymentSpec.components.globalDynamoNamespace DynamoGraphDeploymentSpec.components.lpx.buildId -DynamoGraphDeploymentSpec.components.lpx.settings DynamoGraphDeploymentSpec.components.minAvailable DynamoGraphDeploymentSpec.components.modelRef.name DynamoGraphDeploymentSpec.components.modelRef.revision diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index 5a4beaf83296..f09843f2c071 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -14,7 +14,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -23,8 +22,7 @@ const editedThroughAlphaBuildID = "model/edited-through-alpha" func testCanonicalLPXConfig() *v1beta1.LPXConfig { return &v1beta1.LPXConfig{ - BuildID: "model/build", - Settings: &apiextensionsv1.JSON{Raw: []byte(`{"scheduler":{"num_chips":4}}`)}, + BuildID: "model/build", } } diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 69688a5033ae..9de343728137 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -461,7 +461,7 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(v1beta1.LPXConfig) - (*in).DeepCopyInto(*out) + **out = **in } if in.Checkpoint != nil { in, out := &in.Checkpoint, &out.Checkpoint diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index c3b5d455017e..cb4878a09aac 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -3,21 +3,13 @@ package v1beta1 -import ( - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// LPXConfig identifies the component's compiled model and placement settings. +// LPXConfig identifies the component's compiled model. type LPXConfig struct { // buildId references the immutable model build. // +kubebuilder:validation:MinLength=1 BuildID string `json:"buildId"` - // settings configure model placement. Runtime options belong in role pod templates. - // +optional - // +kubebuilder:validation:Type=object - // +nullable - Settings *apiextensionsv1.JSON `json:"settings,omitempty"` } // SchedulingSpec configures LPX scheduling attempts. diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 0e47ef9d84cd..b936570795e3 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -40,7 +40,6 @@ package v1beta1 import ( batchv1 "k8s.io/api/batch/v1" "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1" @@ -370,7 +369,7 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(LPXConfig) - (*in).DeepCopyInto(*out) + **out = **in } if in.FrontendSidecar != nil { in, out := &in.FrontendSidecar, &out.FrontendSidecar @@ -1166,11 +1165,6 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXConfig) DeepCopyInto(out *LPXConfig) { *out = *in - if in.Settings != nil { - in, out := &in.Settings, &out.Settings - *out = new(apiextensionsv1.JSON) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXConfig. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index c7202de00fb1..9221e9dcc587 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11529,11 +11529,6 @@ spec: description: buildId references the immutable model build. minLength: 1 type: string - settings: - description: settings configure model placement. Runtime options belong in role pod templates. - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true required: - buildId type: object @@ -21072,11 +21067,6 @@ spec: description: buildId references the immutable model build. minLength: 1 type: string - settings: - description: settings configure model placement. Runtime options belong in role pod templates. - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true required: - buildId type: object diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index 8951e6ca15f8..cd60bce0fa41 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -37,7 +37,6 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -1751,10 +1750,7 @@ func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.D Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, Replicas: &one, - LPX: &nvidiacomv1beta1.LPXConfig{ - BuildID: buildID, - Settings: &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":true}`)}, - }, + LPX: &nvidiacomv1beta1.LPXConfig{BuildID: buildID}, Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, { Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 6d14566074c9..d5c71809b143 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -244,9 +244,6 @@ func TestLPXInputRevision(t *testing.T) { lpx.ServingComponent(d).TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} }}, {"component/build", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).LPX.BuildID = "next-build" }}, - {"component/settings", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"prop_sync":false}`)} - }}, {"agent/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(int32(4)) }}, diff --git a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go index 148ab4a5f7d8..fe17aef4fa15 100644 --- a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go @@ -7,7 +7,6 @@ package lpx import ( "context" - "encoding/json" "fmt" "os" "path/filepath" @@ -95,13 +94,12 @@ func normalizeTestSnapshot(t *testing.T, snapshot *BuildSnapshot) NormalizedBuil return normalized } -func projectTestBuild(t *testing.T, snapshot NormalizedBuildSnapshot, - pipeline Pipeline, settings string) *ModelProjection { +func projectTestBuild(t *testing.T, snapshot NormalizedBuildSnapshot, pipeline Pipeline) *ModelProjection { t.Helper() - t.Log("Project the caller-owned normalized build with the requested model settings") + t.Log("Project the caller-owned normalized build for the selected pipeline") projectionBatch, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: pipeline, Models: []string{"default"}, BuildSnapshot: snapshot, ModelSettings: json.RawMessage(settings), + Pipeline: pipeline, Models: []string{"default"}, BuildSnapshot: snapshot, }) require.NoError(t, err) return projectionBatch[0] diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index cbb5d53a5cfc..7bc8c6247570 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -10,7 +10,6 @@ import ( "testing" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -24,7 +23,7 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { t.Log("Project a hybrid engine with the longest PCS name and maximum scheduling replica count") fixture := newV3CompilerFixture() fixture.compilationMode = manifestcapnp.CompilationMode_lpx - projection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyHx16x8x2x3, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) + projection := projectRenderFixture(t, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) projection.stage = testRenderComponentName projection.configuredBuild.IOFPGACount = 1 projection.configuredBuild.IOFanoutFactor = 1 diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 7889925f89bd..60b575443116 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "strconv" "strings" "testing" @@ -28,7 +27,6 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { Models: []string{"default"}, RuntimeBuildRef: "model-build", BuildSnapshot: normalizeTestSnapshot(t, snapshot), - ModelSettings: json.RawMessage(`{"prop_sync":false}`), }) require.NoError(t, err) projection := projectionBatch[0] @@ -97,7 +95,7 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { fixture.partitions[index].id = uint32(index) } snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) - projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineLPX, `{"prop_sync":false}`) + projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineLPX) projection.stage = testRenderComponentName t.Log("Render only projected endpoints without compressing their physical Agent offsets") diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index 2ef60466846e..fd023f6242cb 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -12,7 +12,6 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/validation" ) @@ -24,8 +23,8 @@ func TestPlanMaterializationBounds(t *testing.T) { snapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) hybrid := newV2CompilerFixture() hybrid.compilationMode = manifestcapnp.CompilationMode_lpx - hybridProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, hybrid))) - lpuOnlyProjection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) + hybridProjection := projectRenderFixture(t, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, hybrid))) + lpuOnlyProjection := projectRenderFixture(t, PipelineSingle, snapshot) t.Log("Reserve readable roles at the maximum PCS length and scheduling replica count") for _, test := range []struct { diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index 6a6a7dc3d32d..41ead7f7985d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -47,8 +47,6 @@ type ModelProjectionInput struct { RuntimeBuildRef string // BuildSnapshot is the normalized immutable build input. BuildSnapshot NormalizedBuildSnapshot - // ModelSettings contains the optional per-model JSON settings. - ModelSettings json.RawMessage } // ModelProjection holds scheduler request inputs and runtime rendering state diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index afd298596697..5bf31fced31c 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "strings" "testing" @@ -81,7 +80,7 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Log("Seed conflicting runtime annotations and a nondefault mount") - projection := projectRenderFixture(t, test.family, test.pipeline, test.snapshot) + projection := projectRenderFixture(t, test.pipeline, test.snapshot) require.NotEqual(t, projection.Digest().String(), projection.CompilerSnapshotDigest()) pcs := renderTestPCS(test.pipeline == PipelineLPX) pcs.Annotations = map[string]string{ @@ -157,7 +156,7 @@ func TestRenderMaterializesAgentModelFromBasePodSpec(t *testing.T) { t.Log("Project an LPU-only model with template-owned runtime settings") snapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) - projection := projectRenderFixture(t, lpxv1alpha1.TargetFamilyXt8888, PipelineSingle, snapshot) + projection := projectRenderFixture(t, PipelineSingle, snapshot) t.Log("Construct a base PodSpec with model binding and custom placement") modelAnnotationSource := &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ @@ -315,15 +314,12 @@ func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { } } -func projectRenderFixture(t *testing.T, family lpxv1alpha1.TargetFamily, pipeline Pipeline, snapshot *BuildSnapshot) *ModelProjection { +func projectRenderFixture(t *testing.T, pipeline Pipeline, snapshot *BuildSnapshot) *ModelProjection { t.Helper() intent := ModelProjectionInput{ Pipeline: pipeline, Models: []string{"default"}, RuntimeBuildRef: "model-build", BuildSnapshot: normalizeTestSnapshot(t, snapshot), } - if family == lpxv1alpha1.TargetFamilyXt8888 { - intent.ModelSettings = json.RawMessage(`{"prop_sync":true}`) - } projectionBatch, err := appendModelProjections(nil, intent) require.NoError(t, err) return projectionBatch[0] diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index d2b8fef5f1c5..a9e68fde5ca0 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -7,7 +7,6 @@ package lpx import ( "context" - "encoding/json" "errors" "fmt" "slices" @@ -50,10 +49,7 @@ func ResolveSelectedWorkload( for index, stage := range components { model := stage.LPX configuredModel := stage.ComponentName - var modelSettings json.RawMessage - if model.Settings != nil { - modelSettings = model.Settings.Raw - } + // A validated selection has at most two models, so only the preceding projection can share its build. if len(projections) == 0 || projections[len(projections)-1].runtimeBuildRef != model.BuildID { rawSnapshot, acquireErr := source.AcquireBuildSnapshot(ctx, model.BuildID) @@ -99,7 +95,6 @@ func ResolveSelectedWorkload( Models: modelNames, RuntimeBuildRef: model.BuildID, BuildSnapshot: snapshot, - ModelSettings: modelSettings, } projected, err := appendModelProjections(projections, intent) if err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index b64b64f54330..fef9980997c1 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -15,7 +15,6 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" @@ -170,14 +169,12 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Acquire the selected family fixtures: shared XT build or distinct HX snapshots") var draftSnapshot, targetSnapshot *BuildSnapshot - var settings *apiextensionsv1.JSON if test.family == BuildFamilyHX { draftSnapshot = acquireTestSnapshot(t, writeV3CompilerFixture(t)) targetSnapshot = acquireTestSnapshot(t, writeV3CompilerFixture(t)) } else { draftSnapshot = acquireTestSnapshot(t, writeV2CompilerFixture(t)) targetSnapshot = draftSnapshot - settings = &apiextensionsv1.JSON{Raw: []byte(`{ "prop_sync": true }`)} } t.Log("Build a selected SpecDecode DGD for the fixture's manifest generation") @@ -185,8 +182,6 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { testLPXComponent("lpx", "target-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), testLPXComponent("small", "draft-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}), ) - dgd.Spec.Components[0].LPX.Settings = settings - dgd.Spec.Components[1].LPX.Settings = settings.DeepCopy() dgd.Spec.Components[1].Replicas = ptr.To(int32(2)) compiledAgentCount := int32(1) if test.family == BuildFamilyXT { diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 2efc32d27460..652bb87a79b5 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -175,7 +175,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm @@ -192,7 +192,7 @@ spec: nvidia.com/extra-resources-hash: 5448f938d5d52c694963453bff4e63367aa296f1c0aa36ea0960da60cad289b6 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid @@ -403,7 +403,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -586,7 +586,7 @@ spec: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:12f2d0ae40fa84798f025eafc6bf5ecbe38cdc7adb797e3e41726b4c8c916114 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 6602623fbf0c..ea5d6e12f34d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -39,7 +39,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid @@ -56,7 +56,7 @@ spec: nvidia.com/extra-resources-hash: a2baf441cd41d87e6ae2b9457c6134d41e06134d1091efe63bb4c4a01a17ef5f scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid @@ -218,7 +218,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -375,7 +375,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15dd04d9f3ce9e1a58d776d35892e4fad5c91109b1bf86c386fb00b010273db8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 8e6249a0c897..399e0cf48437 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -33,7 +33,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only @@ -50,7 +50,7 @@ spec: nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -185,7 +185,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -361,7 +361,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9818bb7546b87558d8e18acb0b27ad76009123f72fa25b89398759d6ee75ea18 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index a8a0a31c2b06..5c264153ac1d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -67,7 +67,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -84,7 +84,7 @@ spec: nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a973a2e0a92065344cab16b5b6af504f9403288bc2b332785293de5f036bc404 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b0943511c591eb7201003374eacf778c4545a959d7be251547399296e35ee1ae scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -224,7 +224,7 @@ spec: nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3061125e8a82194e7fadab40ddf2334081340728a740f3bd3d1231f72c061e1b + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d19c12c440463a3ebbc8c86f3fe882a33e7d5f48de493146aaf6d0acfa4fdba7 scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -364,7 +364,7 @@ spec: nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:55b7f43000a720e9afec381ae28b43d4a1436b5b8a2b666c4706d692411d218e + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b06e7ae90a96bd569bf228eb8b1d4b1192c4b0df3af7e33202d1fc1965a5247e scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -503,7 +503,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -688,7 +688,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:75b46c16650574198e849b16926c63faebce97ca1ddffdfd4c7a54e0d7b1db81 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 cliqueNames: - cond - agt0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index ab683ac58a91..0e117c200329 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -26,7 +26,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid @@ -43,7 +43,7 @@ spec: nvidia.com/extra-resources-hash: dc7a6b2e2078e8215ec068db5b51870b05382a0252a8e8c788c34b1173fe44ca scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxStrictHybrid @@ -204,7 +204,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -343,7 +343,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8fdccd781b4850f821d63b39d26c3a176fc1a06c246b350af31af22c7542bd64 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index c11171bfd2b0..034ddf98088e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -25,7 +25,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only @@ -42,7 +42,7 @@ spec: nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -207,7 +207,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -422,7 +422,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ae1c84203872d61daec4df7a2ec2e2e598454e41907d5fc9d6d0bd0887df0dc6 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 3201171c3bb0..8b3b53d25e6d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -46,7 +46,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -63,7 +63,7 @@ spec: nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:032d5479203438990d88c1acb177cde66ba776b7cf3ba2037460a1e5fd9ec4ea + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6b42344f7916f4778ffdef73249a7395bbc7a082466cc457953af5de5544c80f scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -233,7 +233,7 @@ spec: nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ac78cc79aa830484cc606f6bc1f81c9d2e7b3a42033bda25fb7ed3b012354bed + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b94a35c4a019e3213f4497e105522208d2a2707e8469ec13c22d46b99d53c227 scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -403,7 +403,7 @@ spec: nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:bbf856a3258227e3e1c5bfdf3c6929648c06853165e30be35de3cde7820ee06a + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8440780b5cdc30ae4bbcf80fda3cce06027325884a8b69751577112153ab2cf5 scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -572,7 +572,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -797,7 +797,7 @@ spec: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:13d09d8a4fee0861ed745cf2edb1c3e82d8aa0ffecb7a606270c2e311220bef7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 cliqueNames: - cond - agt0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 9fa6e9f4a93a..650b368ae969 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -108,7 +108,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 @@ -125,7 +125,7 @@ spec: nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -264,7 +264,7 @@ spec: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -459,7 +459,7 @@ spec: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:9c481e218d40e2bf27d1b3545862c651436e518d165a23329b5c0b194e72ce4d + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection.go b/deploy/operator/internal/dynamo/lpx/workload_projection.go index 209973c8726a..90e9b4b23f58 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection.go @@ -6,14 +6,10 @@ package lpx import ( - "bytes" "crypto/sha256" "encoding/binary" - "encoding/json" "fmt" "hash" - "maps" - "slices" ) const ( @@ -27,46 +23,16 @@ const ( // appendModelProjections appends one component's immutable model projections to // the caller-owned destination, which may be nil. Existing elements are unchanged. // intent.BuildSnapshot contains a normalized, non-nil build; Models is nonempty -// and Pipeline is selected by the validated resolver. ModelSettings is API-validated -// JSON: omitted, null or an object. Source inputs are not mutated; discard error results. +// and Pipeline is selected by the validated resolver. Source inputs are not mutated; +// discard error results. func appendModelProjections(dst []*ModelProjection, intent ModelProjectionInput) ([]*ModelProjection, error) { - const propSyncSetting = "prop_sync" - - modelSettings, modelSettingsObject := canonicalModelSettings(intent.ModelSettings) - intent.ModelSettings = modelSettings - - // Hybrid runtimes read their manifests directly; only XT scheduler controls are consumed here. - if intent.Pipeline == PipelineLPX { - for key := range modelSettingsObject { - if intent.BuildSnapshot.build.Family != BuildFamilyXT || - (key != propSyncSetting && key != "prop_sync_multiple_load_sets") { - return nil, fmt.Errorf("hybrid %s workloads do not support runtime settings overrides", intent.BuildSnapshot.build.Family) - } - } - } else { - // Conductor templates own runtime configuration; model settings only affect placement. - for _, key := range slices.Sorted(maps.Keys(modelSettingsObject)) { - switch key { - case "cpu_embeddings", propSyncSetting, "prop_sync_multiple_load_sets": - if _, ok := modelSettingsObject[key].(bool); !ok { - return nil, fmt.Errorf("model settings.%s must be a boolean", key) - } - if intent.BuildSnapshot.build.Family == BuildFamilyHX && key != propSyncSetting { - return nil, fmt.Errorf("model settings.%s is not used for HX placement; configure runtime options in the conductor podTemplate", key) - } - default: - return nil, fmt.Errorf("model settings.%s is a runtime setting; configure it in the conductor podTemplate", key) - } - } - } - var projections []*ModelProjection var err error switch intent.BuildSnapshot.build.Family { case BuildFamilyXT: - projections, err = appendV2ModelProjections(dst, intent, modelSettingsObject) + projections, err = appendV2ModelProjections(dst, intent) case BuildFamilyHX: - projections, err = appendV3ModelProjections(dst, intent, modelSettingsObject) + projections, err = appendV3ModelProjections(dst, intent) default: return nil, fmt.Errorf("unsupported LPX target family %q", intent.BuildSnapshot.build.Family) } @@ -104,27 +70,6 @@ func workloadSetDigest(projections []*ModelProjection) (WorkloadDigest, error) { return transcript.sum(), nil } -// canonicalModelSettings canonicalizes an API-validated optional JSON object. -func canonicalModelSettings(raw json.RawMessage) (json.RawMessage, map[string]any) { - trimmed := bytes.TrimSpace(raw) - if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { - return nil, make(map[string]any) - } - - // Decode the admitted object while preserving number spellings for identity. - decoder := json.NewDecoder(bytes.NewReader(trimmed)) - decoder.UseNumber() - var object map[string]any - _ = decoder.Decode(&object) - - // Decoder-owned values encode infallibly in memory; drop Encoder's documented trailing newline. - var output bytes.Buffer - encoder := json.NewEncoder(&output) - encoder.SetEscapeHTML(false) - _ = encoder.Encode(object) - return output.Bytes()[:output.Len()-1], object -} - func newModelProjectionTranscripts(intent ModelProjectionInput, projectionVersion string) []digestTranscript { // Each logical model owns a fresh hash while sharing the component's canonical inputs. transcripts := make([]digestTranscript, len(intent.Models)) @@ -155,7 +100,6 @@ func newModelProjectionTranscripts(intent ModelProjectionInput, projectionVersio transcript.field("workload-mode", []byte(mode)) transcript.field("model", []byte(model)) - transcript.field("model-settings", intent.ModelSettings) } return transcripts } diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go index 2f67048a2a21..ddf34e6c1502 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_test.go @@ -6,52 +6,12 @@ package lpx import ( - "encoding/json" "testing" - manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" ) -func TestHybridSettingsRejectRuntimeOverrides(t *testing.T) { - for _, fixture := range []testV3CapnpFixture{newV2CompilerFixture(), newV3CompilerFixture()} { - t.Log("Acquire a hybrid build for each supported family") - fixture.compilationMode = manifestcapnp.CompilationMode_lpx - snapshot := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) - - t.Log("Allow empty settings and XT scheduler controls, but not runtime overrides") - for _, test := range []struct { - settings string - xtOnly, reject bool - }{ - {settings: ""}, - {settings: "null"}, - {settings: "{}"}, - {settings: `{"prop_sync":true}`, xtOnly: true}, - {settings: `{"prop_sync_multiple_load_sets":true}`, xtOnly: true}, - {settings: `{"batch_size":4}`, reject: true}, - {settings: `{"setup":{"agent_setup_timeout":"600s"}}`, reject: true}, - {settings: `{"cpu_embeddings":true}`, reject: true}, - {settings: `{"batch_folding":false,"num_batch_split_divisions":1}`, reject: true}, - {settings: `{"prop_sync":false,"batch_size":4}`, reject: true}, - } { - t.Run(string(snapshot.build.Family)+"/"+test.settings, func(t *testing.T) { - t.Log("Reject unsupported settings before producing a workload projection") - _, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineLPX, Models: []string{"default"}, BuildSnapshot: snapshot, - ModelSettings: json.RawMessage(test.settings), - }) - if test.reject || (test.xtOnly && snapshot.build.Family != BuildFamilyXT) { - require.ErrorContains(t, err, "do not support runtime settings overrides") - } else { - require.NoError(t, err) - } - }) - } - } -} - func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { t.Parallel() @@ -62,8 +22,8 @@ func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { require.Equal(t, first.contentID, second.contentID) t.Log("Project either immutable snapshot through the same intent") - firstProjection := projectTestBuild(t, first, PipelineSingle, `{"prop_sync":true}`) - secondProjection := projectTestBuild(t, second, PipelineSingle, `{"prop_sync":true}`) + firstProjection := projectTestBuild(t, first, PipelineSingle) + secondProjection := projectTestBuild(t, second, PipelineSingle) t.Log("Publish the canonical compiler snapshot identity independently of build locator") require.Equal(t, "scheduling.lpu.nvidia.com/compiler-snapshot-digest", lpxv1alpha1.CompilerSnapshotDigestAnnotation) @@ -74,65 +34,7 @@ func TestWorkloadDigestIsIndependentOfBuildLocator(t *testing.T) { require.Equal(t, firstProjection.Digest(), secondProjection.Digest()) t.Log("Keep compiler identity separate from downstream workload projection identity") - specDecodeProjection := projectTestBuild(t, first, PipelineSpecDecode, `{"prop_sync":true}`) + specDecodeProjection := projectTestBuild(t, first, PipelineSpecDecode) require.Equal(t, firstProjection.CompilerSnapshotDigest(), specDecodeProjection.CompilerSnapshotDigest()) require.NotEqual(t, firstProjection.Digest(), specDecodeProjection.Digest()) } - -func TestCanonicalModelSettingsRetainRaw(t *testing.T) { - t.Log("Decode canonical model settings while retaining their exact raw bytes") - raw, object := canonicalModelSettings(json.RawMessage( - `{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`, - )) - require.Equal(t, json.RawMessage(`{"exponent":1e3,"float":1.0,"integer":8192,"nested":{"array":[128001,1.5,9223372036854775808]}}`), raw) - require.Equal(t, json.Number("8192"), object["integer"]) - - t.Log("Normalize explicit null settings to a present empty object") - raw, object = canonicalModelSettings(json.RawMessage("null")) - require.Nil(t, raw) - require.NotNil(t, object) - require.Empty(t, object) -} - -func TestProjectModelSettingsPlacementBoundary(t *testing.T) { - t.Log("Prepare both physical families with conductor-owned runtime settings") - builds := []NormalizedBuildSnapshot{ - normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))), - normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV3CompilerFixture(t))), - } - for _, build := range builds { - for _, pipeline := range []Pipeline{PipelineSingle, PipelineSpecDecode} { - for _, test := range []struct { - settings, wantErr string - xtOnly bool - }{ - {settings: `{"sequence_length":4096}`, wantErr: "model settings.sequence_length is a runtime setting; configure it in the conductor podTemplate"}, - {settings: `{"setup":{"agent_setup_timeout":"30s"}}`, wantErr: "model settings.setup is a runtime setting"}, - {settings: `{"z":null,"a":true}`, wantErr: "model settings.a is a runtime setting"}, - {settings: `{"cpu_embeddings":"false"}`, wantErr: "model settings.cpu_embeddings must be a boolean"}, - {settings: `{"prop_sync":null}`, wantErr: "model settings.prop_sync must be a boolean"}, - {settings: `{"prop_sync_multiple_load_sets":1}`, wantErr: "model settings.prop_sync_multiple_load_sets must be a boolean"}, - {settings: `{"cpu_embeddings":true}`, xtOnly: true}, - {settings: `{"cpu_embeddings":false}`, xtOnly: true}, - {settings: `{"prop_sync_multiple_load_sets":true}`, xtOnly: true}, - {settings: `{"prop_sync_multiple_load_sets":false}`, xtOnly: true}, - {settings: `{"prop_sync":false}`}, - } { - t.Run(string(build.build.Family)+"/"+string(pipeline)+"/"+test.settings, func(t *testing.T) { - t.Log("Accept only placement controls consumed by the selected physical family") - _, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: pipeline, Models: []string{"default"}, BuildSnapshot: build, - ModelSettings: json.RawMessage(test.settings), - }) - if test.xtOnly && build.build.Family == BuildFamilyHX { - require.ErrorContains(t, err, "is not used for HX placement; configure runtime options in the conductor podTemplate") - } else if test.wantErr != "" { - require.ErrorContains(t, err, test.wantErr) - } else { - require.NoError(t, err) - } - }) - } - } - } -} diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go index d5b616e4c3c3..8712576753a7 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2.go @@ -14,54 +14,28 @@ import ( lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" ) -const ( - v2ProjectionVersion = "v2-xt-node-local/v1" - maxPropSyncMultipleLoadSetsOffset = int64(3) -) +const v2ProjectionVersion = "v2-xt-node-local/v1" //nolint:gocyclo // V2 projection validates one complete transformation. -func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInput, modelSettings map[string]any) ([]*ModelProjection, error) { +func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInput) ([]*ModelProjection, error) { configured := *intent.BuildSnapshot.build usesResolvedRuntime := intent.Pipeline == PipelineSingle || intent.Pipeline == PipelineSpecDecode - build := &configured - ioFPGACount, ioFanoutFactor := build.IOFPGACount, build.IOFanoutFactor - propSyncEnabled := build.CompilationMode == BuildCompilationModeLPUOnly && len(build.SelectedPropSyncChains) > 0 + ioFPGACount, ioFanoutFactor := configured.IOFPGACount, configured.IOFanoutFactor connectorBuild := intent.BuildSnapshot.build // Apply the selected chain and CPU embedding placement before deriving scheduler requests. if usesResolvedRuntime { - cpuEmbeddings := configured.SupportsCPUEmbeddings - if value, overridden := modelSettings["cpu_embeddings"]; overridden { - cpuEmbeddings = value.(bool) - } if err := configured.consumeRuntimeSelectedPropSyncChain(); err != nil { return nil, fmt.Errorf("resolving configured V2 build: %w", err) } // Omit host-only embeddings by retaining a view of the immutable source partitions. - if cpuEmbeddings && configured.SupportsCPUEmbeddings && configured.StandaloneTokenEmbeddings && + if configured.SupportsCPUEmbeddings && configured.StandaloneTokenEmbeddings && len(configured.Partitions) > 1 && configured.Partitions[0].SourcePartitionID == 0 { configured.Partitions = configured.Partitions[1:] } } - if value, present := modelSettings["prop_sync"]; present { - enabled, ok := value.(bool) - if !ok { - return nil, fmt.Errorf("model settings.prop_sync must be a boolean") - } - propSyncEnabled = enabled - } - maxInterPartitionOffset := int64(0) - if value, present := modelSettings["prop_sync_multiple_load_sets"]; present { - multiple, ok := value.(bool) - if !ok { - return nil, fmt.Errorf("model settings.prop_sync_multiple_load_sets must be a boolean") - } - if multiple { - maxInterPartitionOffset = maxPropSyncMultipleLoadSetsOffset - } - } allocationMetadata := json.RawMessage(`{}`) @@ -70,12 +44,6 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu for index := range transcripts { transcript := &transcripts[index] transcript.field("input-embeddings-on-gpu", []byte{1}) - if propSyncEnabled { - transcript.field("effective-prop-sync", []byte{1}) - } else { - transcript.field("effective-prop-sync", []byte{0}) - } - transcript.intField("max-inter-partition-offset", maxInterPartitionOffset) bindHybridRuntimeIO(transcript, intent.Pipeline, ioFPGACount, ioFanoutFactor) } @@ -96,7 +64,7 @@ func appendV2ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu transcripts[modelIndex].field("xt-shape", []byte(shape)) } } - connectors, err := v2Connectors(connectorBuild, partitions, propSyncEnabled, maxInterPartitionOffset) + connectors, err := v2Connectors(connectorBuild, partitions) if err != nil { return nil, err } @@ -177,15 +145,10 @@ func xtShape(chipCount int) (lpxv1alpha1.Xt8888PartitionShape, int64, error) { func v2Connectors( build *Build, partitions []BuildPartition, - propSyncEnabled bool, - maxOffset int64, ) ([]lpxv1alpha1.PropSyncConnectorRequest, error) { - // Require compiler-authored relationships whenever prop-sync is enabled. + // Only compiler-selected relationships impose placement constraints. if len(build.SelectedPropSyncChains) == 0 { - if !propSyncEnabled { - return []lpxv1alpha1.PropSyncConnectorRequest{}, nil - } - return nil, fmt.Errorf("manifest v2 settings.prop_sync=true requires a selected prop-sync chain") + return []lpxv1alpha1.PropSyncConnectorRequest{}, nil } // Validate explicit chains before ordering their scheduler edges. @@ -215,7 +178,7 @@ func v2Connectors( if i+1 >= len(partitions) { break } - offset := maxOffset + offset := int64(0) connectors = append(connectors, lpxv1alpha1.PropSyncConnectorRequest{ FromPartitionID: fmt.Sprintf("partition-%03d", i), ToPartitionID: fmt.Sprintf("partition-%03d", i+1), diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go index 43d5db3fa9fd..6c973c8b881c 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v2_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "fmt" "math" "testing" @@ -95,7 +94,7 @@ func TestXTShape(t *testing.T) { } } -func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { +func TestProjectModelV2UsesManifestPropSync(t *testing.T) { t.Parallel() t.Log("Acquire a V2 build with compiler-selected prop-sync metadata") @@ -103,56 +102,31 @@ func TestProjectModelV2UsesEffectivePropSyncSettings(t *testing.T) { build := normalized.build require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) - t.Log("Preserve both physical partitions and their selected connector without model settings") - defaultedBatch, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalized, - }) - require.NoError(t, err) - defaulted := defaultedBatch[0] - defaultSpec := defaulted.RequestSpec(&MaterializationPlan{}, "agents") - require.Len(t, defaultSpec.Partitions, 2) - require.Len(t, defaultSpec.PropSyncConnectors, 1) - require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) - - t.Log("Enable prop sync and project its effective connector offset") - projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":true,"prop_sync_multiple_load_sets":true}`) + t.Log("Preserve both physical partitions with their adjacent selected connector") + projection := projectTestBuild(t, normalized, PipelineSingle) spec := projection.RequestSpec(&MaterializationPlan{}, "agents") + require.Len(t, spec.Partitions, 2) require.Len(t, spec.PropSyncConnectors, 1) - require.Equal(t, int64(3), *spec.PropSyncConnectors[0].Requirement.MaxInterPartitionOffset) - - t.Log("Preserve the compiler-selected chain when model settings disable implicit prop sync") - intent := ModelProjectionInput{ - Pipeline: PipelineSingle, - Models: []string{"default"}, - BuildSnapshot: normalized, - ModelSettings: json.RawMessage(`{"prop_sync":false}`), - } - disabledBatch, err := appendModelProjections(nil, intent) - require.NoError(t, err) - disabled := disabledBatch[0] - require.Len(t, disabled.RequestSpec(&MaterializationPlan{}, "agents").PropSyncConnectors, 1) + require.Equal(t, int64(0), *spec.PropSyncConnectors[0].Requirement.MaxInterPartitionOffset) + require.Equal(t, [][]int{{7, 8}}, build.SelectedPropSyncChains) - t.Log("Require manifest-selected evidence when revision 2 enables prop sync") + t.Log("Keep all physical partitions without synthesizing a chain absent from the manifest") build.SelectedPropSyncChains = nil - _, err = appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineSingle, Models: []string{"default"}, - BuildSnapshot: normalized, - ModelSettings: json.RawMessage(`{"prop_sync":true}`), - }) - require.ErrorContains(t, err, "manifest v2 settings.prop_sync=true requires a selected prop-sync chain") + projection = projectTestBuild(t, normalized, PipelineSingle) + spec = projection.RequestSpec(&MaterializationPlan{}, "agents") + require.Len(t, spec.Partitions, 2) + require.Empty(t, spec.PropSyncConnectors) + require.Equal(t, 4, projection.agentReplicas) } -func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { +func TestProjectModelV2SingleEmbeddingPlacementFromManifest(t *testing.T) { for _, test := range []struct { - name, settings string + name string cpuSupported, standalone bool firstPartition int64 }{ {name: "default CPU embeddings", cpuSupported: true, standalone: true, firstPartition: 1}, - {name: "explicit CPU embeddings", settings: `{"cpu_embeddings":true}`, cpuSupported: true, standalone: true, firstPartition: 1}, - {name: "CPU embeddings disabled", settings: `{"cpu_embeddings":false}`, cpuSupported: true, standalone: true}, {name: "CPU embeddings unsupported", standalone: true}, - {name: "CPU embeddings requested but unsupported", settings: `{"cpu_embeddings":true}`, standalone: true}, {name: "partition zero is runnable", cpuSupported: true}, } { t.Run(test.name, func(t *testing.T) { @@ -166,7 +140,7 @@ func TestProjectModelV2SingleEmbeddingPlacementFromModelSettings(t *testing.T) { build.SelectedPropSyncChains = [][]int{{0, 1}} t.Log("Project retained source partitions into placement and runtime metadata") - projection := projectTestBuild(t, normalized, PipelineSingle, test.settings) + projection := projectTestBuild(t, normalized, PipelineSingle) spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Equal(t, test.firstPartition, spec.Partitions[0].CompilerPartitionID) require.EqualValues(t, 2*(2-test.firstPartition), projection.agentReplicas) @@ -195,7 +169,7 @@ func TestProjectModelV2StrictHybridPreservesPartitionZero(t *testing.T) { build.SelectedPropSyncChains = nil t.Log("Keep Cyborg's partition zero independently of Nova's embedding placement") - projection := projectTestBuild(t, normalized, PipelineLPX, "") + projection := projectTestBuild(t, normalized, PipelineLPX) spec := projection.RequestSpec(&MaterializationPlan{}, "agents") require.Equal(t, lpxv1alpha1.WorkloadModeV2StrictHybrid, spec.WorkloadMode) require.Len(t, spec.Partitions, 2) @@ -218,8 +192,8 @@ func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { build.Partitions = append([]BuildPartition{prefix}, append(build.Partitions, suffix)...) build.SelectedPropSyncChains = [][]int{{7, 8}} - t.Log("Project with global prop sync disabled") - projection := projectTestBuild(t, normalized, PipelineSingle, `{"prop_sync":false}`) + t.Log("Project the compiler-selected chain") + projection := projectTestBuild(t, normalized, PipelineSingle) t.Log("Preserve only the source-selected connector and partition identities") spec := projection.RequestSpec(&MaterializationPlan{}, "agents") @@ -236,12 +210,11 @@ func TestProjectModelV2UsesOnlyTheSourceSelectedAdjacentChain(t *testing.T) { build.Partitions[2].Topology = topology _, err = appendModelProjections(nil, ModelProjectionInput{ Pipeline: PipelineSingle, Models: []string{"default"}, BuildSnapshot: normalized, - ModelSettings: json.RawMessage(`{"prop_sync":false}`), }) require.ErrorContains(t, err, "has incompatible topology at partition ID 8") } -func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing.T) { +func TestProjectModelV2SeparatesPhysicalPartitionsFromRuntimeChain(t *testing.T) { t.Parallel() normalized := normalizeTestSnapshot(t, acquireTestSnapshot(t, writeV2CompilerFixture(t))) build := normalized.build @@ -261,7 +234,7 @@ func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing. build.SelectedPropSyncChains = [][]int{{7, 8}} t.Log("Project the selected chain beside an independent third partition") - projection := projectTestBuild(t, normalized, PipelineLPX, "") + projection := projectTestBuild(t, normalized, PipelineLPX) t.Log("Project all physical scheduler partitions and only the selected connector") spec := projection.RequestSpec(&MaterializationPlan{}, "agents") @@ -287,7 +260,7 @@ func TestProjectModelV2PrioritizesSelectedChainWhenPropSyncIsEnabled(t *testing. fourth.SourcePartitionID = 13 build.Partitions = append(build.Partitions, fourth) build.SelectedPropSyncChains = [][]int{{11, 13}, {7, 8}} - connectors, err := v2Connectors(build, build.Partitions, true, maxPropSyncMultipleLoadSetsOffset) + connectors, err := v2Connectors(build, build.Partitions) require.NoError(t, err) require.Len(t, connectors, 2) require.Equal(t, "partition-000", connectors[0].FromPartitionID) @@ -319,7 +292,7 @@ func TestProjectModelV2CollapsesSelectedChainIncludingPartitionZero(t *testing.T build.SupportsCPUEmbeddings = true t.Log("Project without applying Nova's CPU-embedding partition omission") - projection := projectTestBuild(t, normalized, PipelineLPX, "") + projection := projectTestBuild(t, normalized, PipelineLPX) t.Log("Keep all physical scheduler partitions while collapsing their Agent runtime projection") spec := projection.RequestSpec(&MaterializationPlan{}, "agents") @@ -354,7 +327,7 @@ func TestProjectModelV2PreservesAgentReplicasWhenCollapsingSubHostPartitions(t * build.SelectedPropSyncChains = [][]int{{7, 8}} t.Log("Project and collapse the selected runtime chain") - projection := projectTestBuild(t, normalized, PipelineLPX, "") + projection := projectTestBuild(t, normalized, PipelineLPX) t.Log("Preserve both physical scheduler partitions and Agent replicas") require.Len(t, projection.RequestSpec(&MaterializationPlan{}, "agents").Partitions, 2) diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 5f370f40323f..49d1384ce9fa 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -20,7 +20,7 @@ const ( v3HXLogicalDeviceCount = 16 ) -func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInput, modelSettings map[string]any) ([]*ModelProjection, error) { +func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInput) ([]*ModelProjection, error) { runtimeBuild := *intent.BuildSnapshot.build manifestPartitions := runtimeBuild.Partitions selectedPropSyncChains := runtimeBuild.SelectedPropSyncChains @@ -30,22 +30,6 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu // Keep physical artifacts in the scheduler projection; the runtime uses resolved partition metadata. runtimeBuild.Partitions = nil runtimeBuild.SelectedPropSyncChains = nil - propSyncEnabled := runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly && len(selectedPropSyncChains) > 0 - if value, overridden := modelSettings["prop_sync"]; overridden { - enabled, ok := value.(bool) - if !ok { - return nil, fmt.Errorf("model settings.prop_sync must be a boolean") - } - propSyncEnabled = enabled - } - if runtimeBuild.CompilationMode == BuildCompilationModeLPUOnly { - if len(selectedPropSyncChains) > 0 && !propSyncEnabled { - return nil, fmt.Errorf("V3 manifest-selected prop-sync chains require settings.prop_sync=true") - } - if len(selectedPropSyncChains) == 0 && propSyncEnabled { - return nil, fmt.Errorf("V3 settings.prop_sync=true requires a manifest-selected prop-sync chain") - } - } allocationMetadata, connectors, err := projectV3PropSync(manifestPartitions, selectedPropSyncChains, intent.Pipeline) if err != nil { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index 388d2fcaa6eb..194d06e1ed78 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -6,7 +6,6 @@ package lpx import ( - "encoding/json" "testing" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" @@ -206,17 +205,11 @@ func TestProjectModelV3ProjectsSelectedPropSyncChain(t *testing.T) { }]} }`, string(spec.AllocationMetadata.Raw)) - t.Log("Reject model settings that disable a manifest-selected chain") - intent.ModelSettings = json.RawMessage(`{"prop_sync":false}`) - _, err = appendModelProjections(nil, intent) - require.ErrorContains(t, err, "manifest-selected prop-sync chains require settings.prop_sync=true") - - t.Log("Reject global prop sync without manifest-selected evidence in an LPU-only build") + t.Log("Reject a multi-partition LPU-only build without its complete manifest-selected chain") fixture.selectedPropSyncChains = nil intent.BuildSnapshot = normalizeTestSnapshot(t, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) - intent.ModelSettings = json.RawMessage(`{"prop_sync":true}`) _, err = appendModelProjections(nil, intent) - require.ErrorContains(t, err, "settings.prop_sync=true requires a manifest-selected prop-sync chain") + require.ErrorContains(t, err, "LPU-only workloads require a complete adjacent prop-sync connector chain") t.Log("Extend the native selected chain across three HX artifacts with distinct opaque topology names") third := fixture.partitions[0] @@ -226,7 +219,6 @@ func TestProjectModelV3ProjectsSelectedPropSyncChain(t *testing.T) { fixture.selectedPropSyncChains = [][]uint32{{1, 2, 3}} writeTestV3CapnpManifest(t, buildDir, fixture) intent.BuildSnapshot = normalizeTestSnapshot(t, acquireTestSnapshot(t, buildDir)) - intent.ModelSettings = nil projectionBatch, err = appendModelProjections(nil, intent) require.NoError(t, err) projection = projectionBatch[0] @@ -256,7 +248,7 @@ func TestProjectModelV3UsesMultiNodePropSyncBoundary(t *testing.T) { snapshot := acquireTestSnapshot(t, buildDir) t.Log("Project the multi-node prop-sync chain") - projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineSingle, "") + projection := projectTestBuild(t, normalizeTestSnapshot(t, snapshot), PipelineSingle) t.Log("Project logical connections from the source partition's final node") spec := projection.RequestSpec(&MaterializationPlan{}, "agents") diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index a704154ff84b..aa9570351910 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -8,7 +8,6 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8sptr "k8s.io/utils/ptr" ) @@ -31,7 +30,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, }}} - component.LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"stop_tokens":[1],"custom_runtime_knob":{"enabled":true}}`)} component.Roles[0].PodTemplate.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ @@ -105,13 +103,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), wantCELErr: "spec.components[0]: Invalid value: scalingAdapter is not supported when type is lpx", }, - { - name: "LPX settings must be an object", - deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { - dgd.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`[]`)} - }), - wantSchemaErr: `spec.components[0].lpx.settings: Invalid value: spec.components[0].lpx.settings in body must be of type object: "array"`, - }, { name: "LPX integration availability is deferred to the child controller", lpxDisabled: true, @@ -475,8 +466,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, }}} setBetaLPXSpecDec(dgd, k8sptr.To(int32(2))) - dgd.Spec.Components[0].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`{"draft_runtime":{"enabled":true}}`)} - dgd.Spec.Components[1].LPX.Settings = &apiextensionsv1.JSON{Raw: []byte(`null`)} }), }, { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index cf4ea3a4d053..9d5371459c02 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2509,7 +2509,7 @@ _Appears in:_ -LPXConfig identifies the component's compiled model and placement settings. +LPXConfig identifies the component's compiled model. @@ -2520,7 +2520,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `buildId` _string_ | buildId references the immutable model build. | | MinLength: 1
| -| `settings` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | settings configure model placement. Runtime options belong in role pod templates. | | Type: object
Optional: \{\}
| #### MockerSpec diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 11622062c48c..5311d754fbb4 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -59,7 +59,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - DGD-only compiled-model configuration. Set `buildId`; see [LPXConfig](full-api-reference.mdx#lpxconfig) for the field definitions. LPU-only Single and SpecDecode accept only boolean placement `settings`: `prop_sync`, plus XT-only `cpu_embeddings` and `prop_sync_multiple_load_sets`. Runtime options belong in the conductor pod template. + DGD-only compiled-model configuration. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index d0ee1b70c02e..acbb4df21d7a 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3667,7 +3667,7 @@ preferredWeight is required and used only when enforcement is "preferred". Highe -LPXConfig identifies the component's compiled model and placement settings. +LPXConfig identifies the component's compiled model. **Kind:** `type` @@ -3677,12 +3677,6 @@ LPXConfig identifies the component's compiled model and placement settings. buildId references the immutable model build. **Validation:** MinLength: 1 - - -settings configure model placement. Runtime options belong in role pod templates. -See [JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io). -**Validation:** Type: object Optional: \{\} - From 535a00609f2f6c7f4b2694927a05c7464a81c33c Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 18 Sep 2026 13:03:02 +0000 Subject: [PATCH 101/196] fix(operator): preserve LPX endpoint Service allocations Signed-off-by: Andrew Paprotsky (cherry picked from commit 7e1198826a97baa6563817abbbf7d554abb7bcbd) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 9 ++++++++ .../controller/lpx/controller_test.go | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 8944452b86e8..bc90521d618d 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -521,6 +521,15 @@ func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alp return fmt.Errorf("refusing to adopt LPX resource %s/%s without its exact owner", live.GetNamespace(), live.GetName()) } + // Keep the endpoint's API-allocated addresses when replacing its rendered spec. + if service, ok := desired.(*corev1.Service); ok && live != nil { + observed := live.(*corev1.Service) + service.Spec.ClusterIP = observed.Spec.ClusterIP + service.Spec.ClusterIPs = observed.Spec.ClusterIPs + service.Spec.IPFamilies = observed.Spec.IPFamilies + service.Spec.IPFamilyPolicy = observed.Spec.IPFamilyPolicy + } + // Prepare LPX-owned metadata while preserving shared synchronization's bookkeeping. metadataChanged := false if live != nil { diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index fe1eae942890..ebef115b8822 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1020,6 +1020,29 @@ func TestLPXEndpointLifecycle(t *testing.T) { require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) } + t.Log("Preserve API-allocated Service fields when the endpoint selector changes") + require.NoError(t, r.Get(t.Context(), key, service)) + service.Spec.ClusterIP = "10.96.0.10" + service.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} + service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + service.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyPreferDualStack) + require.NoError(t, r.Update(t.Context(), service)) + allocated := service.DeepCopy() + for _, global := range []bool{true, false} { + source.Spec.Components[0].GlobalDynamoNamespace = global + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + require.NoError(t, r.Get(t.Context(), key, service)) + allocated.Spec.Selector[consts.KubeLabelDynamoNamespace] = source.GetDynamoNamespaceForComponent(&source.Spec.Components[0]) + require.Equal(t, allocated.Spec, service.Spec) + require.Equal(t, allocated.OwnerReferences, service.OwnerReferences) + + t.Log("Reconcile the unchanged endpoint without another write") + version := service.ResourceVersion + require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) + require.NoError(t, r.Get(t.Context(), key, service)) + require.Equal(t, version, service.ResourceVersion) + } + t.Log("Remove the LPX-owned discovery endpoint when switching back to non-Kubernetes discovery") delete(source.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) From f8b7ab81deac247927c27e9da424dd643aed5687 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:32:48 +0300 Subject: [PATCH 102/196] refactor(lpx): leave runtime bindings to pod templates (#540) Signed-off-by: Andrew Paprotsky (cherry picked from commit 7d306f7ad2d753e781c445101a73fba60297a7f1) Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 30 ++----- .../internal/dynamo/lpx/lpu_configmap.go | 34 +------- .../internal/dynamo/lpx/lpu_configmap_test.go | 87 +++++-------------- deploy/operator/internal/dynamo/lpx/render.go | 22 ++--- .../internal/dynamo/lpx/render_test.go | 6 +- .../internal/dynamo/lpx/runtime_agent.go | 26 +----- .../dynamo/lpx/runtime_startup_test.go | 39 ++++----- .../from_dgd_yaml/lpx_v2_vllm.input.yaml | 2 + .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 14 ++- .../node-local-v2-hybrid.input.yaml | 2 + .../from_dgd_yaml/node-local-v2-hybrid.yaml | 14 ++- .../node-local-v3-hx-hybrid.input.yaml | 2 + .../node-local-v3-hx-hybrid.yaml | 14 ++- 13 files changed, 89 insertions(+), 203 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index dac4c5dddecf..1e424e786d6f 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -1122,12 +1122,9 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test require.Equal(t, "mlx5_0", runtimeEnv["NIC_NAME"]) require.Equal(t, "19878", runtimeEnv["READINESS_PORT"]) require.Equal(t, "metadata.annotations['"+lpxv1alpha1.PodModelAnnotation+"']", envValueSource(main.Env, "LPU_MODEL_NAME").FieldRef.FieldPath) - gasDirSource := envValueSource(main.Env, "GAS_DIR") - require.NotNil(t, gasDirSource) - require.NotNil(t, gasDirSource.ConfigMapKeyRef) + require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", runtimeEnv["LPX_MODEL_PATH"]) + require.Equal(t, "$(LPX_MODEL_PATH)", runtimeEnv["GAS_DIR"]) lpuConfigName := lpx.LPUConfigMapName(plan.PodCliqueSetName, agent.Annotations[commonconsts.AnnotationExtraResourcesHash]) - require.Equal(t, lpuConfigName, gasDirSource.ConfigMapKeyRef.Name) - require.Equal(t, "gas_dir", gasDirSource.ConfigMapKeyRef.Key) require.NotContains(t, runtimeEnv, "TOPOLOGIES") podIPSource := envValueSource(main.Env, "POD_IP") require.NotNil(t, podIPSource) @@ -1198,7 +1195,6 @@ func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *test t.Log("Keep partition zero as the collapsed runtime root for all three physical partitions") lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, lpuConfigName) require.Equal(t, "0", lpuConfig.Data["partition_ids"]) - require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", lpuConfig.Data["gas_dir"]) t.Log("Verify generated Cyborg and LPU config resource order and contents") decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, decodeConfigName) @@ -1236,13 +1232,12 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. file string hybrid bool conductorConfig bool - hx bool }{ {name: "Single V2", file: "single_v2.input.yaml", conductorConfig: true}, {name: "V2 LPU-only", file: "node-local-v2-lpu-only.input.yaml", conductorConfig: true}, {name: "V2 hybrid", file: "node-local-v2-hybrid.input.yaml", hybrid: true, conductorConfig: true}, - {name: "HX LPU-only", file: "node-local-v3-hx-lpu-only.input.yaml", conductorConfig: true, hx: true}, - {name: "HX hybrid", file: "node-local-v3-hx-hybrid.input.yaml", hybrid: true, hx: true}, + {name: "HX LPU-only", file: "node-local-v3-hx-lpu-only.input.yaml", conductorConfig: true}, + {name: "HX hybrid", file: "node-local-v3-hx-hybrid.input.yaml", hybrid: true}, } intents := []struct { name string @@ -1259,9 +1254,9 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. {name: "image entrypoint and image command"}, {name: "custom pod settings and exec health probes", command: []string{"/opt/custom-runtime"}, customPodSettings: true}, {name: "default startup without config mount", configPath: ptr.To(""), configRole: 1}, - {name: "default startup with misplaced config mount", configPath: ptr.To("/custom"), configRole: 1}, + {name: "default startup with custom read-only config mount", configPath: ptr.To("/custom"), configRole: 1}, {name: "default Agent without config mount", configPath: ptr.To("")}, - {name: "default Agent with misplaced config mount", configPath: ptr.To("/custom")}, + {name: "default Agent with custom read-only config mount", configPath: ptr.To("/custom")}, {name: "default Agent without config mount and unrelated EnvFrom", configPath: ptr.To(""), envFrom: true}, {name: "image entrypoint with user arguments", args: []string{"serve"}}, {name: "explicit executable with image arguments", command: []string{"/opt/custom-runtime"}}, @@ -1415,6 +1410,7 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. main.VolumeMounts = slices.Delete(main.VolumeMounts, configIndex, configIndex+1) } else { main.VolumeMounts[configIndex].MountPath = *intent.configPath + main.VolumeMounts[configIndex].ReadOnly = true } } } @@ -1433,10 +1429,6 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. newLPXRenderDeployment(t, dgd), ) require.Equal(t, before, dgd) - if mode.hx && intent.configPath != nil && *intent.configPath != "" { - require.ErrorContains(t, err, `reserves volume "config" at "/configs"`) - return - } require.NoError(t, err) t.Log("Verify shared defaults remain overridable while runtime-specific omissions stay empty") @@ -1496,13 +1488,7 @@ func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing. actualVolumes := slices.DeleteFunc(slices.Clone(pod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) expectedVolumes := slices.DeleteFunc(slices.Clone(authoredPod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) require.ElementsMatch(t, expectedVolumes, actualVolumes, "%s shared memory volumes", clique.Name) - actualMounts := slices.DeleteFunc(slices.Clone(main.VolumeMounts), func(mount corev1.VolumeMount) bool { - return mount.MountPath != commonconsts.DefaultSharedMemoryMountPath - }) - expectedMounts := slices.DeleteFunc(slices.Clone(authored.VolumeMounts), func(mount corev1.VolumeMount) bool { - return mount.MountPath != commonconsts.DefaultSharedMemoryMountPath - }) - require.ElementsMatch(t, expectedMounts, actualMounts, "%s shared memory mounts", clique.Name) + require.ElementsMatch(t, authored.VolumeMounts, main.VolumeMounts, "%s mounts", clique.Name) switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { case lpxv1alpha1.PodRoleAgent: agents++ diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index e8db8d33538d..30462226ad0f 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -22,10 +22,7 @@ import ( "k8s.io/utils/ptr" ) -const ( - lpuConfigVolumeName = "config" - lpuConfigMountPath = "/configs" -) +const lpuConfigVolumeName = "config" // LPUConfigMapName names the immutable runtime table using its Pod-template content hash. // The root is the PCS identity carried by Grove's part-of Pod label. @@ -33,24 +30,6 @@ func LPUConfigMapName(root, configHash string) string { return fmt.Sprintf("%s-lpu-%.16s", root, configHash) } -func renderLPUConfigMap( - namespace string, - materializationName string, - modelStoragePath string, - projections []*ModelProjection, -) (*corev1.ConfigMap, error) { - data := resolvedPartitionData(projections) - if projections[0].pipeline == PipelineLPX { - // Direct agents consume the build directory and partition table. - modelPath, err := buildRuntimePath(lpuRuntimeBuildRef(projections[0], modelStoragePath), modelStoragePath) - if err != nil { - return nil, fmt.Errorf("resolve gas_dir: %w", err) - } - data["gas_dir"] = modelPath - } - return renderRuntimeConfigMap(namespace, materializationName+"-lpu", data) -} - func renderRuntimeConfigMap(namespace, namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ @@ -213,16 +192,5 @@ func withLPUConfigVolume(spec *corev1.PodSpec, configMapName string, allowOverri }}, }) } - container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) - found = false - configMount := corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath} - for _, mount := range container.VolumeMounts { - if mount.Name == lpuConfigVolumeName || mount.MountPath == lpuConfigMountPath { - if !allowOverrides && (found || mount != configMount) { - return fmt.Errorf("selected LPX main container reserves volume %q at %q", lpuConfigVolumeName, lpuConfigMountPath) - } - found = true - } - } return nil } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 420ada0012af..5c4bf7084232 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -87,49 +87,6 @@ func TestLPURuntimeBuildRef(t *testing.T) { } } -func TestRenderLPUConfigMapPreservesV2HybridGasDir(t *testing.T) { - t.Parallel() - - t.Log("Construct a V2 hybrid projection with an image-owned runtime build") - projection := &ModelProjection{ - model: "default", - pipeline: PipelineLPX, - runtimeBuildRef: "model-build", - configuredBuild: Build{ - Path: "file:///snapshot-build", - Family: BuildFamilyXT, - }, - } - - t.Log("Render the hybrid ConfigMap from the selected pipeline") - configMap, err := renderLPUConfigMap( - "test-namespace", - "test-dgd", - "/models", - []*ModelProjection{projection}, - ) - - t.Log("Keep gas_dir rooted at the selected image-owned build") - require.NoError(t, err) - require.NotContains(t, configMap.Data, "model_config.toml") - require.NotContains(t, configMap.Data, "datacenter.toml") - require.Equal(t, "/models/model-build", configMap.Data["gas_dir"]) - require.True(t, *configMap.Immutable) - require.Equal(t, LPUConfigMapName("test-dgd", LPUConfigMapHash(configMap)), configMap.Name) - - t.Log("Render the preserved runtime with an invalid snapshot build reference") - projection.configuredBuild.Path = "relative-build" - _, err = renderLPUConfigMap( - "test-namespace", - "test-dgd", - "/models", - []*ModelProjection{projection}, - ) - - t.Log("Return the model-path error before projecting gas_dir") - require.ErrorContains(t, err, "resolve gas_dir: parse build path \"relative-build\": ref \"relative-build\" must be an absolute path or URL") -} - func TestResolvedPartitionDataOmitsXTModelColumnsBeforeMaterialization(t *testing.T) { t.Parallel() @@ -159,32 +116,28 @@ func TestResolvedPartitionDataOmitsXTModelColumnsBeforeMaterialization(t *testin }, data) } -func TestConductorConfigMapContainsOnlyPartitions(t *testing.T) { - t.Parallel() - - for _, family := range []string{"v2", "v3"} { - t.Run(family, func(t *testing.T) { - t.Log("Project a compiler build with a template-owned runtime configuration") - var path string - if family == "v2" { - path = writeV2CompilerFixture(t) - } else { - path = writeV3CompilerFixture(t) +func TestLPUConfigVolumeRejectsAuthoredSourceMismatch(t *testing.T) { + for _, test := range []struct { + name string + source corev1.VolumeSource + }{ + {name: "other ConfigMap", source: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "other-config"}, + }}}, + {name: "other volume source", source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Keep an authored runtime mount with a conflicting generated-volume source") + spec := corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main", VolumeMounts: []corev1.VolumeMount{{Name: "config", MountPath: "/runtime/partitions", ReadOnly: true}}}}, + Volumes: []corev1.Volume{{Name: "config", VolumeSource: test.source}}, } - projections, err := appendModelProjections(nil, ModelProjectionInput{ - Pipeline: PipelineSingle, Models: []string{"default"}, - BuildSnapshot: normalizeTestSnapshot(t, acquireTestSnapshot(t, path)), - }) - require.NoError(t, err) + before := spec.DeepCopy() - t.Log("Retain only the resolved partition files in the immutable ConfigMap") - configMap, err := renderLPUConfigMap("test", "test-dgd", "/models", projections) - require.NoError(t, err) - require.Equal(t, resolvedPartitionData(projections), configMap.Data) - require.NotContains(t, configMap.Data, "model_config.toml") - require.NotContains(t, configMap.Data, "datacenter.toml") - require.NotEmpty(t, configMap.Data["partition_paths"]) - require.NotEmpty(t, configMap.Data["topologies"]) + t.Log("Reject the source mismatch before changing the HX template") + err := withLPUConfigVolume(&spec, "generated-config", false) + require.ErrorContains(t, err, `volume "config" is reserved for ConfigMap "generated-config"`) + require.Equal(t, *before, spec) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 7b6419e6a6ee..2fa61eb13af9 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -90,7 +90,7 @@ func RenderSelectedNodeLocal( if err != nil { return nil, err } - configMap, err := renderLPUConfigMap(namespace, plan.PodCliqueSetName, modelStoragePath, projections) + configMap, err := renderRuntimeConfigMap(namespace, plan.PodCliqueSetName+"-lpu", resolvedPartitionData(projections)) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func RenderSelectedNodeLocal( var conductor *grovev1alpha1.PodCliqueTemplateSpec if conductorTemplateName != "" { container := common.FindContainerByName(conductorTemplate.Spec.Containers, commonconsts.MainContainerName) - if err := applyConductorModelPaths(container, projections, modelStoragePath); err != nil { + if err := applyModelPaths(container, projections, modelStoragePath); err != nil { return nil, err } annotations := roleAnnotations(conductorTemplate.Annotations, lpxv1alpha1.PodRoleConductor, workloadDigest) @@ -149,6 +149,14 @@ func RenderSelectedNodeLocal( if storagePath != modelStoragePath { return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStoragePath) } + + // Publish the model path before template-owned hybrid Agent bindings. + if hybrid { + container := common.FindContainerByName(template.Spec.Containers, commonconsts.MainContainerName) + if err := applyModelPaths(container, projections, modelStoragePath); err != nil { + return nil, fmt.Errorf("stage %s: %w", stage, err) + } + } var conductorSpec *corev1.PodSpec if stage == conductorStage && conductor != nil { conductorSpec = &conductor.Spec.PodSpec @@ -252,15 +260,9 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo if err := withLPUConfigVolume(conductorPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { return err } + configureNodeLocalConductorRuntime(conductorPodSpec, allocation) } - if workload.Pipeline() == PipelineLPX { - configureDirectHybridAgentRuntime(agentPodSpec, configMapName) - } else { - if conductorPodSpec != nil { - configureNodeLocalConductorRuntime(conductorPodSpec, allocation) - } - configureAgentIdentity(agentPodSpec) - } + configureAgentIdentity(agentPodSpec) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 5bf31fced31c..daa812641045 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -139,7 +139,7 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}) + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/configs"}) } else { cyborg := namedClique(t, rendered, "cond").Spec.PodSpec.Containers[0] require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/model-cache"}) @@ -346,7 +346,7 @@ func renderTestPCS(hybrid bool) *grovev1alpha1.PodCliqueSet { Containers: []corev1.Container{{Name: "main", Image: "cyborg", Env: []corev1.EnvVar{ {Name: "SERVER_HOSTS_FILE", Value: "/tmp/lpu_servers"}, }, VolumeMounts: []corev1.VolumeMount{ - {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + {Name: lpuConfigVolumeName, MountPath: "/configs"}, {Name: "model-storage", MountPath: "/models"}, }}}, ResourceClaims: []corev1.PodResourceClaim{{ @@ -376,7 +376,7 @@ func renderTestPodSpec() corev1.PodSpec { Name: "main", Image: "runtime", VolumeMounts: []corev1.VolumeMount{ {Name: "model-storage", MountPath: "/models"}, - {Name: lpuConfigVolumeName, MountPath: lpuConfigMountPath}, + {Name: lpuConfigVolumeName, MountPath: "/configs"}, {Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}, {Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}, }, diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 0184f47be7dc..70daf3a9961f 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -15,25 +15,6 @@ import ( corev1 "k8s.io/api/core/v1" ) -// configureDirectHybridAgentRuntime lowers an LPX-scheduled PodSpec into the -// direct agent runtime used by conductorless hybrid workloads. -func configureDirectHybridAgentRuntime( - agentPodSpec *corev1.PodSpec, - lpuConfigMapName string, -) { - agent := configureAgentIdentity(agentPodSpec) - - agent.Env = append(agent.Env, - corev1.EnvVar{ - Name: "GAS_DIR", - ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: lpuConfigMapName}, - Key: "gas_dir", - }}, - }, - ) -} - // configureNodeLocalConductorRuntime consumes a fresh conductor PodSpec with a validated main container. func configureNodeLocalConductorRuntime( conductorPodSpec *corev1.PodSpec, @@ -46,8 +27,8 @@ func configureNodeLocalConductorRuntime( retargetMainContainerReferences(conductorPodSpec, conductor) } -// applyConductorModelPaths binds nonempty canonical projections into the fresh conductor container. -func applyConductorModelPaths(container *corev1.Container, projections []*ModelProjection, modelStoragePath string) error { +// applyModelPaths binds nonempty canonical projections into a fresh runtime container. +func applyModelPaths(container *corev1.Container, projections []*ModelProjection, modelStoragePath string) error { type modelPathBinding struct { name string projection *ModelProjection @@ -78,11 +59,10 @@ func applyConductorModelPaths(container *corev1.Container, projections []*ModelP // configureAgentIdentity names the main runtime and retargets its container references. // agentPodSpec must be non-nil and contain the validated main container. -func configureAgentIdentity(agentPodSpec *corev1.PodSpec) *corev1.Container { +func configureAgentIdentity(agentPodSpec *corev1.PodSpec) { agent := common.FindContainerByName(agentPodSpec.Containers, commonconsts.MainContainerName) agent.Name = lpuAgentContainerName retargetMainContainerReferences(agentPodSpec, agent) - return agent } // retargetMainContainerReferences requires its non-nil target to point into the non-nil PodSpec's Containers. diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index a1068319f3cd..3187b4862560 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -22,14 +22,11 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { name string family BuildFamily conductor bool - direct bool }{ {name: "XT conductor", family: BuildFamilyXT, conductor: true}, {name: "HX conductor", family: BuildFamilyHX, conductor: true}, {name: "XT worker", family: BuildFamilyXT}, {name: "HX worker", family: BuildFamilyHX}, - {name: "XT direct hybrid", family: BuildFamilyXT, direct: true}, - {name: "HX direct hybrid", family: BuildFamilyHX, direct: true}, } { for _, startup := range []struct { name string @@ -98,12 +95,9 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { t.Log("Apply runtime bindings for the selected family and role") configureAgentScheduling(&pod, role.family) - switch { - case role.conductor: + if role.conductor { configureNodeLocalConductorRuntime(&pod, "allocation") - case role.direct: - configureDirectHybridAgentRuntime(&pod, "config") - default: + } else { configureAgentIdentity(&pod) } @@ -122,12 +116,8 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { require.Equal(t, authored.SecurityContext, container.SecurityContext) t.Log("Keep static environment values template-owned, including intentional omission") - dynamicEnv := map[string]bool{ - "LPX_ALLOCATION": role.conductor, - "GAS_DIR": role.direct, - } - container.Env = slices.DeleteFunc(slices.Clone(container.Env), func(variable corev1.EnvVar) bool { return dynamicEnv[variable.Name] }) if role.conductor { + container.Env = slices.DeleteFunc(slices.Clone(container.Env), func(variable corev1.EnvVar) bool { return variable.Name == "LPX_ALLOCATION" }) authored.Env = slices.DeleteFunc(slices.Clone(authored.Env), func(variable corev1.EnvVar) bool { return variable.Name == "LPX_ALLOCATION" }) } require.Equal(t, authored.Env, container.Env) @@ -145,7 +135,7 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { } } -func TestConductorModelPathsPrecedeAuthoredReferences(t *testing.T) { +func TestModelPathsPrecedeAuthoredReferences(t *testing.T) { for _, test := range []struct { name string family BuildFamily @@ -154,11 +144,13 @@ func TestConductorModelPathsPrecedeAuthoredReferences(t *testing.T) { }{ {name: "XT Single GCS", family: BuildFamilyXT, pipeline: PipelineSingle}, {name: "HX Single local", family: BuildFamilyHX, pipeline: PipelineSingle, local: true}, + {name: "XT hybrid local", family: BuildFamilyXT, pipeline: PipelineLPX, local: true}, + {name: "HX hybrid GCS", family: BuildFamilyHX, pipeline: PipelineLPX}, {name: "XT multiple drafts local", family: BuildFamilyXT, pipeline: PipelineSpecDecode, local: true}, {name: "HX multiple drafts GCS", family: BuildFamilyHX, pipeline: PipelineSpecDecode}, } { t.Run(test.name, func(t *testing.T) { - t.Log("Resolve each model from the registry reference and the conductor's custom mount") + t.Log("Resolve each model from the registry reference and the runtime's custom mount") count := 1 names := []string{"LPX_MODEL_PATH"} if test.pipeline == PipelineSpecDecode { @@ -187,8 +179,12 @@ func TestConductorModelPathsPrecedeAuthoredReferences(t *testing.T) { } t.Log("Author dependent values before duplicate forged bindings") + runtimeVariable := "A_RUNTIME_MODEL" + if test.pipeline == PipelineLPX { + runtimeVariable = "GAS_DIR" + } authored := []corev1.EnvVar{ - {Name: "A_RUNTIME_MODEL", Value: "$(" + names[0] + ")"}, + {Name: runtimeVariable, Value: "$(" + names[0] + ")"}, {Name: "OTHER", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, } container := corev1.Container{Command: []string{"custom-runtime"}, Args: []string{"--unchanged"}, Env: slices.Clone(authored)} @@ -199,25 +195,26 @@ func TestConductorModelPathsPrecedeAuthoredReferences(t *testing.T) { wantContainer.Env = append(want, authored...) t.Log("Publish only the first draft and final target once, before all authored references") - require.NoError(t, applyConductorModelPaths(&container, projections, "/custom/models")) + require.NoError(t, applyModelPaths(&container, projections, "/custom/models")) require.Equal(t, *wantContainer, container) t.Log("Repeated rendering keeps environment order, startup and authoritative values unchanged") - require.NoError(t, applyConductorModelPaths(&container, projections, "/custom/models")) + require.NoError(t, applyModelPaths(&container, projections, "/custom/models")) require.Equal(t, *wantContainer, container) }) } } -func TestConductorModelPathsRejectInvalidReferences(t *testing.T) { +func TestModelPathsRejectInvalidReferences(t *testing.T) { for _, test := range []struct { pipeline Pipeline name string }{ {pipeline: PipelineSingle, name: "LPX_MODEL_PATH"}, + {pipeline: PipelineLPX, name: "LPX_MODEL_PATH"}, {pipeline: PipelineSpecDecode, name: "LPX_TARGET_MODEL_PATH"}, } { - t.Run(test.name, func(t *testing.T) { + t.Run(string(test.pipeline)+"/"+test.name, func(t *testing.T) { t.Log("Keep the target reference invalid after a valid speculative draft") projections := []*ModelProjection{{pipeline: test.pipeline, configuredBuild: Build{Path: "gs://registry/model"}}} if test.pipeline == PipelineSpecDecode { @@ -228,7 +225,7 @@ func TestConductorModelPathsRejectInvalidReferences(t *testing.T) { before := container.DeepCopy() t.Log("Report the failing binding without publishing a partial environment") - err := applyConductorModelPaths(&container, projections, "/custom/models") + err := applyModelPaths(&container, projections, "/custom/models") require.ErrorContains(t, err, "resolve "+test.name) require.ErrorContains(t, err, "bad path segment") require.Equal(t, *before, container) diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml index e1b870e3f893..76b92f999ae4 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.input.yaml @@ -202,6 +202,8 @@ spec: successThreshold: 1 timeoutSeconds: 1 env: + - name: GAS_DIR + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: '1024' - name: NIC_NAME diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 652bb87a79b5..eb9dec585e21 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -32,7 +32,6 @@ metadata: namespace: inference-engine --- data: - gas_dir: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c nodes_per_partition: |- 1 1 @@ -167,7 +166,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 + name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 namespace: inference-engine --- metadata: @@ -189,7 +188,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 5448f938d5d52c694963453bff4e63367aa296f1c0aa36ea0960da60cad289b6 + nvidia.com/extra-resources-hash: e586b76cc8ccd9a1ed2e1b0297032eb1da420881dcfdcb632a4f00f4971308ff scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 @@ -276,16 +275,15 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: GAS_DIR - valueFrom: - configMapKeyRef: - key: gas_dir - name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] + - name: LPX_MODEL_PATH + value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -394,7 +392,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-lpu-5448f938d5d52c69 + name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 name: config replicas: 17 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index e8e1d6c49191..923756e3c6fb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -122,6 +122,8 @@ spec: successThreshold: 1 timeoutSeconds: 1 env: + - name: GAS_DIR + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: '1024' - name: NIC_NAME diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index ea5d6e12f34d..af2053019458 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -15,7 +15,6 @@ metadata: namespace: test --- data: - gas_dir: /nfs/node-local-v2-connected-lpx nodes_per_partition: "4" partition_ids: "0" partition_indices: "0" @@ -31,7 +30,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e + name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 namespace: test --- metadata: @@ -53,7 +52,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: a2baf441cd41d87e6ae2b9457c6134d41e06134d1091efe63bb4c4a01a17ef5f + nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 @@ -91,16 +90,15 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: GAS_DIR - valueFrom: - configMapKeyRef: - key: gas_dir - name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] + - name: LPX_MODEL_PATH + value: /nfs/node-local-v2-connected-lpx - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -209,7 +207,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-lpu-a2baf441cd41d87e + name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 name: config replicas: 4 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index 769722dfffaa..b44a806bbef4 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -107,6 +107,8 @@ spec: successThreshold: 1 timeoutSeconds: 1 env: + - name: GAS_DIR + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: '1024' - name: NIC_NAME diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 0e117c200329..b065484cd8b8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 data: - gas_dir: /nfs/node-local-v3-hx-connected-lpx nodes_per_partition: "1" partition_ids: "1" partition_indices: "0" @@ -18,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 + name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd namespace: test --- metadata: @@ -40,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: dc7a6b2e2078e8215ec068db5b51870b05382a0252a8e8c788c34b1173fe44ca + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 @@ -78,16 +77,15 @@ spec: - name: ETCD_ENDPOINTS value: etcd-address - name: GAS_DIR - valueFrom: - configMapKeyRef: - key: gas_dir - name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 + value: $(LPX_MODEL_PATH) - name: GLUE_RDMA_PATH_MTU value: "1024" - name: LPU_MODEL_NAME valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] + - name: LPX_MODEL_PATH + value: /nfs/node-local-v3-hx-connected-lpx - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -196,7 +194,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-hybrid-be18-lpu-dc7a6b2e2078e821 + name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: agt From c18f27d2d6d8b47590bd9c292ba5851195a9576f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 18 Sep 2026 20:58:51 +0000 Subject: [PATCH 103/196] chore(codeowners): cover LPX integration filenames Signed-off-by: Andrew Paprotsky --- .github/codeowners/areas.yaml | 2 ++ CODEOWNERS | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 5a48f5f03fce..bd94a9a30743 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -994,6 +994,8 @@ shared: owners: - operator - lpx +- glob: deploy/**/*lpx* + owners: [operator, lpx] - glob: deploy/operator/internal/controller/*checkpoint* owners: - operator diff --git a/CODEOWNERS b/CODEOWNERS index f3e3673bbff2..717c59f97009 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -584,6 +584,7 @@ requirements.txt /agent-docs/ @ai-dynamo/dynamo-process-codeowners @ai-dynamo/dynamo-agents-codeowners /.lycheeignore @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-docs-codeowners /tests/deploy/ @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners +/deploy/**/*lpx* @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-lpx-codeowners /benchmarks/router/ @ai-dynamo/dynamo-performance-codeowners @ai-dynamo/dynamo-router-codeowners /recipes/**/vllm/** @ai-dynamo/dynamo-performance-codeowners @ai-dynamo/dynamo-backend-vllm-codeowners /.devcontainer/vllm/ @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-backend-vllm-codeowners From bb71fcfbffb3c40ab2f6298be665b12bd19bfa67 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 18 Sep 2026 21:33:16 +0000 Subject: [PATCH 104/196] docs(kubernetes): remove LPX details from DGD guide Signed-off-by: Andrew Paprotsky --- .../dynamo/lpx/runtime_startup_test.go | 1 - .../node-local-v3-hx-lpu-only.input.yaml | 36 ---------- .../node-local-v3-hx-lpu-only.yaml | 36 ---------- .../node-local-v3-hx-specdecode.input.yaml | 36 ---------- .../node-local-v3-hx-specdecode.yaml | 68 ------------------- .../model-deployment/deploy-with-dgd.md | 13 ---- .../dynamo-component-deployment.mdx | 2 - 7 files changed, 192 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go index 3187b4862560..1e7b242d8bf7 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go @@ -75,7 +75,6 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { } if startup.command != nil { pod.Containers[1].Env = append(pod.Containers[1].Env, - corev1.EnvVar{Name: "GROQ_V2_RESET_ON_OPEN", Value: "template-value"}, corev1.EnvVar{Name: "TEMPLATE_SECRET", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: "settings"}, Key: "value", }}}, diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index 50451eb85667..bd4307b8c6f1 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -62,26 +62,6 @@ spec: - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars - - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 - - --agent-env-vars - - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false - - --agent-env-vars - - GROQ_HAL_V2_BACKGROUND_METRICS=false - - --agent-env-vars - - GROQ_PG613_PORT_CONFIG=EB3 - - --agent-env-vars - - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false - - --agent-env-vars - - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 - - --agent-env-vars - - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 - - --agent-env-vars - - GROQ_V2_RESET_ON_OPEN=0 - - --agent-env-vars - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false - --agent-env-vars - LPU_FPGA_RECONFIGURE_METHOD=cvp @@ -183,22 +163,6 @@ spec: env: - name: BMC_SSH_ENABLED value: 'false' - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: 'false' - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: 'false' - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: '0' - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: '0' - - name: GROQ_V2_RESET_ON_OPEN - value: '0' - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: 'false' - name: LPU_FPGA_RECONFIGURE_METHOD diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 034ddf98088e..339a323cad30 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -76,22 +76,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: "false" - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: "false" - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: "0" - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: "0" - - name: GROQ_V2_RESET_ON_OPEN - value: "0" - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: "false" - name: LPU_FPGA_RECONFIGURE_METHOD @@ -262,26 +246,6 @@ spec: - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars - - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 - - --agent-env-vars - - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false - - --agent-env-vars - - GROQ_HAL_V2_BACKGROUND_METRICS=false - - --agent-env-vars - - GROQ_PG613_PORT_CONFIG=EB3 - - --agent-env-vars - - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false - - --agent-env-vars - - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 - - --agent-env-vars - - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 - - --agent-env-vars - - GROQ_V2_RESET_ON_OPEN=0 - - --agent-env-vars - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false - --agent-env-vars - LPU_FPGA_RECONFIGURE_METHOD=cvp diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 7b76189d0f48..44a18ac8cea2 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -41,22 +41,6 @@ spec: env: - name: BMC_SSH_ENABLED value: 'false' - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: 'false' - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: 'false' - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: '0' - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: '0' - - name: GROQ_V2_RESET_ON_OPEN - value: '0' - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: 'false' - name: LPU_FPGA_RECONFIGURE_METHOD @@ -173,26 +157,6 @@ spec: - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars - - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 - - --agent-env-vars - - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false - - --agent-env-vars - - GROQ_HAL_V2_BACKGROUND_METRICS=false - - --agent-env-vars - - GROQ_PG613_PORT_CONFIG=EB3 - - --agent-env-vars - - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false - - --agent-env-vars - - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 - - --agent-env-vars - - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 - - --agent-env-vars - - GROQ_V2_RESET_ON_OPEN=0 - - --agent-env-vars - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false - --agent-env-vars - LPU_FPGA_RECONFIGURE_METHOD=cvp diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 8b3b53d25e6d..c63f47f48b24 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -97,22 +97,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: "false" - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: "false" - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: "0" - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: "0" - - name: GROQ_V2_RESET_ON_OPEN - value: "0" - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: "false" - name: LPU_FPGA_RECONFIGURE_METHOD @@ -267,22 +251,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: "false" - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: "false" - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: "0" - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: "0" - - name: GROQ_V2_RESET_ON_OPEN - value: "0" - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: "false" - name: LPU_FPGA_RECONFIGURE_METHOD @@ -437,22 +405,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: GROQ_HAL_V2_BACKGROUND_METRICS - value: "false" - - name: GROQ_PG613_PORT_CONFIG - value: EB3 - - name: GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2 - value: "false" - - name: GROQ_PG613_SKIP_HOST_LINK_PROFILE - value: "0" - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_0 - value: 0000:9a:00.0 - - name: GROQ_V2_FPGA_BUS_ADDRESS_ID_1 - value: 0000:7e:00.0 - - name: GROQ_V2_HEALTH_CHECK_ON_OPEN - value: "0" - - name: GROQ_V2_RESET_ON_OPEN - value: "0" - name: LPU_FPGA_AUTOMATIC_JTAG_RECOVERY value: "false" - name: LPU_FPGA_RECONFIGURE_METHOD @@ -627,26 +579,6 @@ spec: - --agent-env-vars - BMC_SSH_ENABLED=false - --agent-env-vars - - GROQ_FPGA_VSAP_DRAM_WAIT_CYCLES=0 - - --agent-env-vars - - GROQ_FPGA_VSAP_SETUP_MPINGO_KERNELS=false - - --agent-env-vars - - GROQ_HAL_V2_BACKGROUND_METRICS=false - - --agent-env-vars - - GROQ_PG613_PORT_CONFIG=EB3 - - --agent-env-vars - - GROQ_PG613_REQUIRE_ALTAIR_PCIE_GEN2=false - - --agent-env-vars - - GROQ_PG613_SKIP_HOST_LINK_PROFILE=1 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_0=0000:9a:00.0 - - --agent-env-vars - - GROQ_V2_FPGA_BUS_ADDRESS_ID_1=0000:7e:00.0 - - --agent-env-vars - - GROQ_V2_HEALTH_CHECK_ON_OPEN=0 - - --agent-env-vars - - GROQ_V2_RESET_ON_OPEN=0 - - --agent-env-vars - LPU_FPGA_AUTOMATIC_JTAG_RECOVERY=false - --agent-env-vars - LPU_FPGA_RECONFIGURE_METHOD=cvp diff --git a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md index d07705479dab..781be4e83917 100644 --- a/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md +++ b/docs/fern/pages/kubernetes/model-deployment/deploy-with-dgd.md @@ -257,19 +257,6 @@ vLLM and SGLang take tensor/pipeline/data parallelism as CLI flags (covered in t You normally do not need to set the top-level `spec.backendFramework` field — the operator infers the backend from the worker command. Set it explicitly (`vllm`, `sglang`, or `trtllm`) only when a feature needs the framework known up front, such as GMS failover or multinode TensorRT-LLM. -For LPX, set `type: lpx`, `lpx.buildId`, and the required [agent and conductor roles](../../reference/kubernetes-api/dynamo-component-deployment.mdx#spec-reference). For LP20 and LP30, role templates own images, commands or image entrypoints, runtime arguments and environment, init containers, probes, runtime storage, and security contexts. - -Configure Nova in the conductor template: - -| Pipeline | Model environment bindings | Resolved-partition directory | -| --- | --- | --- | -| Single | `NOVA_MODEL_PATH=$(LPX_MODEL_PATH)` | `NOVA_RESOLVED_PARTITIONS_DIR` | -| `NOVA_PIPELINE_TYPE=SpecDecode` | `NOVA_DRAFT_MODEL_PATH=$(LPX_DRAFT_MODEL_PATH)`, `NOVA_TARGET_MODEL_PATH=$(LPX_TARGET_MODEL_PATH)` | `NOVA_DRAFT_RESOLVED_PARTITIONS_DIR`, `NOVA_TARGET_RESOLVED_PARTITIONS_DIR` | - -Dynamo resolves these model paths from each `buildId` and the model-storage mount; templates need no registry or cache-layout paths. Mount the generated `config` volume at `/configs` and set the resolved-partition variables above to `/configs`. Dynamo retains this ConfigMap for selected partition files; it generates no Nova model or datacenter TOML. Nova reads model defaults from the build manifests. Put runtime overrides, such as `NOVA_AGENT_CONNECT_TIMEOUT` and `NOVA_AGENT_SETUP_TIMEOUT`, in the template. - -Include `--allocation` and `$(LPX_ALLOCATION)` in Nova's arguments to consume the operator's allocation. All LPX components share one LPX deployment and PodCliqueSet, separate from ordinary components. - For per-backend setup and tuning, see [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/overview.md), [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/overview.md), and [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/overview.md). > [!NOTE] diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 5311d754fbb4..c64db8696c60 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -104,8 +104,6 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Native Pod template for this role. LPX-only; required for every declared role. Each template requires a `main` container with an explicit image. Agent and conductor templates are independent. - The template owns runtime startup and configuration for LP20 and LP30. See [Deploy with DGD](../../kubernetes/model-deployment/deploy-with-dgd.md) for Nova inputs and the operator's model-path, partition, and allocation bindings. - core/v1.PodTemplateSpec From 709ef36df90039c379916ff51e4fc66f10031e00 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Tue, 22 Sep 2026 07:12:29 +0000 Subject: [PATCH 105/196] fix(operator): reject absolute builds with a configured LPX registry Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller_test.go | 2 +- .../internal/dynamo/lpx/build_snapshot_gcs_test.go | 11 ++++++----- .../internal/dynamo/lpx/build_snapshot_test.go | 2 +- .../internal/dynamo/lpx/compiler_fixture_test.go | 2 +- deploy/operator/internal/dynamo/lpx/model_registry.go | 7 +++++-- .../internal/dynamo/lpx/model_registry_test.go | 8 ++++++++ .../kubernetes-api/dynamo-component-deployment.mdx | 2 ++ 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index ebef115b8822..ff4d5c8ebedb 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1392,7 +1392,7 @@ func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent } - registry, err := lpx.NewModelRegistry("gs://bucket/registry", nil) + registry, err := lpx.NewModelRegistry("", nil) require.NoError(t, err) observedRegistry := &fakeModelDownloadRegistry{ModelRegistry: registry} child := newLPXTestDeployment(t, source) diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go index bb45982788c9..c4909d2f98fa 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_gcs_test.go @@ -94,12 +94,13 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { shortCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() for _, testCase := range []struct { - ref string - ctx context.Context + registryURL string + ref string + ctx context.Context }{ - {ref: "model/build", ctx: t.Context()}, + {registryURL: "gs://bucket/registry", ref: "model/build", ctx: t.Context()}, {ref: "gs://bucket/registry/model/build", ctx: t.Context()}, - {ref: "model/build", ctx: shortCtx}, + {registryURL: "gs://bucket/registry", ref: "model/build", ctx: shortCtx}, } { t.Run(testCase.ref, func(t *testing.T) { t.Log("Create an independent client and registry for this reference form") @@ -110,7 +111,7 @@ func TestGCSModelRegistrySnapshotUsesManifestV2FromModelExpress(t *testing.T) { }, list: &modelpb.ModelFileList{Files: []*modelpb.ModelFileInfo{{RelativePath: gbuildManifestV2CapnpFile}}}, } - registry, err := NewModelRegistry("gs://bucket/registry", client) + registry, err := NewModelRegistry(testCase.registryURL, client) require.NoError(t, err) t.Log("Normalize the manifest and retain the current registry locator") diff --git a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go index b90f0be39709..0e2caa092389 100644 --- a/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go +++ b/deploy/operator/internal/dynamo/lpx/build_snapshot_test.go @@ -105,7 +105,7 @@ func TestAcquireBuildSnapshotRejectsLegacyCompilerMetadata(t *testing.T) { } { require.NoError(t, os.WriteFile(filepath.Join(buildDir, path), []byte(`{}`), 0o600)) } - registry, err := NewModelRegistry(buildDir, nil) + registry, err := NewModelRegistry("", nil) require.NoError(t, err) t.Log("Require revision-2 binary compiler metadata without fallback") diff --git a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go index fe17aef4fa15..868297720a85 100644 --- a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go @@ -80,7 +80,7 @@ func testLPXPodTemplate(image string) *corev1.PodTemplateSpec { func acquireTestSnapshot(t *testing.T, buildDir string) *BuildSnapshot { t.Helper() - registry, err := NewModelRegistry(buildDir, nil) + registry, err := NewModelRegistry("", nil) require.NoError(t, err) snapshot, err := registry.AcquireBuildSnapshot(t.Context(), buildDir) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go index 0b72b79cc53f..bb36311726da 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -121,11 +121,14 @@ func (r *defaultModelRegistry) EnsureDownloaded(ctx context.Context, buildURL ur } } -// BuildURL resolves id against the configured registry URL. The receiver must -// be non-nil and is not mutated. +// BuildURL resolves id against the configured registry URL. Absolute references +// require an unconfigured registry. The receiver must be non-nil and is not mutated. func (r *defaultModelRegistry) BuildURL(id string) (*url.URL, error) { refURL, err := parseBuildRef(id) if err == nil { + if r.registryURL != nil { + return nil, fmt.Errorf("build ID %q must be relative when model registry URL is configured", id) + } return refURL, nil } if !errors.Is(err, errRelativeBuildRef) { diff --git a/deploy/operator/internal/dynamo/lpx/model_registry_test.go b/deploy/operator/internal/dynamo/lpx/model_registry_test.go index 3b3967baf391..4e4f97344b03 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry_test.go @@ -44,6 +44,14 @@ func TestRegistryBuildURL(t *testing.T) { {name: "relative id with gcs registry", registryURL: "gs://bucket/registry", id: "model/build", want: "gs://bucket/registry/model/build"}, {name: "relative id with file registry", registryURL: "file:///registry", id: "model/build", want: "file:///registry/model/build"}, {name: "absolute path", id: "/models/build", want: "file:///models/build"}, + {name: "gcs url with gcs registry", registryURL: "gs://bucket/registry", id: "gs://other-bucket/build", wantErrMessage: `build ID "gs://other-bucket/build" must be relative when model registry URL is configured`}, + {name: "file url with gcs registry", registryURL: "gs://bucket/registry", id: "file:///models/build", wantErrMessage: `build ID "file:///models/build" must be relative when model registry URL is configured`}, + {name: "absolute path with gcs registry", registryURL: "gs://bucket/registry", id: "/models/build", wantErrMessage: `build ID "/models/build" must be relative when model registry URL is configured`}, + {name: "gcs url with file registry", registryURL: "file:///registry", id: "gs://bucket/build", wantErrMessage: `build ID "gs://bucket/build" must be relative when model registry URL is configured`}, + {name: "file url with file registry", registryURL: "file:///registry", id: "file:///models/build", wantErrMessage: `build ID "file:///models/build" must be relative when model registry URL is configured`}, + {name: "absolute path with file registry", registryURL: "file:///registry", id: "/models/build", wantErrMessage: `build ID "/models/build" must be relative when model registry URL is configured`}, + {name: "absolute url within registry", registryURL: "gs://bucket/registry", id: "gs://bucket/registry/build", wantErrMessage: `build ID "gs://bucket/registry/build" must be relative when model registry URL is configured`}, + {name: "padded absolute path", registryURL: "file:///registry", id: " /models/build ", wantErrMessage: `build ID " /models/build " must be relative when model registry URL is configured`}, {name: "relative id escaping registry", registryURL: "gs://bucket/registry", id: "../model/build", wantErrMessage: "build ID \"../model/build\" must remain within the configured model registry"}, {name: "relative id without registry", id: "model/build", wantErrMessage: "model registry URL is not configured"}, {name: "empty id", wantErrMessage: "empty ref"}, diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index c33ba1a22587..799b269ba98a 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -64,6 +64,8 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c DGD-only compiled-model configuration. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). + + When the operator's `lpx.modelRegistryURL` is configured, `buildId` must be relative to that registry; absolute paths and URLs require an empty registry URL. From 268fd3870b4f69ab88a537ab4c28fee6aa40260a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Tue, 22 Sep 2026 07:28:52 +0000 Subject: [PATCH 106/196] fix(operator): skip LPX in shared checkpoint reconciliation Signed-off-by: Andrew Paprotsky --- .../controller/dgd_checkpoints_reconciler.go | 2 +- .../controller/dgd_checkpoints_reconciler_test.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go b/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go index 8f25b14e0139..15aa2d29ebf4 100644 --- a/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go +++ b/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go @@ -99,7 +99,7 @@ func (r *dgdCheckpointsReconciler) Reconcile( component := &dgd.Spec.Components[i] componentName := component.ComponentName checkpointConfig := dynamo.GetCheckpoint(component) - if checkpointConfig == nil { + if checkpointConfig == nil || component.IsLPX() { continue } if !r.runtimeConfig.Gate.Enabled(features.Checkpoint) { diff --git a/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go b/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go index be2c2a5cf559..d987c1fbcc38 100644 --- a/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go @@ -921,6 +921,21 @@ func TestDGDCheckpointsReconciler_ExplicitRestoreDoesNotDependOnActiveWorkerHash assert.True(t, info.Ready) require.NotNil(t, info.NativeSnapshot) assert.Equal(t, referenced.UID, info.NativeSnapshot.UID) + + t.Log("Verify automatic and referenced LPX checkpoint settings do not block the following worker") + dgd.Spec.Components = append([]v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "lpx", + ComponentType: v1beta1.ComponentTypeLPX, + Experimental: &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ + Enabled: true, + }}, + }}, dgd.Spec.Components...) + for _, checkpointRef := range []*string{nil, ptr.To("lpx-snapshot")} { + dgd.Spec.Components[0].Experimental.Checkpoint.CheckpointRef = checkpointRef + withLPX, err := newTestDGDCheckpointsReconciler(reconciler).Reconcile(ctx, dgd) + assert.NoError(t, err) + assert.Equal(t, result, withLPX) + } } func TestDGDCheckpointsReconciler_ExplicitRestoreIsPortableAcrossDGDIdentity(t *testing.T) { From c1ba618875cc62e0acfbf24a10041654e466c24e Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Tue, 22 Sep 2026 07:50:51 +0000 Subject: [PATCH 107/196] fix(operator): grant Grove owner-reference permission Signed-off-by: Andrew Paprotsky --- .../platform/components/operator/files/role.yaml | 6 ++++++ .../helm/charts/platform/tests/lpx_config_test.yaml | 12 +++++++++++- .../operator/internal/controller/lpx/controller.go | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 857338198f89..ba3393b16969 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -196,6 +196,12 @@ rules: - patch - update - watch +- apiGroups: + - grove.io + resources: + - podcliquesets/finalizers + verbs: + - update - apiGroups: - inference.networking.k8s.io resources: diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index 87f2f502f3d6..dcd0065fad39 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -49,7 +49,7 @@ tests: path: data["config.yaml"] pattern: '(?m)^lpu:' - - it: renders the exact inert LpuPipelineRequest manager grant + - it: renders the exact inert LpuPipelineRequest and Grove owner grants template: charts/dynamo-operator/templates/manager-rbac.yaml asserts: - contains: @@ -66,3 +66,13 @@ tests: - list - watch documentIndex: 0 + - contains: + path: rules + content: + apiGroups: + - grove.io + resources: + - podcliquesets/finalizers + verbs: + - update + documentIndex: 0 diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index bc90521d618d..3623da1b7889 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -125,6 +125,7 @@ func (r *graphReconciler) unavailableReason() string { // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update // +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups=grove.io,resources=podcliquesets/finalizers,verbs=update func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) { deployment := &v1alpha1.LPXGraphDeployment{} From cbe156469b649782f0201ff8370973b14bea1201 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Tue, 22 Sep 2026 08:19:16 +0000 Subject: [PATCH 108/196] fix(operator): avoid idle LPX deadline polling Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/attempt_deadline.go | 7 +++++-- .../controller/lpx/attempt_deadline_test.go | 4 +++- .../internal/controller/lpx/controller.go | 6 +++++- .../controller/lpx/deadline_retirement_test.go | 15 ++++++++++----- .../internal/controller/lpx/lifecycle_test.go | 6 +++++- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go index 09e24b667f97..1c92909bee92 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline.go @@ -136,9 +136,12 @@ func (r *graphReconciler) reconcileLPXRequestDeadlines( return nil, nextDeadline, nil } - // Lower Grove before deleting the affected engine suffix; convergence is asynchronous. - // Requeue even after the last recorded UID disappears so an authorized retry can proceed. + // Request and Grove watches drive cleanup while expired UIDs remain. retirementErr := r.retireExpiredLPXRequests(ctx, deployment, source, pcs, requests, expired) + if len(deployment.Status.ExpiredRequestUIDs) > 0 { + return &lpxDeadlineExceeded{}, nextDeadline, retirementErr + } + // Clearing the last UID only changes status; explicitly wake an authorized retry. return &lpxDeadlineExceeded{requeueAfter: lpxRetirementRequeueAfter}, nextDeadline, retirementErr } diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go index 39deb9424f04..2482e50d3bc9 100644 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go @@ -52,6 +52,7 @@ func TestLPXDeadlineUsesEachSchedulingCycleStart(t *testing.T) { require.NoError(t, err) require.IsType(t, &lpxDeadlineExceeded{}, classification) require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) + require.Zero(t, projectLPXLifecycleStatus(classification), "the finalizer watch needs no additional poll") } func TestLPXDeadlineIgnoresRequestsOutsideCurrentPublication(t *testing.T) { @@ -226,10 +227,11 @@ func TestLPXDeadlineRetainsOnlyRecordedRequestUIDs(t *testing.T) { request = getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name) classification, _, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, nil, nil) require.NoError(t, err) - require.Positive(t, projectLPXLifecycleStatus(classification).RequeueAfter) if uid == "recorded" { + require.Zero(t, projectLPXLifecycleStatus(classification)) require.Equal(t, []types.UID{uid}, dgd.Status.ExpiredRequestUIDs) } else { + require.Positive(t, projectLPXLifecycleStatus(classification).RequeueAfter) require.Nil(t, dgd.Status.ExpiredRequestUIDs) } require.Equal(t, request, getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name)) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 3623da1b7889..12c6ac168e20 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -261,7 +261,11 @@ func (r *graphReconciler) completeReconcile( ) deadlineAt, err = wake, errors.Join(err, deadlineErr) if classification != nil { - state, result = lpxResult(classification), projectLPXLifecycleStatus(classification) + state = lpxResult(classification) + // Preserve independent work such as polling an unfinished model download. + if deadlineResult := projectLPXLifecycleStatus(classification); deadlineResult.RequeueAfter > 0 { + result = deadlineResult + } deadlineExceeded, recordDeadlineFailure = lpxDeadlineFlags(classification) } } diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go index b50f20ff3f27..919ebafb0bdb 100644 --- a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go +++ b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go @@ -16,7 +16,9 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) @@ -134,13 +136,16 @@ func TestLPXExpiredInteriorReplicaDoesNotScaleDown(t *testing.T) { createLPXTestObjects(t, ctx, r.Client, objects...) publishSelectedLPXForTest(t, ctx, r, child, desired) group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - pcs := findLPXTestPodCliqueSet(t, objects) expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) + child.Status.ExpiredRequestUIDs = []types.UID{expired.UID} + require.NoError(t, r.Status().Update(ctx, child)) - t.Log("Leave the PCSG, its PodCliques, and the interior LPR untouched") - require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) + t.Log("Wait for watched changes without polling or changing the live engines") + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Equal(t, []types.UID{expired.UID}, child.Status.ExpiredRequestUIDs) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(3), group.Spec.Replicas) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go index cd60bce0fa41..6771b8a92060 100644 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ b/deploy/operator/internal/controller/lpx/lifecycle_test.go @@ -219,6 +219,7 @@ func TestSelectedLPXSuccessorBuildDownloadsBeforePublication(t *testing.T) { t.Log("Publish the previous build before switching to an uncached successor") ctx := t.Context() child, source, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, baseRegistry, ctx, child, source) objects := lpxMaterializedObjects(t, r, child, source, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -235,13 +236,15 @@ func TestSelectedLPXSuccessorBuildDownloadsBeforePublication(t *testing.T) { source.Generation++ require.NoError(t, r.Update(ctx, source)) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Status.ExpiredRequestUIDs = []types.UID{previous.UID} + require.NoError(t, r.Status().Update(ctx, child)) child.Generation++ var err error child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") require.NoError(t, err) require.NoError(t, r.Update(ctx, child)) - t.Log("Start downloading despite snapshot unavailability, preserving the existing request") + t.Log("Keep the download poll despite an interior expiry, preserving the existing request") request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} result, err := r.Reconcile(ctx, request) require.NoError(t, err) @@ -250,6 +253,7 @@ func TestSelectedLPXSuccessorBuildDownloadsBeforePublication(t *testing.T) { require.Equal(t, previous, getLPXRequest(t, ctx, r.Client, previous.Namespace, previous.Name)) require.NoError(t, r.Get(ctx, request.NamespacedName, child)) require.Equal(t, child.Generation, child.Status.ObservedGeneration) + require.Equal(t, []types.UID{previous.UID}, child.Status.ExpiredRequestUIDs) t.Log("Resolve the successor after download completes on the next reconciliation") _, err = r.Reconcile(ctx, request) From 281cfd37cc6095d044bbd62c26ff7047c2ce7eeb Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Mon, 21 Sep 2026 13:32:35 -0700 Subject: [PATCH 109/196] refactor(lpx)!: remove placement status from LPXGD Signed-off-by: Steven Davidovitz --- .../conversion_field_coverage_test.go | 2 - .../dynamographdeployment_conversion.go | 32 +++----- ...amographdeployment_conversion_bugs_test.go | 28 ------- .../dynamographdeployment_conversion_test.go | 10 +-- .../api/v1alpha1/lpx_conversion_test.go | 5 -- .../api/v1alpha1/lpxgraphdeployment_types.go | 3 - .../api/v1alpha1/zz_generated.deepcopy.go | 5 -- deploy/operator/api/v1beta1/common.go | 8 +- deploy/operator/api/v1beta1/lpx_types.go | 3 - .../api/v1beta1/zz_generated.deepcopy.go | 5 -- .../nvidia.com_dynamographdeployments.yaml | 50 ------------ .../bases/nvidia.com_lpxgraphdeployments.yaml | 25 ------ .../controller/dgd_grove_lpx_status.go | 22 ----- .../internal/controller/dgd_grove_program.go | 8 -- .../dgd_lpx_handoff_envtest_test.go | 6 -- .../controller/dgd_lpx_handoff_test.go | 81 +++++-------------- .../controller/lpx/modeldownload_test.go | 7 +- .../additional-resources/api-reference-k8s.md | 10 +-- .../dynamo-graph-deployment.mdx | 5 +- .../kubernetes-api/full-api-reference.mdx | 16 ++-- 20 files changed, 49 insertions(+), 282 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 784fed49b73a..6ccb1fa01ca9 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -191,8 +191,6 @@ DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions DynamoGraphDeploymentStatus.lpx.modelDownload.builds DynamoGraphDeploymentStatus.lpx.modelDownload.lastCheckedAt -DynamoGraphDeploymentStatus.lpx.placement.score -DynamoGraphDeploymentStatus.lpx.placement.state DynamoGraphDeploymentStatus.observedGeneration DynamoGraphDeploymentStatus.placement.score DynamoGraphDeploymentStatus.placement.state diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index e8e98e8714c0..2b46041df31e 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -655,8 +655,12 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds dst.ObservedGeneration = src.ObservedGeneration dst.State = v1beta1.DGDState(src.State) if src.Placement != nil { - dst.Placement = &v1beta1.PlacementStatus{} - ConvertFromPlacementStatus(src.Placement, dst.Placement) + dst.Placement = &v1beta1.PlacementStatus{ + State: v1beta1.PlacementScoreState(src.Placement.State), + } + if src.Placement.Score != nil { + dst.Placement.Score = ptr.To(*src.Placement.Score) + } } else { dst.Placement = nil } @@ -699,8 +703,12 @@ func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStat dst.ObservedGeneration = src.ObservedGeneration dst.State = DGDState(src.State) if src.Placement != nil { - dst.Placement = &PlacementStatus{} - ConvertToPlacementStatus(src.Placement, dst.Placement) + dst.Placement = &PlacementStatus{ + State: PlacementScoreState(src.Placement.State), + } + if src.Placement.Score != nil { + dst.Placement.Score = ptr.To(*src.Placement.Score) + } } else { dst.Placement = nil } @@ -737,22 +745,6 @@ func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStat } } -// ConvertFromPlacementStatus converts placement status from v1alpha1 to v1beta1. -func ConvertFromPlacementStatus(src *PlacementStatus, dst *v1beta1.PlacementStatus) { - *dst = v1beta1.PlacementStatus{State: v1beta1.PlacementScoreState(src.State)} - if src.Score != nil { - dst.Score = ptr.To(*src.Score) - } -} - -// ConvertToPlacementStatus converts placement status from v1beta1 to v1alpha1. -func ConvertToPlacementStatus(src *v1beta1.PlacementStatus, dst *PlacementStatus) { - *dst = PlacementStatus{State: PlacementScoreState(src.State)} - if src.Score != nil { - dst.Score = ptr.To(*src.Score) - } -} - // ConvertFromRestartStatus converts restart status from v1alpha1 to v1beta1. func ConvertFromRestartStatus(src *RestartStatus, dst *v1beta1.RestartStatus) { *dst = v1beta1.RestartStatus{ diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go index 94e1a1a3fb3e..d83afecf0b65 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_bugs_test.go @@ -24,38 +24,10 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" ) -func TestDGDPlacementStatusSurvivesConversionAlongsideLPX(t *testing.T) { - t.Log("Define distinct public and LPX-owned placement values on the hub") - src := &v1beta1.DynamoGraphDeployment{ - Status: v1beta1.DynamoGraphDeploymentStatus{ - Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, - LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStatePartial}, - }, - }, - } - - t.Log("Convert through the served alpha version and back") - alpha := &DynamoGraphDeployment{} - if err := alpha.ConvertFrom(src); err != nil { - t.Fatal(err) - } - got := &v1beta1.DynamoGraphDeployment{} - if err := alpha.ConvertTo(got); err != nil { - t.Fatal(err) - } - - t.Log("Preserve both paths independently without dropping or conflating the public field") - if diff := cmp.Diff(src.Status, got.Status); diff != "" { - t.Fatalf("status round-trip mismatch (-want +got):\n%s", diff) - } -} - func TestBugDGD_IntermediateHubAddsMainOnlyPodTemplateRoundTrips(t *testing.T) { alpha := &DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "main-only-pod-template", Namespace: "ns"}, diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index 664e0aeda274..5749987055f5 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -949,7 +949,7 @@ func TestDGD_FromV1alpha1_SubComponentType(t *testing.T) { // TestDGD_RoundTrip_Status exercises every populated Status sub-struct so that // the ConvertTo / ConvertFrom status paths are covered (conditions, services -// map, restart, checkpoints, LPX placement, rollingUpdate). +// map, restart, checkpoints, rollingUpdate). func TestDGD_RoundTrip_Status(t *testing.T) { now := metav1.NewTime(metav1.Now().Rfc3339Copy().Time) later := metav1.NewTime(now.Time.Add(60 * time.Second)) @@ -959,15 +959,9 @@ func TestDGD_RoundTrip_Status(t *testing.T) { ObservedGeneration: 7, State: v1beta1.DGDStateSuccessful, Placement: &v1beta1.PlacementStatus{ - Score: ptr.To(0.92), + Score: ptr.To(0.87), State: v1beta1.PlacementScoreStateReported, }, - LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - Placement: &v1beta1.PlacementStatus{ - Score: ptr.To(0.87), - State: v1beta1.PlacementScoreStateReported, - }, - }, Conditions: []metav1.Condition{ { Type: "Ready", diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index f09843f2c071..230844e518a0 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -94,7 +94,6 @@ func TestDynamoGraphDeploymentRoleShapesRoundTrip(t *testing.T) { func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { checkedAt := metav1.NewTime(time.Unix(1_800_000_000, 0)) - placementScore := 0.75 src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "lpx", Namespace: "ns"}, Spec: v1beta1.DynamoGraphDeploymentSpec{ @@ -117,10 +116,6 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { Builds: []string{"gs://models/draft", "gs://models/target"}, LastCheckedAt: &checkedAt, }, - Placement: &v1beta1.PlacementStatus{ - Score: &placementScore, - State: v1beta1.PlacementScoreStateReported, - }, }, }, } diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index 568f53c4c399..d5ccc36ba65d 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -42,9 +42,6 @@ type LPXGraphDeploymentStatus struct { // modelDownload retains the existing remote-build download progress. // +optional ModelDownload *v1beta1.ModelDownloadStatus `json:"modelDownload,omitempty"` - // placement retains scheduler placement progress. - // +optional - Placement *v1beta1.PlacementStatus `json:"placement,omitempty"` } // +kubebuilder:object:root=true diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 9de343728137..b6025b327dbb 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1470,11 +1470,6 @@ func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) *out = new(v1beta1.ModelDownloadStatus) (*in).DeepCopyInto(*out) } - if in.Placement != nil { - in, out := &in.Placement, &out.Placement - *out = new(v1beta1.PlacementStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXGraphDeploymentStatus. diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index e3fea1966413..a592501cd9df 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -711,10 +711,10 @@ const ( // status object so future placement signals (e.g. scheduler contract version, // last-report timestamp, per-unit reports) can be added without a schema break. // -// The generic score source is an open question in DEP #10064 (Grove mirror, -// typed Grove scheduler API, or unstructured provider). Until one is selected, -// the DGD controller writes this field only as a compatibility projection of -// current LPX placement. +// The score source is an open question in DEP #10064 (Grove mirror, typed Grove +// scheduler API, or unstructured provider). Until a source is selected and +// implemented, the DGD controller does not write this field; the schema and +// conversion are landed here so downstream consumers can rely on the shape. type PlacementStatus struct { // score is the DGD-level scheduler placement score aggregated from // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index cb4878a09aac..517ae757a11d 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -28,9 +28,6 @@ type DynamoGraphDeploymentLPXStatus struct { // modelDownload contains the model download status for remote builds. // +optional ModelDownload *ModelDownloadStatus `json:"modelDownload,omitempty"` - // placement contains LPX scheduling progress and placement signals. - // +optional - Placement *PlacementStatus `json:"placement,omitempty"` } // ModelDownloadStatus contains the status of remote LPU model downloads. diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index b936570795e3..bec6a9fb5e21 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -516,11 +516,6 @@ func (in *DynamoGraphDeploymentLPXStatus) DeepCopyInto(out *DynamoGraphDeploymen *out = new(ModelDownloadStatus) (*in).DeepCopyInto(*out) } - if in.Placement != nil { - in, out := &in.Placement, &out.Placement - *out = new(PlacementStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentLPXStatus. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 9784eae672d6..4432bc77ea7d 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20273,31 +20273,6 @@ spec: format: date-time type: string type: object - placement: - description: placement contains LPX scheduling progress and placement signals. - properties: - score: - description: |- - score is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation - uses the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - state: - description: |- - state indicates placement score reporting state. See PlacementScoreState - for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string - type: object type: object observedGeneration: description: ObservedGeneration is the most recent generation observed by the controller. @@ -38134,31 +38109,6 @@ spec: format: date-time type: string type: object - placement: - description: placement contains LPX scheduling progress and placement signals. - properties: - score: - description: |- - score is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation - uses the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - state: - description: |- - state indicates placement score reporting state. See PlacementScoreState - for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string - type: object type: object observedGeneration: description: observedGeneration is the most recent generation observed by the controller. diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index 73553a0dfefb..27325ceb12b1 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -267,31 +267,6 @@ spec: Every spec.inputRevision change advances the child generation. format: int64 type: integer - placement: - description: placement retains scheduler placement progress. - properties: - score: - description: |- - score is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation - uses the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - state: - description: |- - state indicates placement score reporting state. See PlacementScoreState - for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string - type: object type: object required: - spec diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 63318fa8c0df..ceffe97fea49 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -7,7 +7,6 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -41,7 +40,6 @@ func mergeLPXChildStatus( if observed { status = &v1beta1.DynamoGraphDeploymentLPXStatus{ ModelDownload: child.Status.ModelDownload.DeepCopy(), - Placement: child.Status.Placement.DeepCopy(), } for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { @@ -76,23 +74,3 @@ func mergeLPXChildStatus( } return ordinary, status } - -// lpxPlacementProjection retains an independent ordinary-DGD placement value, -// but mirrors current LPX placement and clears stale mirrors. -func lpxPlacementProjection( - source *v1beta1.DynamoGraphDeployment, - current *v1beta1.PlacementStatus, - previousLPX *v1beta1.DynamoGraphDeploymentLPXStatus, - nextLPX *v1beta1.DynamoGraphDeploymentLPXStatus, -) *v1beta1.PlacementStatus { - if nextLPX != nil { - return nextLPX.Placement - } - if len(lpx.Components(source)) > 0 { - return nil - } - if previousLPX != nil && apiequality.Semantic.DeepEqual(current, previousLPX.Placement) { - return nil - } - return current -} diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index a7516ca1b072..a5a7dd71e7c8 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -125,7 +125,6 @@ func (p *groveProgram) Reconcile( } // Do not keep reporting retired LPX capacity when ordinary reconciliation fails. - programResult.Status.Placement = lpxPlacementProjection(req.DGD, programResult.Status.Placement, programResult.Status.LPX, nil) programResult.Status.LPX = nil for name := range programResult.Status.Components { if req.DGD.GetComponentByName(name) == nil { @@ -195,14 +194,7 @@ func (p *groveProgram) Reconcile( } } - previousLPX := programResult.Status.LPX result, programResult.Status.LPX = mergeLPXChildStatus(req.DGD, child, result) - programResult.Status.Placement = lpxPlacementProjection( - req.DGD, - programResult.Status.Placement, - previousLPX, - programResult.Status.LPX, - ) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) if child != nil && !child.DeletionTimestamp.IsZero() { programResult.RequeueAfter = 5 * time.Second diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 97b4f4f2dfd8..88ec3fb34a9b 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -388,10 +388,4 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { storedSource := &v1beta1.DynamoGraphDeployment{} require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) require.Equal(t, source.Status, storedSource.Status) - - t.Log("Placement status persists independently of request deadlines") - stored.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} - require.NoError(t, env.Client().Status().Update(t.Context(), stored)) - require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(stored), stored)) - require.Equal(t, 0.75, *stored.Status.Placement.Score) } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index c41c4304454b..1c6719943b37 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -72,9 +72,7 @@ func projectLPXChildStatus( result *ReconcileResult, status *v1beta1.DynamoGraphDeploymentStatus, ) { - previousLPX := status.LPX *result, status.LPX = mergeLPXChildStatus(source, child, *result) - status.Placement = lpxPlacementProjection(source, status.Placement, previousLPX, status.LPX) } func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { @@ -220,20 +218,17 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), }} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"build"}} - placementScore := 0.75 - child.Status.Placement = &v1beta1.PlacementStatus{Score: &placementScore, State: v1beta1.PlacementScoreStateReported} + source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4)), Ready: true}}} - status := v1beta1.DynamoGraphDeploymentStatus{} + status := *source.Status.DeepCopy() projectLPXChildStatus(source, child, &result, &status) require.Equal(t, v1beta1.DGDStateSuccessful, result.State) require.Len(t, result.ComponentStatus, 2) require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) - require.Equal(t, status.LPX.Placement, status.Placement) + require.Equal(t, source.Status.Placement, status.Placement) status.LPX.ModelDownload.Builds[0] = "parent-copy" - *status.LPX.Placement.Score = 0.25 require.Equal(t, "build", child.Status.ModelDownload.Builds[0]) - require.Equal(t, placementScore, *child.Status.Placement.Score) t.Log("Reject missing or stale children and stale Ready conditions") for _, failure := range []string{"missing-child", "generation", "condition"} { @@ -250,12 +245,11 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { previous := status projectLPXChildStatus(source, stale, &pending, &previous) require.Equal(t, v1beta1.DGDStatePending, pending.State, failure) + require.Equal(t, source.Status.Placement, previous.Placement, failure) if failure == "missing-child" || failure == "generation" { require.Nil(t, previous.LPX, failure) - require.Nil(t, previous.Placement, failure) } else { require.NotNil(t, previous.LPX, failure) - require.Equal(t, previous.LPX.Placement, previous.Placement, failure) } } @@ -263,14 +257,14 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { partial := child.Status.Components["lpx"] partial.AvailableReplicas, partial.Ready = ptr.To(int32(2)), false child.Status.Components["lpx"] = partial - child.Status.ModelDownload, child.Status.Placement = nil, nil + child.Status.ModelDownload = nil result.Reason = "CheckpointReady" meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "LPXRetiring"}) projectLPXChildStatus(source, child, &result, &status) require.Equal(t, v1beta1.DGDStatePending, result.State) require.Equal(t, Reason("LPXRetiring"), result.Reason) require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{}, status.LPX) - require.Nil(t, status.Placement) + require.Equal(t, source.Status.Placement, status.Placement) require.Equal(t, partial, result.ComponentStatus["lpx"]) source.Status.Components = result.ComponentStatus request := &v1beta1.DynamoGraphDeploymentRequest{} @@ -293,7 +287,7 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { source.Spec.Components = nil projectLPXChildStatus(source, child, &result, &status) require.Nil(t, status.LPX) - require.Nil(t, status.Placement) + require.Equal(t, source.Status.Placement, status.Placement) } func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { @@ -313,11 +307,10 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"stale-build"}} - child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "PublicationDenied", Message: "Check the namespace quota"}} result := ReconcileResult{State: v1beta1.DGDStateSuccessful} status := v1beta1.DynamoGraphDeploymentStatus{LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: child.Status.ModelDownload.DeepCopy(), Placement: child.Status.Placement.DeepCopy(), + ModelDownload: child.Status.ModelDownload.DeepCopy(), }} switch test.name { case "observed-result": @@ -349,7 +342,6 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { if test.name == "observed-result" { require.NotNil(t, status.LPX) require.NotNil(t, status.LPX.ModelDownload) - require.NotNil(t, status.LPX.Placement) require.Equal(t, int32(1), result.ComponentStatus["lpx"].Replicas) t.Log("A retained ordinary rollout cannot hide the child failure in final DGD status") @@ -430,8 +422,8 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { } } -func TestLPXHandoffOrdinaryScalingPreservesPlacement(t *testing.T) { - t.Log("Seed the shared child's current readiness and placement") +func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { + t.Log("Seed the shared child's current readiness") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-specdecode") source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1)), @@ -439,7 +431,6 @@ func TestLPXHandoffOrdinaryScalingPreservesPlacement(t *testing.T) { ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:0"}}}}, }) - child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Status.ObservedGeneration = child.Generation child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ "lpx": {Replicas: 1, AvailableReplicas: ptr.To(int32(1)), Ready: true}, @@ -466,14 +457,12 @@ func TestLPXHandoffOrdinaryScalingPreservesPlacement(t *testing.T) { require.Equal(t, v1beta1.DGDStateSuccessful, projected.Status.State) require.True(t, meta.IsStatusConditionTrue(projected.Status.Conditions, "Ready")) require.Equal(t, source.Generation, projected.Status.ObservedGeneration) - require.Equal(t, beforeChild.Status.Placement, current.Status.Placement) - require.Equal(t, beforeChild.Status.Placement, projected.Status.LPX.Placement) require.Equal(t, beforeSource, source) source.Status = projected.Status } require.Equal(t, int64(8), source.Generation) - t.Log("Scale prefill without changing the child or its placement") + t.Log("Scale prefill without changing the child or its readiness") source.GetComponentByName("prefill").Replicas = ptr.To(int32(4)) source.Generation++ current, err := handoff.Reconcile(t.Context(), source) @@ -628,14 +617,12 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { for _, test := range []struct { - name string - retainLPX bool - childGone bool - independentPlacement bool + name string + retainLPX bool + childGone bool }{ {name: "remove"}, {name: "retain", retainLPX: true}, - {name: "independent placement", independentPlacement: true}, {name: "pending child already gone", childGone: true}, } { t.Run(test.name, func(t *testing.T) { @@ -658,17 +645,14 @@ func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { source.Generation++ require.NoError(t, kube.Update(t.Context(), source)) source.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ - Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92)}, + ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}}, } - source.Status.Placement = source.Status.LPX.Placement.DeepCopy() + source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} source.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ "lpx": {Replicas: 1}, "prefill": {Replicas: 2}, } - if test.independentPlacement { - source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.5)} - } if test.childGone { - source.Status.LPX, source.Status.Placement = nil, nil + source.Status.LPX = nil source.Status.Components["lpx"] = v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup} } previousStatus := source.Status.DeepCopy() @@ -698,11 +682,7 @@ func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { require.Equal(t, source.Status.LPX, result.Status.LPX) require.Equal(t, source.Status.Components, result.Status.Components) } - if test.retainLPX || test.independentPlacement { - require.Equal(t, source.Status.Placement, result.Status.Placement) - } else { - require.Nil(t, result.Status.Placement) - } + require.Equal(t, source.Status.Placement, result.Status.Placement) }) } } @@ -864,7 +844,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { } func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { - t.Log("Seed an ordinary DGD with obsolete LPX-owned download and placement status") + t.Log("Seed an ordinary DGD with obsolete LPX download status and independent placement") checkedAt := metav1.Unix(1, 0) source := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "download-status", Namespace: "default", Generation: 1}, @@ -873,7 +853,6 @@ func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}, LastCheckedAt: &checkedAt}, - Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported}, }, }, } @@ -897,25 +876,3 @@ func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { require.Nil(t, updated.Status.LPX) require.Equal(t, source.Status.Placement, updated.Status.Placement) } - -func TestOrdinaryDGDClearsMirroredLPXPlacement(t *testing.T) { - t.Log("Seed an ordinary DGD with the same compatibility projection at both placement paths") - placement := &v1beta1.PlacementStatus{ - Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported, - } - source := &v1beta1.DynamoGraphDeployment{ - Status: v1beta1.DynamoGraphDeploymentStatus{ - Placement: placement.DeepCopy(), - LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{Placement: placement.DeepCopy()}, - }, - } - status := source.Status.DeepCopy() - result := &ReconcileResult{} - - t.Log("Project status after the obsolete LPX child is absent") - projectLPXChildStatus(source, nil, result, status) - - t.Log("Clear both the obsolete nested status and its matching top-level projection") - require.Nil(t, status.LPX) - require.Nil(t, status.Placement) -} diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index ae4aa0ea9f33..16f661fddc33 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -275,7 +275,7 @@ func TestModelDownloadSpecChangeRemainsFailClosed(t *testing.T) { } func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { - t.Log("A new revision retains old placement and download observations without a scheduling deadline") + t.Log("A new revision starts with old download observations and no scheduling deadline") source := newModelDownloadDGD(modelDownloadTestSecondBuildID) source.Generation++ child := newLPXTestDeployment(t, source) @@ -283,7 +283,6 @@ func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: ptr.To(metav1.Now()), } - child.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.75), State: v1beta1.PlacementScoreStateReported} child.Generation++ previousGeneration := child.Status.ObservedGeneration require.Nil(t, lpxRequestDeadlineSeconds(source)) @@ -291,16 +290,14 @@ func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { r := newLPXTestReconciler(t, registry, child, source) r.runtimeConfig.Gate = features.Gates{LPX: true} require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - previousPlacement := child.Status.Placement.DeepCopy() - t.Log("Disabled reconciliation reports the new failure without marking stale placement as observed") + t.Log("Disabled reconciliation reports the new failure without marking the new revision as observed") for range 2 { _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) require.Equal(t, previousGeneration, child.Status.ObservedGeneration) require.Nil(t, child.Status.ModelDownload) - require.Equal(t, previousPlacement, child.Status.Placement) require.Empty(t, registry.calls, "the early exit has not checked either build") failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") require.Equal(t, child.Generation, failed.ObservedGeneration) diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index faacfe9ff7d0..d3726f688d5b 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2025,7 +2025,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `modelDownload` _[ModelDownloadStatus](#modeldownloadstatus)_ | modelDownload contains the model download status for remote builds. | | Optional: \{\}
| -| `placement` _[PlacementStatus](#placementstatus)_ | placement contains LPX scheduling progress and placement signals. | | Optional: \{\}
| #### v1beta1 DynamoGraphDeploymentRequest @@ -2700,15 +2699,14 @@ PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. -The generic score source is an open question in DEP #10064 (Grove mirror, -typed Grove scheduler API, or unstructured provider). Until one is selected, -the DGD controller writes this field only as a compatibility projection of -current LPX placement. +The score source is an open question in DEP #10064 (Grove mirror, typed Grove +scheduler API, or unstructured provider). Until a source is selected and +implemented, the DGD controller does not write this field; the schema and +conversion are landed here so downstream consumers can rely on the shape. _Appears in:_ -- [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) - [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) | Field | Description | Default | Validation | diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 04b55631d726..943c27601787 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -184,16 +184,13 @@ The operator maintains observed state under `status`. - LPX download and placement observations for the current configuration. Omitted when no current LPX result exists, including ordinary-only deployments. LPX component entries and aggregate readiness remain under `status.components`, `status.state`, and `status.conditions`. + LPX model download observations for the current configuration. Omitted when no current LPX result exists, including ordinary-only deployments. LPX component entries and aggregate readiness remain under `status.components`, `status.state`, and `status.conditions`. Downloaded builds and the time they were last checked. - - Placement progress and scheduling-attempt details. The attempt's `observedGeneration` refers to this DGD's generation. - diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 7276da939187..ccacd66f6cfa 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -2941,12 +2941,6 @@ modelDownload contains the model download status for remote builds. See [ModelDownloadStatus](#modeldownloadstatus). **Validation:** Optional: \{\} - - -placement contains LPX scheduling progress and placement signals. -See [PlacementStatus](#v1beta1-placementstatus). -**Validation:** Optional: \{\} - @@ -3853,14 +3847,14 @@ PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. -The generic score source is an open question in DEP #10064 (Grove mirror, -typed Grove scheduler API, or unstructured provider). Until one is selected, -the DGD controller writes this field only as a compatibility projection of -current LPX placement. +The score source is an open question in DEP #10064 (Grove mirror, typed Grove +scheduler API, or unstructured provider). Until a source is selected and +implemented, the DGD controller does not write this field; the schema and +conversion are landed here so downstream consumers can rely on the shape. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus), [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) +**Appears in:** [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) score is the DGD-level scheduler placement score aggregated from relevant scheduler placement units. Normalized to [0.0, 1.0] where higher is better and 1.0 represents the best possible placement. Aggregation uses the minimum across placement units so the value is a worst-placement signal for the graph. Scores are only comparable across DGDs that share the same scheduler scoring contract and version. From fdbbb9810dddcafbcc67e92cc419ad2e3a6638ef Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Thu, 17 Sep 2026 14:59:22 -0700 Subject: [PATCH 110/196] refactor(lpx)!: lpx controller readability and refactor Signed-off-by: Steven Davidovitz --- deploy/operator/api/AGENTS.md | 5 + .../api/v1alpha1/lpxgraphdeployment_types.go | 25 +- .../api/v1alpha1/zz_generated.deepcopy.go | 6 - deploy/operator/cmd/crd-apply/main_test.go | 6 +- .../bases/nvidia.com_lpxgraphdeployments.yaml | 17 +- .../controller/dgd_grove_lpx_restart.go | 13 +- .../controller/dgd_grove_lpx_restart_test.go | 71 + .../controller/dgd_grove_lpx_status.go | 18 +- .../controller/dgd_grove_lpx_status_test.go | 87 + .../dgd_lpx_handoff_envtest_test.go | 72 +- .../controller/dgd_lpx_handoff_test.go | 25 +- .../internal/controller/lpx/AGENTS.md | 238 ++ .../internal/controller/lpx/CLAUDE.md | 1 + .../controller/lpx/attempt_deadline.go | 224 -- .../controller/lpx/attempt_deadline_test.go | 352 -- .../controller/lpx/component_roles_test.go | 88 - .../internal/controller/lpx/controller.go | 785 ++-- .../controller/lpx/controller_test.go | 3549 ++++++++++++----- .../controller/lpx/deadline_retirement.go | 177 - .../lpx/deadline_retirement_test.go | 291 -- .../internal/controller/lpx/eviction.go | 50 +- .../internal/controller/lpx/eviction_test.go | 136 +- .../internal/controller/lpx/lifecycle.go | 936 ----- .../internal/controller/lpx/lifecycle_test.go | 2146 ---------- .../internal/controller/lpx/modeldownload.go | 104 +- .../controller/lpx/modeldownload_test.go | 218 +- .../controller/lpx/pipeline_request.go | 219 + .../lpx/pipeline_request_deadline.go | 257 ++ .../lpx/pipeline_request_deadline_test.go | 406 ++ .../controller/lpx/pipeline_request_test.go | 374 ++ .../internal/controller/lpx/preflight.go | 80 - .../internal/controller/lpx/render.go | 75 +- .../internal/controller/lpx/render_test.go | 1518 +++---- .../internal/controller/lpx/status.go | 253 ++ .../internal/controller/lpx/status_test.go | 367 ++ .../internal/controller/lpx/testdata/dgd.yaml | 51 + .../lpx/testdata/hybrid-conductor.yaml | 20 + .../pipeline-request-ready.golden.json | 82 + .../testdata/pipeline-request-statuses.yaml | 36 + .../operator/internal/controller/lpx/util.go | 110 + .../internal/controller/lpx/util_test.go | 219 + .../operator/internal/controller/lpx/watch.go | 161 +- .../internal/controller/lpx/watch_test.go | 301 +- deploy/operator/internal/dynamo/graph_lpx.go | 30 +- .../internal/dynamo/graph_lpx_test.go | 31 +- deploy/operator/internal/dynamo/grove_lpx.go | 18 +- .../internal/dynamo/grove_lpx_test.go | 3 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 1 + .../dynamo/lpx/materialization_test.go | 2 +- .../internal/dynamo/lpx/pod_roles_test.go | 73 - deploy/operator/internal/dynamo/lpx/render.go | 4 +- .../internal/dynamo/lpx/render_test.go | 61 +- .../dynamo/lpx/resolve_selected_workload.go | 35 +- .../lpx/resolve_selected_workload_test.go | 38 - ..._startup_test.go => runtime_agent_test.go} | 78 + .../operator/internal/dynamo/lpx/selection.go | 19 +- .../internal/dynamo/lpx/selection_test.go | 129 +- .../dynamo/lpx/selection_validation.go | 276 -- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 14 +- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 12 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 10 +- .../node-local-v2-specdecode.yaml | 18 +- .../node-local-v3-hx-hybrid.yaml | 6 +- .../node-local-v3-hx-lpu-only.yaml | 10 +- .../node-local-v3-hx-specdecode.yaml | 18 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 10 +- .../validation/dynamographdeployment.go | 64 +- ...hdeployment_lpx_validation_envtest_test.go | 160 +- .../webhook/validation/shared_v1beta1.go | 87 +- .../dynamo-component-deployment.mdx | 2 + .../dynamo-graph-deployment.mdx | 11 +- 71 files changed, 7246 insertions(+), 8143 deletions(-) create mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go create mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_status_test.go create mode 100644 deploy/operator/internal/controller/lpx/AGENTS.md create mode 100644 deploy/operator/internal/controller/lpx/CLAUDE.md delete mode 100644 deploy/operator/internal/controller/lpx/attempt_deadline.go delete mode 100644 deploy/operator/internal/controller/lpx/attempt_deadline_test.go delete mode 100644 deploy/operator/internal/controller/lpx/component_roles_test.go delete mode 100644 deploy/operator/internal/controller/lpx/deadline_retirement.go delete mode 100644 deploy/operator/internal/controller/lpx/deadline_retirement_test.go delete mode 100644 deploy/operator/internal/controller/lpx/lifecycle.go delete mode 100644 deploy/operator/internal/controller/lpx/lifecycle_test.go create mode 100644 deploy/operator/internal/controller/lpx/pipeline_request.go create mode 100644 deploy/operator/internal/controller/lpx/pipeline_request_deadline.go create mode 100644 deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go create mode 100644 deploy/operator/internal/controller/lpx/pipeline_request_test.go delete mode 100644 deploy/operator/internal/controller/lpx/preflight.go create mode 100644 deploy/operator/internal/controller/lpx/status.go create mode 100644 deploy/operator/internal/controller/lpx/status_test.go create mode 100644 deploy/operator/internal/controller/lpx/testdata/dgd.yaml create mode 100644 deploy/operator/internal/controller/lpx/testdata/hybrid-conductor.yaml create mode 100644 deploy/operator/internal/controller/lpx/testdata/pipeline-request-ready.golden.json create mode 100644 deploy/operator/internal/controller/lpx/testdata/pipeline-request-statuses.yaml create mode 100644 deploy/operator/internal/controller/lpx/util.go create mode 100644 deploy/operator/internal/controller/lpx/util_test.go rename deploy/operator/internal/dynamo/lpx/{runtime_startup_test.go => runtime_agent_test.go} (76%) delete mode 100644 deploy/operator/internal/dynamo/lpx/selection_validation.go diff --git a/deploy/operator/api/AGENTS.md b/deploy/operator/api/AGENTS.md index 4916d0c481a7..4553346f3708 100644 --- a/deploy/operator/api/AGENTS.md +++ b/deploy/operator/api/AGENTS.md @@ -8,3 +8,8 @@ code and conversion tests, or explicitly document why conversion is unaffected. Follow its invariants for hub/spoke conversion, sparse annotation preservation, live-source precedence, structural helper naming, and fuzz coverage. + +Keep public API comments concise and focused on the field's contract, defaults, +and validation constraints. Put reconciliation mechanics and detailed rationale +in comments beside the implementation or in its scoped `AGENTS.md`, not in API +field descriptions. diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index d5ccc36ba65d..70462ceaac42 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -6,7 +6,21 @@ package v1alpha1 import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" +) + +const ( + // LPXReadyCondition reports if LPXGraphDeployment object is ready. + // Note: This condition type is defined to ensure consistent naming of conditions across objects. + // Please use object specific variants of this condition which provides more details for each context where + // the same condition type exists. + LPXReadyCondition = "Ready" +) + +// LPX readiness reasons distinguish progress from failure when Ready is false. +const ( + LPXReadyReasonReady = "Ready" + LPXReadyReasonPending = "Pending" + LPXReadyReasonFailed = "Failed" ) // LPXGraphDeploymentSpec is the operator's handoff for all LPX components in a DGD. @@ -26,16 +40,13 @@ type LPXGraphDeploymentStatus struct { // Every spec.inputRevision change advances the child generation. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // conditions report compilation, placement, and runtime readiness. + // conditions reports readiness and scheduling deadline failures. + // Ready uses reason Ready, Pending, or Failed and a diagnostic message. + // SchedulingFailed independently retains deadline failure and retry state. // +optional // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:"conditions,omitempty"` - // expiredRequestUIDs records requests selected for deadline cleanup until they disappear, - // even if their scheduler phase or the deployment input changes. - // +optional - // +listType=set - ExpiredRequestUIDs []types.UID `json:"expiredRequestUIDs,omitempty"` // components reports logical replicas by authored component name, never Agent Pods. // +optional Components map[string]v1beta1.ComponentReplicaStatus `json:"components,omitempty"` diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index b6025b327dbb..dfdcbdebf529 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -44,7 +44,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" configv1alpha1 "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1" ) @@ -1453,11 +1452,6 @@ func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ExpiredRequestUIDs != nil { - in, out := &in.ExpiredRequestUIDs, &out.ExpiredRequestUIDs - *out = make([]types.UID, len(*in)) - copy(*out, *in) - } if in.Components != nil { in, out := &in.Components, &out.Components *out = make(map[string]v1beta1.ComponentReplicaStatus, len(*in)) diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index 6d62a6830e6b..0430bc6c2e0d 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -148,10 +148,8 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { } else if _, exists := status.Properties["lpx"]; exists { t.Errorf("%s %s unexpectedly groups private child status", name, version.Name) } - for _, field := range []string{"modelDownload", "placement"} { - if status.Properties[field].Type != objectType { - t.Errorf("%s %s is missing the typed %s projection", name, version.Name, field) - } + if status.Properties["modelDownload"].Type != objectType { + t.Errorf("%s %s is missing the typed modelDownload projection", name, version.Name) } } } diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index 27325ceb12b1..6430f0cd5c2e 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -176,7 +176,10 @@ spec: description: components reports logical replicas by authored component name, never Agent Pods. type: object conditions: - description: conditions report compilation, placement, and runtime readiness. + description: |- + conditions reports readiness and scheduling deadline failures. + Ready uses reason Ready, Pending, or Failed and a diagnostic message. + SchedulingFailed independently retains deadline failure and retry state. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -234,18 +237,6 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map - expiredRequestUIDs: - description: |- - expiredRequestUIDs records requests selected for deadline cleanup until they disappear, - even if their scheduler phase or the deployment input changes. - items: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - type: array - x-kubernetes-list-type: set modelDownload: description: modelDownload retains the existing remote-build download progress. properties: diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index b774cade5a52..9b265ed45dc4 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -93,11 +93,20 @@ func (r *lpxRestartProgressResolver) observeRestart( child := &v1alpha1.LPXGraphDeployment{} if err := r.reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || - dynamo.ValidateLPXSource(child, source) != nil || source.Spec.Restart == nil || + !metav1.IsControlledBy(child, source) || source.Spec.Restart == nil || child.Annotations[dynamo.LPXRestartAnnotation] != source.Spec.Restart.ID { return nil } - ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + + // Restart progress is resolved before handoff; an old Ready child cannot cover a newer DGD input. + restart := dynamo.LPXRestartToken(source, child.Annotations[dynamo.LPXRestartAnnotation]) + revision, err := dynamo.LPXInputRevision(source, restart) + if err != nil || child.Spec.InputRevision != revision { + return nil + } + + // The current input must also have a Ready receipt for the child's current generation. + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { return nil } diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go new file mode 100644 index 000000000000..1a82d3037a0f --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { + for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeSequential, v1beta1.RestartStrategyTypeParallel} { + t.Run(string(strategy), func(t *testing.T) { + t.Log("Observe a Ready child after delivering the selected LPX restart") + ctx := t.Context() + _, dgd, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + dgd.Spec.Restart = &v1beta1.Restart{ID: "restart-1", Strategy: &v1beta1.RestartStrategy{Type: strategy}} + resolver := newLPXRestartProgressResolver(kube) + dgd.Status.Restart = newDGDRestartReconciler().Resolve(ctx, dgd, &dgd.Status, resolver.Resolve).Status + handoff := &dgdLPXHandoff{client: kube} + child, err := handoff.Reconcile(ctx, dgd) + require.NoError(t, err) + require.Equal(t, "restart-1", child.Annotations[dynamo.LPXRestartAnnotation]) + child.Status.ObservedGeneration = child.Generation + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Ready: true}} + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, + ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, + }) + require.NoError(t, kube.Status().Update(ctx, child)) + require.Empty(t, resolver.Resolve(ctx, dgd, []string{"lpx"})) + + t.Log("An input edit before handoff must keep the restart in progress despite the old Ready receipt") + lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "runtime:changed" + dgd.Generation++ + revision, err := dynamo.LPXInputRevision(dgd, "restart-1") + require.NoError(t, err) + require.NotEqual(t, revision, child.Spec.InputRevision) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) + restart := newDGDRestartReconciler().Resolve(ctx, dgd, &dgd.Status, resolver.Resolve) + require.Equal(t, []string{"lpx"}, restart.Status.InProgress) + require.NotEqual(t, v1beta1.RestartPhaseCompleted, restart.Status.Phase) + + t.Log("Complete only after the handoff's new child generation becomes Ready") + updated, err := handoff.Reconcile(ctx, dgd) + require.NoError(t, err) + require.Equal(t, revision, updated.Spec.InputRevision) + + // The fake client does not increment generation for the handoff's spec update. + updated.Generation++ + require.NoError(t, kube.Update(ctx, updated)) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) + updated.Status.ObservedGeneration = updated.Generation + meta.SetStatusCondition(&updated.Status.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, + ObservedGeneration: updated.Generation, Reason: v1alpha1.LPXReadyReasonReady, + }) + require.NoError(t, kube.Status().Update(ctx, updated)) + require.Empty(t, resolver.Resolve(ctx, dgd, []string{"lpx"})) + restart = newDGDRestartReconciler().Resolve(ctx, dgd, &dgd.Status, resolver.Resolve) + require.Equal(t, v1beta1.RestartPhaseCompleted, restart.Status.Phase) + }) + } +} diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index ceffe97fea49..9b38b5ffa4c3 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -48,20 +48,20 @@ func mergeLPXChildStatus( } } - if ordinary.State != v1beta1.DGDStateFailed && current { - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") - if failed != nil && failed.Status == metav1.ConditionTrue && failed.ObservedGeneration == child.Generation { + // A current failure is actionable even if reconciliation did not finish. + if current { + ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") + if ordinary.State != v1beta1.DGDStateFailed && ready != nil && + ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionFalse && + ready.Reason == v1alpha1.LPXReadyReasonFailed { ordinary.State = v1beta1.DGDStateFailed - ordinary.Reason, ordinary.Message = Reason(failed.Reason), Message(failed.Message) + ordinary.Reason, ordinary.Message = Reason(ready.Reason), Message(ready.Message) return ordinary, status } - } - if observed { - ready := meta.FindStatusCondition(child.Status.Conditions, "Ready") - if ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { + if observed && ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { return ordinary, status } - if ordinary.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { + if observed && ordinary.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { ordinary.State = v1beta1.DGDStatePending ordinary.Reason, ordinary.Message = Reason(ready.Reason), Message(ready.Message) return ordinary, status diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go new file mode 100644 index 000000000000..52782f3eb8b9 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestLPXStatusUsesOneCurrentChild(t *testing.T) { + const stale = "stale" + for _, scenario := range []string{"ready", "no-download", stale, "retiring", "failure"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Create three independent components under one child") + source := newLPXHandoffSource(t, "node-local-v2-hybrid") + other := newLPXHandoffSource(t, "single_v2").Spec.Components[0] + other.ComponentName = "other" + third := *other.DeepCopy() + third.ComponentName = "third" + source.Spec.Components = append(source.Spec.Components, other, third) + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(source).Build() + child, err := (&dgdLPXHandoff{client: kube}).Reconcile(t.Context(), source) + require.NoError(t, err) + children := &v1alpha1.LPXGraphDeploymentList{} + require.NoError(t, kube.List(t.Context(), children)) + require.Len(t, children.Items, 1) + child.Generation = 2 + + t.Log("Record one download result for all builds and independent component replica counts") + checkedAt := metav1.Unix(1, 0) + child.Status = v1alpha1.LPXGraphDeploymentStatus{ + ObservedGeneration: 2, + ModelDownload: &v1beta1.ModelDownloadStatus{ + Builds: []string{"gs://models/first", "gs://models/second"}, LastCheckedAt: &checkedAt, + }, + Components: map[string]v1beta1.ComponentReplicaStatus{}, + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: 2}}, + } + for i, component := range source.Spec.Components { + child.Status.Components[component.ComponentName] = v1beta1.ComponentReplicaStatus{Ready: true, Replicas: int32(i + 1)} + } + + t.Log("Gate the graph on the shared child's current lifecycle") + want := v1beta1.DGDStateSuccessful + switch scenario { + case "no-download": + child.Status.ModelDownload = nil + case stale: + child.Status.ObservedGeneration-- + want = v1beta1.DGDStatePending + case "retiring": + now := metav1.Now() + child.DeletionTimestamp = &now + want = v1beta1.DGDStatePending + case "failure": + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: 2, Reason: v1alpha1.LPXReadyReasonFailed}} + want = v1beta1.DGDStateFailed + } + before := child.DeepCopy() + result, status := mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStateSuccessful}) + require.Equal(t, want, result.State) + + t.Log("Publish the aggregate download only for a current child and retain per-component readiness") + if scenario == "ready" || scenario == "failure" || scenario == "no-download" { + require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: before.Status.ModelDownload}, status) + require.Equal(t, before.Status.Components, result.ComponentStatus) + if status.ModelDownload != nil { + status.ModelDownload.Builds[0] = "parent-copy" + status.ModelDownload.LastCheckedAt.Time = metav1.Unix(2, 0).Time + } + } else { + require.Nil(t, status) + for _, component := range result.ComponentStatus { + require.False(t, component.Ready) + require.Zero(t, component.Replicas) + } + } + require.Equal(t, before, child) + }) + } +} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 88ec3fb34a9b..234a518efd44 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -218,14 +218,16 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { return } - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + failed := meta.FindStatusCondition(child.Status.Conditions, "Ready") ready := meta.FindStatusCondition(source.Status.Conditions, "Ready") if !assert.NotNil(c, failed) || !assert.NotNil(c, ready) { return } assert.Greater(c, child.Generation, child.Status.ObservedGeneration) assert.Equal(c, child.Generation, failed.ObservedGeneration) - assert.Equal(c, metav1.ConditionTrue, failed.Status) + assert.Equal(c, metav1.ConditionFalse, failed.Status) + assert.Len(c, child.Status.Conditions, 1) + assert.Equal(c, v1alpha1.LPXReadyReasonFailed, failed.Reason) assert.Contains(c, failed.Message, "exceeded quota: block-lpx-publication") assert.Equal(c, v1beta1.DGDStateFailed, source.Status.State) assert.Equal(c, source.Generation, ready.ObservedGeneration) @@ -244,42 +246,23 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { require.NoError(t, env.Client().List(t.Context(), requests, client.InNamespace(env.Namespace()))) require.Empty(t, requests.Items) - t.Log("Accept an LPX runtime-invalid edit and report its exact rejection on the public DGD") - require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { + t.Log("Reject invalid placement at admission without changing the source or child revision") + sourceGeneration, childGeneration := source.Generation, child.Generation + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { if err := env.Client().Get(t.Context(), key, source); err != nil { return err } source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "manual-placement" return env.Client().Update(t.Context(), source) - })) - require.EventuallyWithT(t, func(c *assert.CollectT) { - if !assert.NoError(c, env.Client().Get(t.Context(), key, source)) || !assert.NoError(c, env.Client().Get(t.Context(), key, child)) { - return - } - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") - ready := meta.FindStatusCondition(source.Status.Conditions, "Ready") - if !assert.NotNil(c, failed) || !assert.NotNil(c, ready) { - return - } - assert.Equal(c, child.Generation, failed.ObservedGeneration) - assert.Equal(c, "LPXRejected", failed.Reason) - assert.Contains(c, failed.Message, "spec.components[0].roles[1].podTemplate.spec.nodeName") - assert.Contains(c, failed.Message, "LPX owns role addressing and placement") - assert.Equal(c, v1beta1.DGDStateFailed, source.Status.State) - assert.Equal(c, source.Generation, ready.ObservedGeneration) - assert.Equal(c, metav1.ConditionFalse, ready.Status) - assert.Equal(c, failed.Reason, ready.Reason) - assert.Equal(c, failed.Message, ready.Message) - }, 20*time.Second, 50*time.Millisecond) - - t.Log("Repair the accepted configuration so the same DGD and child can reconcile again") - require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := env.Client().Get(t.Context(), key, source); err != nil { - return err - } - source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName = "" - return env.Client().Update(t.Context(), source) - })) + }) + require.True(t, apierrors.IsInvalid(err), "%v", err) + require.ErrorContains(t, err, "spec.components[0].roles[1].podTemplate.spec.nodeName") + require.ErrorContains(t, err, "LPX owns role addressing and placement") + require.NoError(t, env.Client().Get(t.Context(), key, source)) + require.NoError(t, env.Client().Get(t.Context(), key, child)) + require.Equal(t, sourceGeneration, source.Generation) + require.Equal(t, childGeneration, child.Generation) + require.Empty(t, source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeName) t.Log("Release quota and observe alpha LGD ownership of the published PCS and runtime resources without waiting for scheduling") require.NoError(t, env.Client().Delete(t.Context(), quota)) @@ -326,14 +309,27 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NotEmpty(t, child.UID) require.NotEqual(t, source.UID, child.UID) require.Equal(t, metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK), metav1.GetControllerOf(child)) - require.NoError(t, dynamo.ValidateLPXSource(child, source)) + revision, err := dynamo.LPXInputRevision(source, child.Annotations[dynamo.LPXRestartAnnotation]) + require.NoError(t, err) + require.Equal(t, revision, child.Spec.InputRevision) t.Log("Project the observed download payload and its actionable failure") child.Status.ObservedGeneration = child.Generation child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} - child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, - LastTransitionTime: metav1.Now(), Reason: "PublicationDenied", Message: "Check the namespace quota"}} + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, + LastTransitionTime: metav1.Now(), Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} + failureTime := metav1.NewTime(time.Now().Add(-time.Minute).UTC().Truncate(time.Second)) + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ + Type: "SchedulingFailed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, + LastTransitionTime: failureTime, Reason: "LPXSchedulingDeadlineExceeded", Message: "An LPX request exceeded its scheduling deadline", + }) require.NoError(t, env.Client().Status().Update(t.Context(), child)) + require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Len(t, child.Status.Conditions, 2) + schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, "SchedulingFailed") + require.NotNil(t, schedulingFailed) + require.True(t, failureTime.Equal(&schedulingFailed.LastTransitionTime)) + require.Equal(t, child.Generation, schedulingFailed.ObservedGeneration) projected := v1beta1.DynamoGraphDeploymentStatus{} result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} projectLPXChildStatus(source, child, result, &projected) @@ -361,7 +357,9 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, err) require.Equal(t, child.Generation+1, updated.Generation) require.NotEqual(t, child.Spec.InputRevision, updated.Spec.InputRevision) - require.NoError(t, dynamo.ValidateLPXSource(updated, source)) + revision, err = dynamo.LPXInputRevision(source, updated.Annotations[dynamo.LPXRestartAnnotation]) + require.NoError(t, err) + require.Equal(t, revision, updated.Spec.InputRevision) require.Equal(t, child.Status.ObservedGeneration, updated.Status.ObservedGeneration) result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} projectLPXChildStatus(source, updated, result, &projected) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 1c6719943b37..ec5c3f0ef642 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -177,7 +177,10 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.NotEmpty(t, child.Spec.InputRevision) - require.NoError(t, dynamo.ValidateLPXSource(child, source)) + require.Equal(t, metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK), metav1.GetControllerOf(child)) + revision, err := dynamo.LPXInputRevision(source, child.Annotations[dynamo.LPXRestartAnnotation]) + require.NoError(t, err) + require.Equal(t, revision, child.Spec.InputRevision) require.NotContains(t, child.Annotations, "lpx.nvidia.com/podcliqueset-name") pending := ReconcileResult{State: v1beta1.DGDStatePending} projectLPXChildStatus(source, child, &pending, &v1beta1.DynamoGraphDeploymentStatus{}) @@ -259,10 +262,10 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { child.Status.Components["lpx"] = partial child.Status.ModelDownload = nil result.Reason = "CheckpointReady" - meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "LPXRetiring"}) + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonPending, Message: "Waiting for removed LPX requests"}) projectLPXChildStatus(source, child, &result, &status) require.Equal(t, v1beta1.DGDStatePending, result.State) - require.Equal(t, Reason("LPXRetiring"), result.Reason) + require.Equal(t, Reason(v1alpha1.LPXReadyReasonPending), result.Reason) require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{}, status.LPX) require.Equal(t, source.Status.Placement, status.Placement) require.Equal(t, partial, result.ComponentStatus["lpx"]) @@ -307,7 +310,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"stale-build"}} - child.Status.Conditions = []metav1.Condition{{Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "PublicationDenied", Message: "Check the namespace quota"}} + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} result := ReconcileResult{State: v1beta1.DGDStateSuccessful} status := v1beta1.DynamoGraphDeploymentStatus{LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ ModelDownload: child.Status.ModelDownload.DeepCopy(), @@ -320,7 +323,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { case "old-condition": child.Status.Conditions[0].ObservedGeneration-- case "cleared-condition": - child.Status.Conditions[0].Status = metav1.ConditionFalse + child.Status.Conditions[0].Reason = v1alpha1.LPXReadyReasonPending case "deleting": child.DeletionTimestamp = ptr.To(metav1.Now()) case "ordinary-failure": @@ -407,7 +410,9 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { source.Status.Restart = completed.Status source.Spec.Restart.ID = "restart-2" source.Generation++ - require.NoError(t, dynamo.ValidateLPXSource(updated, source)) + revision, err := dynamo.LPXInputRevision(source, updated.Annotations[dynamo.LPXRestartAnnotation]) + require.NoError(t, err) + require.Equal(t, revision, updated.Spec.InputRevision) unchanged, err = handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.Equal(t, updated, unchanged) @@ -744,7 +749,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { child, err := (&dgdLPXHandoff{client: kube}).Reconcile(t.Context(), source) require.NoError(t, err) child.Status.ObservedGeneration = child.Generation - child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: "ModelDownloadPending"}} + child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonPending, Message: "Waiting for model downloads"}} require.NoError(t, kube.Status().Update(t.Context(), child)) config := &configv1alpha1.OperatorConfiguration{} config.Namespace.Restricted = source.Namespace @@ -819,15 +824,15 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { t.Log("An actionable child failure remains visible while the ordinary checkpoint is still pending") require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), child)) meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ - Type: "Failed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, - Reason: "LPXReconciliationFailed", Message: "publish LPX workload: storage volume mount is missing", + Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, + Reason: v1alpha1.LPXReadyReasonFailed, Message: "publish LPX workload: storage volume mount is missing", }) require.NoError(t, kube.Status().Update(t.Context(), child)) result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) require.NoError(t, err) require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) failed := meta.FindStatusCondition(result.Status.Conditions, "Ready") - require.Equal(t, "LPXReconciliationFailed", failed.Reason) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, failed.Reason) require.Contains(t, failed.Message, "storage volume mount is missing") require.Equal(t, v1beta1.ComponentCheckpointStatus{CheckpointName: friendlyCheckpointName}, result.Status.Checkpoints["prefill"]) diff --git a/deploy/operator/internal/controller/lpx/AGENTS.md b/deploy/operator/internal/controller/lpx/AGENTS.md new file mode 100644 index 000000000000..05b13310204f --- /dev/null +++ b/deploy/operator/internal/controller/lpx/AGENTS.md @@ -0,0 +1,238 @@ + + +# Scope + +- Review against supported failure scenarios, including ownership errors, missing + wakeups, unsafe deletion, and invalid API writes. +- Ask before changing documented trade-offs. Update these instructions when an + intentional contract changes. +- Support DGD input edits and external scaling. +- Recovery from direct edits, force-deletion, or orphaning of managed + resources are out of scope unless explicitly requested. +- Do not add migration or recovery machinery solely for unsupported cases. + +# Reconciliation + +- Keep reconciliation linear: observe, derive desired state, apply changes, and + return `ctrl.Result` and an error. Keep ordering and early returns visible. +- Pass native API objects and focused values directly. Avoid custom request or + result bags, classification structs, and callback pipelines. Calculations and + renderers may return values or native resources. +- Validate ownership and input freshness at the observation boundary, then trust + those objects downstream. Document meaningful nil cases; do not repeat checks + that callers have already established. +- Admission validates static user intent. Reconciliation validates the input + handoff, observed resource structure, and compiler-dependent constraints. + Placement, topology discovery, and GPU-shape calculation belong to the + compiler/rendering and scheduler layers. +- Name functions after the resource or condition they handle. Prefer + `pipelineRequest`, `schedulingFailed`, and `dgd` to ambiguous terms such as + "attempt", "source", or "LPX deadline". + +# Ownership and Identity + +- The DynamoGraphDeployment (DGD) owns the LPXGraphDeployment (LPXGD). The LPXGD + owns its PodCliqueSet (PCS), runtime ConfigMaps, and Services. The PCS owns the + LPUPipelineRequests (LPRs) and Grove hierarchy. One PodCliqueScalingGroup (PCSG) + under PCS ordinal zero holds all engine replicas. +- LPRs have a controlling PCS owner reference with `blockOwnerDeletion: true`. + Foreground PCS deletion keeps the owner present until blocking LPRs are gone. + This is the supported lifecycle, not a guarantee against external orphaning. +- The parent controller deletes the LPXGD when the DGD no longer selects LPX. + A missing/deleting DGD or incomplete input handoff stops publication; it does + not authorize cleanup. A DGD UID mismatch is an error. Disabled integration + also leaves existing workloads intact. +- Index LPRs by controlling PCS UID, validating the owner's API version and kind + in the index. Do not add a PCS-name index or namespace-wide request scan. +- PCS identity derives from the LPXGD UID and stays stable across input edits. + LPR names identify the LPXGD namespace, name, UID, model, and PCSG ordinal. +- Keep identity and change detection separate: `InputRevision` synchronizes the + parent-to-child handoff; workload digests detect immutable workload changes; + `CompilerSnapshotDigestAnnotation` records compiler provenance and is part of + immutable LPR intent. Neither input revision nor generation defines a batch. +- Accept restart completion only when the child's input revision matches the + current DGD. A matching restart token and an old Ready condition are insufficient. + +# Observation and Retries + +- Read through the cached client. Observe dependencies and validate them once + per reconciliation; do not add uncached reads or pre-write revalidation. +- Observations are not an atomic snapshot. Publishing previously observed + intent during a concurrent edit is accepted; watches drive convergence. + UID/resource-version preconditions and write conflicts protect mutations. +- Use watches for cache visibility, publication, finalization, and input or + scheduler changes. A watch-driven wait returns a zero result, not a polling + interval. Keep progress gates explicit rather than treating a timer as a + "stop this pass" signal. +- Reserve `RequeueAfter` for scheduling deadlines, bounded error retries while + a deadline is active, and external download checks. The other deliberate use + is the follow-up after recording `SchedulingFailed`, because status-only + LPXGD updates are filtered. Ordinary errors use controller-runtime backoff. +- After writing a PCS, wait for its watched observation before publishing LPRs. + An LPR `AlreadyExists` response means wait for observation, not adopt an + unverified object. Never adopt a foreign resource with the expected name. + +# Publication + +- Retain matching LPRs and their status across input edits. Collect missing + requests in the same ordered pass that constructs desired requests. An + immutable mismatch invalidates that result and requires PCS replacement. +- Publish missing requests independently in PCSG ordinal/model order. Partial + publication and idempotent retries are supported; exactly-once batches are + not required. Creation order does not guarantee scheduler order or prevent + interior scheduling failures. +- Wait for terminating requests to disappear before reusing their names. + Do not introduce batch finalizers or cleanup protocols. + +# Capacity and Deletion + +- Keep the top-level PCS replica count at one; engine capacity belongs to the + PCSG. Correct top-level replica drift through normal synchronization, not + PCS replacement. +- Seed the PCS's scaling-group template from immutable `MinAvailable` (default + one), never from live or desired capacity. Apply explicit desired capacity + through PCSG `/scale` before publishing requests. The seed stays positive and + unchanged even when live capacity is zero. +- Explicit DGD replicas give this controller capacity ownership. Omitted + replicas leave capacity externally managed: derive request ordinals from the + observed PCSG and never write `/scale`, including during deadline cleanup. +- External capacity is not persisted across PCS or PCSG replacement. A new + group starts from the immutable template seed; its external scaler must + reapply the desired count. Explicit DGD replicas are reapplied by this controller. +- Validate the single-group template, group ownership, and deletion state at + observation. A foreign PCSG is an ownership error, not a cache miss. A nil PCSG + means the group is missing or the owned group is deleting; it does not mean + zero capacity or completed pod deletion. +- Initial PCS creation needs no PCSG. With explicit replicas and no existing + LPRs, PCS synchronization may also proceed before the group appears. With + explicit replicas, deadlines may be recorded while the group is unavailable. + Request deletion, publication, and readiness require an observed group; + externally managed capacity also requires it once the PCS exists. +- Render a complete valid replacement before deleting an existing workload. + A replacement error preserves the existing workload, even if its digest differs. +- For immutable workload or request changes, foreground-delete the PCS and + recreate it after garbage collection. +- Grove makes PCS clique composition and scaling-group `CliqueNames` immutable. + Model removal therefore requires PCS replacement. Do not trim group membership + while retaining templates; those templates can become standalone cliques. +- For replica-only scale-in, lower PCSG capacity before deleting removed LPRs. + Preserve the PCS and clique templates. +- For replica-only scale-out, wait for old request names to disappear, update + capacity, then publish missing LPRs. +- Write scale-down before deleting LPRs, but do not wait for Grove or pod + deletion. Asynchronous pod cleanup allows scheduler finalizers to complete. +- Never directly delete Pods or PodCliques from graph reconciliation or LPR + lifecycle helpers. Grove scaling and owner garbage collection own workload + cleanup; scheduler Pod references and release journals do not authorize it. +- A terminating LPR is pending cleanup, not a reason to delete its PCS or Pods. + Wait for scheduler finalization through watches, including when external + capacity is unchanged. The eviction controller has a separate disruption policy. + +# Scheduling Cycles + +- Scheduling deadlines apply to individual LPR cycles, not all deployment + Pending states, runtime readiness, or asynchronous cleanup. +- Start the deadline at `LPR.status.schedulingStartedAt`. Without that timestamp, + no deadline is running. Do not substitute creation time, managed fields, or a + deployment-level timestamp. The scheduler supplies a new start when a surviving + request begins another cycle, including Bound-to-Pending transitions. +- Current-generation `Bound`, `NoFit`, and `Unsupported` states are exempt. + `Degraded`, `Releasing`, and `Released` are exempt only with a current-generation + status and a committed execution accepted for that generation. +- Exemption is not readiness: `NoFit` remains pending, and `Bound` still requires + Grove runtime readiness. +- The deadline is best effort: download checks, compiler-registry resolution and + desired-state validation precede expiry evaluation. Their failures can delay + failure reporting and deadline cleanup indefinitely, including after expiry. + +# Scheduling Failure and Cleanup + +- Persist `SchedulingFailed` and observe it in a later reconciliation before + cleanup. Its timestamp must cover the expired scheduling cycles; a newer cycle + needs a renewed failure record. +- Re-evaluate expiry from current request status on every pass. Do not delete + a request solely because it expired in an earlier pass. The condition records + failure, not a deletion queue; do not persist an expired-request UID list. +- One expired model request makes its entire engine replica eligible, including + sibling model requests. Cleanup may remove only a trailing suffix. + Any interior expired ordinal blocks all deadline scale-down and request + deletion, even if another expired suffix exists. For four replicas, expiry at + `{2,3}` permits scale to two; `{0}` or `{0,3}` permits no cleanup. +- Continue cleanup beyond an already-lowered group count on subsequent passes. + Delete non-expired siblings before expired requests so a partial failure + leaves expiry evidence for the next reconciliation. +- A scheduling failure blocks publication, not normal request cleanup after + explicit DGD scale-in or external scale-in, even when an interior failure + blocks deadline cleanup. Lower explicitly managed PCSG capacity before + deleting removed requests; never write externally managed capacity. +- Preserve healthy higher ordinals even when interior failures leave the + deployment failed or pending indefinitely. + +# Input Edits and Retries + +- Failure remains in effect for the recorded child generation, preventing + automatic scale-up or recreation of expired requests. +- A new input revision bypasses previous-revision failure. Resolve the new + desired state first; if it errors, report that error instead of applying + deadline cleanup to the old revision. +- Retained requests keep their scheduling timestamps. An already-expired request + covered by a durable failure uses that record for cleanup even after an input + edit; preserve its original failure generation so the edit still permits retry. + A cycle started after the failure needs a new record and can fail the new + revision. An edit does not reset a retained cycle's deadline. Requests removed + by the edit follow normal scale-in or replacement cleanup. + +# Status and Runtime Artifacts + +- `Ready` is the only general outcome condition, with reason `Ready`, `Pending`, + or `Failed`. Keep `SchedulingFailed` as independent failure and retry evidence, + not duplicate timestamp/generation fields. An error may replace the Ready + diagnostic without clearing scheduling failure. +- `Ready.LastTransitionTime` changes only when its boolean status changes. + Renewing `SchedulingFailed` deliberately gives it a new timestamp even when + it remains true. +- Persist child status once at the reconciliation boundary, including early + errors. Record errors before converting them into bounded deadline retries. + Advance `ObservedGeneration` after error-free reconciliation, including pending + or disabled outcomes; this does not imply readiness or freshness of every field. +- Convert workload errors into deadline retries only after deciding generation + acknowledgement and persisting status. Status-write failures remain errors. + `reconcileWorkload` preserves its error and puts any bounded deadline retry in + `ctrl.Result`; only the outer `Reconcile` converts that pair to a successful retry. +- Synchronize propagated PCS and serving Service labels and annotations through + `WithMetadataSync`. Remove previously tracked keys omitted from desired state; + preserve untracked metadata and the shared helper's bookkeeping annotations. +- Check downloads before acquiring compiler metadata or deriving deadlines. + Reuse fresh successful observations by build URL across revisions, without + renewing `LastCheckedAt`. Check new or expired builds. Only a Ready deployment + whose generation has been observed may ignore a failed periodic refresh. +- Runtime ConfigMaps are immutable, content-addressed, LPXGD-owned, and labeled + with its UID. Find obsolete maps by label, verify ownership before deletion, + and retain desired names. Delete obsolete maps only after readiness, not during + PCS replacement, so existing pods keep their configuration. + +# Tests + +- Pair test files with production files and test behavior in the package that + owns it. Exercise controller ordering through `Reconcile`; do not retest shared + synchronization, conversion, or fake-client behavior here. +- Use data-only tables for variations of one function. Keep complex setup and + multi-reconcile scenarios as separate tests, with their steps visible in the + test body rather than table callbacks. +- Share constructors, fixture loaders, and the test-client harness, not mutable + fixtures or a test-only implementation of controller decisions. Use minimal + native inputs; add compiler fixtures and Grove hierarchies only when relevant. +- Load reusable workloads from `testdata/` and derive child ownership and input + revisions from them. Construct compiler artifacts programmatically when testing + malformed or variant binary metadata. +- Use golden files for substantial deterministic output, not scalar assertions. + Review regenerated diffs; never update expectations merely to silence failures. +- Fake clients do not enforce Grove admission, garbage collection, or RBAC. + Check API mutability and permissions when changing writes. +- For code changes, run `go test ./...` and `make lint` from `deploy/operator`, + using the repository's envtest assets. Run LPX race tests for lifecycle changes. +- Regenerate the condition golden with + `go test ./internal/controller/lpx -run TestPipelineRequestReadyConditionGolden -args -update`. diff --git a/deploy/operator/internal/controller/lpx/CLAUDE.md b/deploy/operator/internal/controller/lpx/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline.go b/deploy/operator/internal/controller/lpx/attempt_deadline.go deleted file mode 100644 index 1c92909bee92..000000000000 --- a/deploy/operator/internal/controller/lpx/attempt_deadline.go +++ /dev/null @@ -1,224 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - "time" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" -) - -const ( - lpxSchedulingFailedCondition = "SchedulingFailed" - lpxSchedulingDeadlineExceededReason = "LPXSchedulingDeadlineExceeded" -) - -type lpxDeadlineExceeded struct { - requeueAfter time.Duration - recordFailure bool -} - -func (*lpxDeadlineExceeded) lpxClassification() {} - -// completeLPXDeadline preserves deadline wakes and converts errors before an -// active deadline into bounded retries. -func completeLPXDeadline( - ctx context.Context, - deadlineAt time.Time, - result ctrl.Result, - reconcileErr error, -) (ctrl.Result, error) { - if deadlineAt.IsZero() { - return result, reconcileErr - } - if reconcileErr != nil { - log.FromContext(ctx).Error(reconcileErr, "retrying reconciliation under the LPX scheduling deadline") - } - delay := time.Until(deadlineAt) - if reconcileErr != nil { - if delay <= 0 || delay > lpxRetirementRequeueAfter { - delay = lpxRetirementRequeueAfter - } - return ctrl.Result{RequeueAfter: delay}, nil - } - if delay <= 0 { - delay = time.Nanosecond - } - if result.RequeueAfter <= 0 || delay < result.RequeueAfter { - result.RequeueAfter = delay - } - return result, nil -} - -// reconcileLPXRequestDeadlines derives every scheduling clock from the start -// of the corresponding desired LPR's current scheduler-owned scheduling cycle. -// Recorded expiry survives phase/input changes until the exact request disappears. -// A nil PCS leaves no active clock; already recorded cleanup waits for owner GC. -func (r *graphReconciler) reconcileLPXRequestDeadlines( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - desired []lpxModelMaterializing, -) (lpxClassification, time.Time, error) { - deadlineSeconds := lpxRequestDeadlineSeconds(source) - active := deadlineSeconds != nil && pcs != nil && pcs.DeletionTimestamp.IsZero() && metav1.IsControlledBy(pcs, deployment) - recorded := sets.New(deployment.Status.ExpiredRequestUIDs...) - if !active && recorded.Len() == 0 { - return nil, time.Time{}, nil - } - requests, err := r.listOwnedLPXRequests(ctx, deployment, pcs) - if err != nil { - return nil, time.Time{}, err - } - desiredNames := make(map[string]struct{}, len(desired)) - for index := range desired { - desiredNames[desired[index].requestName] = struct{}{} - } - - // Expire each desired nonterminal request independently and retain the earliest future wake. - now := time.Now() - expired := make([]*lpxv1alpha1.LPUPipelineRequest, 0) - deployment.Status.ExpiredRequestUIDs = nil - recordFailure := false - var nextDeadline time.Time - for index := range requests { - request := &requests[index] - if recorded.Has(request.UID) { - deployment.Status.ExpiredRequestUIDs = append(deployment.Status.ExpiredRequestUIDs, request.UID) - if request.DeletionTimestamp.IsZero() { - expired = append(expired, request) - } - continue - } - if !active { - continue - } - if _, current := desiredNames[request.Name]; !current { - continue - } - if !request.DeletionTimestamp.IsZero() || lpxRequestHasTerminalSchedulingDisposition(request) { - continue - } - startedAt, known := lpxSchedulingStartedAt(request) - if !known { - continue - } - deadline := startedAt.Add(time.Duration(*deadlineSeconds) * time.Second) - if !now.Before(deadline) { - deployment.Status.ExpiredRequestUIDs = append(deployment.Status.ExpiredRequestUIDs, request.UID) - recordFailure = true - continue - } - if nextDeadline.IsZero() || deadline.Before(nextDeadline) { - nextDeadline = deadline - } - } - - // Persist exact request identities and retry authorization before cleanup. - if recordFailure { - return &lpxDeadlineExceeded{requeueAfter: time.Nanosecond, recordFailure: true}, time.Time{}, nil - } - if recorded.Len() == 0 { - return nil, nextDeadline, nil - } - - // Request and Grove watches drive cleanup while expired UIDs remain. - retirementErr := r.retireExpiredLPXRequests(ctx, deployment, source, pcs, requests, expired) - if len(deployment.Status.ExpiredRequestUIDs) > 0 { - return &lpxDeadlineExceeded{}, nextDeadline, retirementErr - } - // Clearing the last UID only changes status; explicitly wake an authorized retry. - return &lpxDeadlineExceeded{requeueAfter: lpxRetirementRequeueAfter}, nextDeadline, retirementErr -} - -func lpxRequestHasTerminalSchedulingDisposition(request *lpxv1alpha1.LPUPipelineRequest) bool { - if request.Status == nil || request.Status.ObservedGeneration == nil || - *request.Status.ObservedGeneration != request.Generation { - return false - } - switch request.Status.Phase { - case lpxv1alpha1.RequestPhaseBound, lpxv1alpha1.RequestPhaseNoFit, lpxv1alpha1.RequestPhaseUnsupported: - return true - case lpxv1alpha1.RequestPhaseDegraded, lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: - committed := request.Status.Committed - return committed != nil && committed.Execution.AcceptedGeneration != nil && - *committed.Execution.AcceptedGeneration == request.Generation - default: - return false - } -} - -// lpxSchedulingStartedAt requires a nonnil request and reports whether the -// scheduler published a usable start for its current scheduling cycle. -func lpxSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) (time.Time, bool) { - if request.Status == nil || request.Status.SchedulingStartedAt == nil || request.Status.SchedulingStartedAt.IsZero() { - return time.Time{}, false - } - return request.Status.SchedulingStartedAt.Time, true -} - -func lpxRequestDeadlineSeconds(source *nvidiacomv1beta1.DynamoGraphDeployment) *int64 { - if source == nil || source.Spec.Scheduling == nil { - return nil - } - return source.Spec.Scheduling.AttemptDeadlineSeconds -} - -// lpxDeadlineFailureCurrent fences publication until expired requests disappear -// and a later input revision explicitly authorizes a retry. -func lpxDeadlineFailureCurrent(deployment *nvidiacomv1alpha1.LPXGraphDeployment) bool { - failed := meta.FindStatusCondition(deployment.Status.Conditions, lpxSchedulingFailedCondition) - return len(deployment.Status.ExpiredRequestUIDs) > 0 || (failed != nil && - failed.Status == metav1.ConditionTrue && failed.Reason == lpxSchedulingDeadlineExceededReason && - failed.ObservedGeneration >= deployment.Generation) -} - -// projectLPXSchedulingFailureCondition keeps retry authorization independent -// from the aggregate Failed condition, which may report another controller error. -func projectLPXSchedulingFailureCondition( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - previous *nvidiacomv1alpha1.LPXGraphDeploymentStatus, - state reconcileOutcome, - recordFailure bool, -) { - prior := meta.FindStatusCondition(previous.Conditions, lpxSchedulingFailedCondition) - if state.Reason == lpxSchedulingDeadlineExceededReason { - observedGeneration := deployment.Generation - if prior != nil && prior.Status == metav1.ConditionTrue && !recordFailure { - observedGeneration = prior.ObservedGeneration - } - if recordFailure { - meta.RemoveStatusCondition(&deployment.Status.Conditions, lpxSchedulingFailedCondition) - } - meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, - ObservedGeneration: observedGeneration, Reason: state.Reason, Message: state.Message, - }) - return - } - if prior == nil { - return - } - if deployment.Generation <= prior.ObservedGeneration || len(deployment.Status.ExpiredRequestUIDs) > 0 { - return - } - meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionFalse, - ObservedGeneration: deployment.Generation, Reason: "LPXSchedulingRetryAuthorized", - Message: "The expired request was removed and current input may be published", - }) -} diff --git a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go b/deploy/operator/internal/controller/lpx/attempt_deadline_test.go deleted file mode 100644 index 2482e50d3bc9..000000000000 --- a/deploy/operator/internal/controller/lpx/attempt_deadline_test.go +++ /dev/null @@ -1,352 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - "errors" - "testing" - "time" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" -) - -func TestLPXDeadlineUsesEachSchedulingCycleStart(t *testing.T) { - t.Log("Publish two requests created together whose scheduling cycles start at different times") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - firstStarted := time.Now().UTC().Truncate(time.Second) - created := firstStarted.Add(-time.Hour) - first := deadlineTestRequest(dgd, pcs, "first", created, lpxv1alpha1.RequestPhasePending) - first.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(firstStarted)) - second := deadlineTestRequest(dgd, pcs, "second", created, lpxv1alpha1.RequestPhasePending) - second.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(firstStarted.Add(10 * time.Second))) - r := newLPXTestReconciler(t, nil, dgd, source, pcs, first, second) - - t.Log("Wake at the first scheduling cycle's deadline without using object creation time") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(first, second)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) - - t.Log("Keep that deadline wake while a recorded expiry waits for its finalizer") - second.Finalizers = []string{"scheduler.example/cleanup"} - require.NoError(t, r.Update(t.Context(), second)) - require.NoError(t, r.Delete(t.Context(), second)) - dgd.Status.ExpiredRequestUIDs = []types.UID{second.UID} - classification, wake, err = r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(first, second)) - require.NoError(t, err) - require.IsType(t, &lpxDeadlineExceeded{}, classification) - require.True(t, firstStarted.Add(30*time.Second).Equal(wake)) - require.Zero(t, projectLPXLifecycleStatus(classification), "the finalizer watch needs no additional poll") -} - -func TestLPXDeadlineIgnoresRequestsOutsideCurrentPublication(t *testing.T) { - t.Log("Observe one current request and one expired request retired by a source edit") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - started := time.Now().UTC().Truncate(time.Second) - current := deadlineTestRequest(dgd, pcs, "current", started, lpxv1alpha1.RequestPhasePending) - stale := deadlineTestRequest(dgd, pcs, "stale", started.Add(-time.Minute), lpxv1alpha1.RequestPhasePending) - r := newLPXTestReconciler(t, nil, dgd, source, pcs, current, stale) - - t.Log("Keep the current wake without converting stale cleanup into deadline failure") - classification, wake, err := r.reconcileLPXRequestDeadlines( - t.Context(), dgd, source, pcs, deadlineTestDesired(current), - ) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, started.Add(30*time.Second).Equal(wake)) -} - -func TestLPXDeadlineWaitsForSchedulingCycleStart(t *testing.T) { - t.Log("Observe an old pending request before the scheduler initializes its cycle timestamp") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "pending", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending) - request.Status.SchedulingStartedAt = nil - r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - - t.Log("Do not infer a deadline from creation time or the current wall clock") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, wake.IsZero()) -} - -func TestLPXDeadlineRestartsWhenReleasedRequestReturnsToPending(t *testing.T) { - t.Log("Observe a previously planned request in a new pending scheduling cycle") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - restarted := time.Now().UTC().Truncate(time.Second) - created := restarted.Add(-time.Hour) - request := deadlineTestRequest(dgd, pcs, "pending-again", created, lpxv1alpha1.RequestPhasePending) - request.Status.LastPlanRevision = 1 - request.Status.SchedulingStartedAt = ptr.To(metav1.NewTime(restarted)) - r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - - t.Log("Deadline the new cycle from the scheduler's reset timestamp") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, restarted.Add(30*time.Second).Equal(wake)) -} - -func TestLPXTerminalRequestHasNoDeadline(t *testing.T) { - t.Log("Observe a bound request") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "bound", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) - r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - - t.Log("Terminal scheduler disposition has neither expiry nor a wake") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, wake.IsZero()) -} - -func TestLPXCompletedSchedulingDoesNotRestartAfterDegradation(t *testing.T) { - t.Log("Observe a request whose initial scheduling completed before runtime degradation") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "degraded", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) - request.Status.Phase = lpxv1alpha1.RequestPhaseDegraded - r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, wake.IsZero()) -} - -func TestLPXDeadlineIgnoresRetiringRequests(t *testing.T) { - for _, test := range []struct { - name string - deleteRequest bool - deletePodClique bool - }{ - {name: "request is terminating", deleteRequest: true}, - {name: "owner PCS is terminating", deletePodClique: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Observe an expired request that intentional cleanup already owns") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "expired", time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending) - deletingAt := metav1.Now() - if test.deleteRequest { - request.Finalizers = []string{"scheduler.example/cleanup"} - request.DeletionTimestamp = &deletingAt - } - if test.deletePodClique { - pcs.Finalizers = []string{"grove.example/cleanup"} - pcs.DeletionTimestamp = &deletingAt - } - r := newLPXTestReconciler(t, nil, dgd, source, pcs, request) - - t.Log("Do not turn an intentional removal into a scheduling failure") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, deadlineTestDesired(request)) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, wake.IsZero()) - }) - } -} - -func TestLPXDeadlineListFailurePropagatesWithoutSyntheticWake(t *testing.T) { - t.Log("Fail the authoritative LPR list before any deadline can be derived") - dgd, source, _ := newLPXTestDGD(t, "single") - source.Spec.Scheduling = deadlineTestScheduling() - pcs := deadlineTestPCS(dgd, "pcs-uid") - r := newLPXTestReconciler(t, nil, dgd, source, pcs) - listErr := errors.New("request list unavailable") - r.apiReader = interceptor.NewClient(r.apiReader.(client.WithWatch), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return listErr - }, - }) - - t.Log("Return the real error so controller-runtime applies error backoff") - classification, wake, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, pcs, nil) - require.ErrorIs(t, err, listErr) - require.Nil(t, classification) - require.True(t, wake.IsZero()) - result, err := completeLPXDeadline(t.Context(), wake, ctrl.Result{}, err) - require.ErrorIs(t, err, listErr) - require.Zero(t, result) -} - -func TestLPXDeadlineFailureRequiresPostFailureGeneration(t *testing.T) { - t.Log("Record a deadline failure at the generation current when the failure occurred") - dgd, _, _ := newLPXTestDGD(t, "single") - dgd.Generation++ - dgd.Status.Conditions = []metav1.Condition{{ - Type: lpxSchedulingFailedCondition, - Status: metav1.ConditionTrue, - Reason: lpxSchedulingDeadlineExceededReason, - ObservedGeneration: dgd.Generation, - }} - require.True(t, lpxDeadlineFailureCurrent(dgd)) - - t.Log("Only a generation after the failure authorizes another publication") - dgd.Generation++ - require.False(t, lpxDeadlineFailureCurrent(dgd)) -} - -func TestLPXDeadlineRetainsOnlyRecordedRequestUIDs(t *testing.T) { - for _, uid := range []types.UID{"recorded", "replacement"} { - t.Run(string(uid), func(t *testing.T) { - t.Log("Retain exact expiry identity even without a current deadline, desired request or PCS") - dgd, source, _ := newLPXTestDGD(t, "single") - dgd.Status.ExpiredRequestUIDs = []types.UID{"recorded"} - pcs := deadlineTestPCS(dgd, "pcs-uid") - request := deadlineTestRequest(dgd, pcs, "same-name", time.Now(), lpxv1alpha1.RequestPhaseBound) - request.UID = uid - r := newLPXTestReconciler(t, nil, dgd, source, request) - request = getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name) - classification, _, err := r.reconcileLPXRequestDeadlines(t.Context(), dgd, source, nil, nil) - require.NoError(t, err) - if uid == "recorded" { - require.Zero(t, projectLPXLifecycleStatus(classification)) - require.Equal(t, []types.UID{uid}, dgd.Status.ExpiredRequestUIDs) - } else { - require.Positive(t, projectLPXLifecycleStatus(classification).RequeueAfter) - require.Nil(t, dgd.Status.ExpiredRequestUIDs) - } - require.Equal(t, request, getLPXRequest(t, t.Context(), r.Client, request.Namespace, request.Name)) - }) - } -} - -func TestProjectLPXDeadlineFailureAdvancesAStaleFailureFence(t *testing.T) { - dgd, _, _ := newLPXTestDGD(t, "single") - previous := dgd.Status.DeepCopy() - previous.Conditions = []metav1.Condition{{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, - Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, - LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)), - }} - dgd.Status = *previous.DeepCopy() - dgd.Generation++ - - t.Log("A newly expired request replaces the stale failure record") - projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{ - State: nvidiacomv1beta1.DGDStateFailed, Reason: lpxSchedulingDeadlineExceededReason, - Message: "expired", - }, true) - failed := meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition) - require.NotNil(t, failed) - require.Equal(t, dgd.Generation, failed.ObservedGeneration) - require.True(t, failed.LastTransitionTime.After(previous.Conditions[0].LastTransitionTime.Time)) -} - -func TestProjectLPXDeadlineFailureRemainsStickyAtRecordedGeneration(t *testing.T) { - t.Log("Record a deadline failure for the current generation") - dgd, _, _ := newLPXTestDGD(t, "single") - previous := dgd.Status.DeepCopy() - previous.Conditions = []metav1.Condition{{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, - Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, - }} - dgd.Status = *previous.DeepCopy() - - t.Log("Do not let a late successful observation authorize the same generation") - projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{ - State: nvidiacomv1beta1.DGDStateSuccessful, Reason: "Ready", Message: "ready", - }, false) - failed := meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition) - require.NotNil(t, failed) - require.Equal(t, metav1.ConditionTrue, failed.Status) - require.Equal(t, lpxSchedulingDeadlineExceededReason, failed.Reason) - - t.Log("A later shape edit cannot clear retry authorization while expired requests survive") - dgd.Generation++ - dgd.Status.ExpiredRequestUIDs = []types.UID{"expired"} - projectLPXSchedulingFailureCondition(dgd, previous, reconcileOutcome{Reason: lpxRetiringReason}, false) - require.Equal(t, previous.Conditions[0], *meta.FindStatusCondition(dgd.Status.Conditions, lpxSchedulingFailedCondition)) -} - -func TestCompleteLPXDeadlineBoundsTransientErrors(t *testing.T) { - reconcileErr := errors.New("temporary failure") - result, err := completeLPXDeadline(context.Background(), time.Now().Add(time.Minute), ctrl.Result{}, reconcileErr) - require.NoError(t, err) - require.Positive(t, result.RequeueAfter) - require.LessOrEqual(t, result.RequeueAfter, lpxRetirementRequeueAfter) - - result, err = completeLPXDeadline(context.Background(), time.Time{}, ctrl.Result{}, reconcileErr) - require.ErrorIs(t, err, reconcileErr) - require.Zero(t, result) -} - -func deadlineTestRequest(dgd *nvidiacomv1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { - generation := int64(1) - schedulingStartedAt := metav1.NewTime(created.UTC().Truncate(time.Second)) - request := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: dgd.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", - Generation: generation, CreationTimestamp: metav1.NewTime(created), - Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, - Annotations: map[string]string{lpxDeploymentUIDAnnotation: string(dgd.UID)}, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }, - Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, - Status: &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: phase, ObservedGeneration: &generation, - SchedulingStartedAt: &schedulingStartedAt, - }, - } - if phase == lpxv1alpha1.RequestPhaseBound { - request.Status.LastPlanRevision = 1 - request.Status.Committed = &lpxv1alpha1.Committed{ - Execution: lpxv1alpha1.CommittedExecution{AcceptedGeneration: &generation, NodeLocal: &lpxv1alpha1.NodeLocalExecution{}}, - Plan: lpxv1alpha1.CommittedPlan{ - PlannedFromGeneration: generation, Revision: 1, PlanDigest: "sha256:plan", - Placement: lpxv1alpha1.PlanPlacement{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, NodeLocal: &lpxv1alpha1.NodeLocalPlacement{}}, - }, - } - } - return request -} - -func deadlineTestDesired(requests ...*lpxv1alpha1.LPUPipelineRequest) []lpxModelMaterializing { - desired := make([]lpxModelMaterializing, 0, len(requests)) - for _, request := range requests { - desired = append(desired, lpxModelMaterializing{requestName: request.Name}) - } - return desired -} - -func deadlineTestPCS(dgd *nvidiacomv1alpha1.LPXGraphDeployment, uid types.UID) *grovev1alpha1.PodCliqueSet { - return &grovev1alpha1.PodCliqueSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.PCSNameForLPX(dgd), Namespace: dgd.Namespace, UID: uid, ResourceVersion: "1", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, - }, - Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, - } -} - -func deadlineTestScheduling() *nvidiacomv1beta1.SchedulingSpec { - return &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} -} diff --git a/deploy/operator/internal/controller/lpx/component_roles_test.go b/deploy/operator/internal/controller/lpx/component_roles_test.go deleted file mode 100644 index 07cfecd0d9c8..000000000000 --- a/deploy/operator/internal/controller/lpx/component_roles_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "testing" - - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { - t.Log("Materialize two draft instances and one target in their single shared scaling group") - deployment, source, registry := newLPXSpecDecodeTestDGD(t) - prepare, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), deployment, source) - objects := lpxMaterializedObjects(t, prepare, deployment, source, selected) - for _, object := range objects { - switch live := object.(type) { - case *grovev1alpha1.PodCliqueScalingGroup: - live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 - live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 - case *grovev1alpha1.PodClique: - live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas - live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas - } - } - pcs := findLPXTestPodCliqueSet(t, objects) - - t.Log("Observe each draft condition against an independent copy of the ready baseline") - for _, scenario := range []string{"ready", "partial", "missing", "foreign", "old revision", "unobserved"} { - t.Run(scenario, func(t *testing.T) { - t.Log("Seed isolated Grove observations and fetch the first draft") - kubeClient := fake.NewClientBuilder().WithScheme(prepare.Scheme()).Build() - createLPXTestObjects(t, t.Context(), kubeClient, objects...) - firstDraft := findLPXTestClique(t, objects, selected.plan.Agents[0].CliqueName).DeepCopy() - require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(firstDraft), firstDraft)) - require.Equal(t, "draft", firstDraft.Labels[lpx.StageLabel]) - - t.Log("Make only the first draft incomplete while retaining a fully ready second draft and target") - switch scenario { - case "partial": - firstDraft.Status.ReadyReplicas-- - case "foreign": - firstDraft.OwnerReferences = []metav1.OwnerReference{{UID: "foreign", Controller: ptr.To(true)}} - case "old revision": - firstDraft.Status.CurrentPodCliqueSetGenerationHash = ptr.To("old") - case "unobserved": - firstDraft.Status.ObservedGeneration = ptr.To(firstDraft.Generation - 1) - } - if scenario == "missing" { - require.NoError(t, kubeClient.Delete(t.Context(), firstDraft)) - } else { - require.NoError(t, kubeClient.Update(t.Context(), firstDraft)) - } - - t.Log("Project logical draft counts and make both authored components await the complete pair") - readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, source, deployment, pcs) - require.NoError(t, err) - require.Len(t, readiness.ComponentStatuses, 2) - draft, target := readiness.ComponentStatuses["draft"], readiness.ComponentStatuses["lpx"] - require.Equal(t, v1beta1.ComponentKindPodClique, draft.ComponentKind) - require.Len(t, draft.ComponentNames, 2) - require.Equal(t, int32(1), target.Replicas) - if scenario == "ready" { - require.True(t, readiness.Ready) - require.Equal(t, int32(2), draft.Replicas) - require.Equal(t, int32(2), draft.UpdatedReplicas) - require.Equal(t, ptr.To(int32(2)), draft.ReadyReplicas) - require.Equal(t, ptr.To(int32(2)), draft.ScheduledReplicas) - require.Equal(t, ptr.To(int32(1)), target.AvailableReplicas) - } else { - require.False(t, readiness.Ready) - require.Equal(t, ptr.To(int32(1)), draft.ReadyReplicas) - require.Equal(t, ptr.To(int32(0)), target.AvailableReplicas) - } - require.Equal(t, readiness.Ready, draft.Ready) - require.Equal(t, readiness.Ready, target.Ready) - }) - } -} diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 12c6ac168e20..32655331bfad 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -7,7 +7,7 @@ import ( "context" "errors" "fmt" - "maps" + "slices" "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" @@ -17,550 +17,452 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" ) -const legacyLPXGraphDeploymentFinalizer = "nvidia.com/lpx-graph-deployment" - -func lpxSourceKey(deployment *v1alpha1.LPXGraphDeployment) (client.ObjectKey, error) { - owner := metav1.GetControllerOf(deployment) - if owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" || owner.Name == "" { - return client.ObjectKey{}, fmt.Errorf("LPXGraphDeployment requires a DynamoGraphDeployment controller owner") - } - return client.ObjectKey{Namespace: deployment.Namespace, Name: owner.Name}, nil -} - -func (r *graphReconciler) resolveLPXSource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) (*v1beta1.DynamoGraphDeployment, *lpxRejected, error) { - sourceKey, err := lpxSourceKey(deployment) - if err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil - } - source := &v1beta1.DynamoGraphDeployment{} - if err := r.Get(ctx, sourceKey, source); err != nil { - if apierrors.IsNotFound(err) { - return nil, &lpxRejected{reason: "The exact source DGD no longer exists"}, nil - } - return nil, nil, err - } - if err := dynamo.ValidateLPXSource(deployment, source); err != nil { - return source, nil, err - } - return source, nil, nil -} - -// graphReconciler owns the complete compiled engine lifecycle. -// The source DGD supplies intent, never an alternate status or owner identity. +// graphReconciler reconciles LPXGraphDeployments using their owning DGD's configuration. type graphReconciler struct { - client.Client + enabled bool recorder events.EventRecorder - apiReader client.Reader runtimeConfig *commoncontroller.RuntimeConfig modelRegistry lpx.ModelRegistry - Config *configv1alpha1.OperatorConfiguration - DockerSecretRetriever dynamo.SecretsRetriever -} + config *configv1alpha1.OperatorConfiguration + dockerSecretRetriever dynamo.SecretsRetriever -func (r *graphReconciler) GetRecorder() events.EventRecorder { - return r.recorder + client.Client } // Setup registers the LPX controller and its dependencies. func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runtimeConfig *commoncontroller.RuntimeConfig, secrets dynamo.SecretsRetriever) error { - // Disabled integration still serves child status; deletion uses owner garbage collection. r := &graphReconciler{ - Client: mgr.GetClient(), + // Disabled integration still serves child status; deletion uses owner garbage collection. + enabled: integrationDisabledMessage(runtimeConfig) == "", recorder: mgr.GetEventRecorder("lpxgraphdeployment"), - apiReader: mgr.GetAPIReader(), runtimeConfig: runtimeConfig, - Config: config, - DockerSecretRetriever: secrets, + config: config, + dockerSecretRetriever: secrets, + Client: mgr.GetClient(), } - // Resolve availability before activating any runtime dependencies. - enabled := r.unavailableReason() == "" - if enabled { + if r.enabled { var err error r.modelRegistry, err = newLPXModelRegistry(config) if err != nil { return err } - } - if err := r.setupWithManager(mgr); err != nil { - return fmt.Errorf("unable to create LPXGraphDeployment controller: %w", err) - } - if enabled { if err := setupLPUEviction(mgr); err != nil { return fmt.Errorf("unable to create LPU Eviction controller: %w", err) } } + + if err := r.setupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create LPXGraphDeployment controller: %w", err) + } + return nil } -// unavailableReason owns the LPX activation policy; an empty reason means enabled. -// Status remains available regardless of this policy. -func (r *graphReconciler) unavailableReason() string { - if !r.runtimeConfig.Gate.Enabled(features.Grove) { - return "Grove is disabled" - } - if !r.runtimeConfig.Gate.Enabled(features.LPX) { - return "LPX integration is disabled" - } - return "" +func (r *graphReconciler) GetRecorder() events.EventRecorder { + return r.recorder } // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update -// +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups=grove.io,resources=podcliquesets/finalizers,verbs=update +// +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete +// Reconcile observes dependencies once, then persists the final child status. +// Cached misses and write conflicts are retried; no APIReader is used. +// These observations are not an atomic snapshot; watches converge later edits. func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) { deployment := &v1alpha1.LPXGraphDeployment{} if err := r.Get(ctx, req.NamespacedName, deployment); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } if !deployment.DeletionTimestamp.IsZero() { - if controllerutil.RemoveFinalizer(deployment, legacyLPXGraphDeploymentFinalizer) { - return ctrl.Result{}, r.Update(ctx, deployment) - } return ctrl.Result{}, nil } - unavailableReason := r.unavailableReason() - previous := deployment.Status.DeepCopy() - if deployment.Status.ObservedGeneration != deployment.Generation { - deployment.Status.Components = nil - deployment.Status.ModelDownload = nil - } - state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} - var source *v1beta1.DynamoGraphDeployment - var pcs *grovev1alpha1.PodCliqueSet - var selected *lpxMaterializing - defer func() { - result, err = r.completeReconcile(ctx, deployment, previous, lpxReconcileCompletion{ - unavailableReason: unavailableReason, - state: state, - source: source, - pcs: pcs, - selected: selected, - result: result, - reconcileErr: err, - }) - }() + // Persist the workload outcome before converting errors into deadline retries. + defer func(previous *v1alpha1.LPXGraphDeploymentStatus) { + if err != nil { + setReadyCondition(deployment, v1beta1.DGDStateFailed, err.Error()) + } else { + deployment.Status.ObservedGeneration = deployment.Generation + } - // Like Grove, disabling integration reports unavailability without touching workloads. - if unavailableReason != "" { - state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXUnavailable", unavailableReason + if statusErr := r.updateStatus(ctx, deployment, previous); statusErr != nil { + err = errors.Join(err, statusErr) + result = ctrl.Result{} + return + } + + // Workload errors retain a retry delay only while a scheduling deadline is active. + if err != nil && result.RequeueAfter > 0 { + log.FromContext(ctx).Error(err, "retrying reconciliation under the pipeline request scheduling deadline") + err = nil + } + }(deployment.Status.DeepCopy()) + + if !r.enabled { + setReadyCondition(deployment, v1beta1.DGDStateFailed, integrationDisabledMessage(r.runtimeConfig)) return ctrl.Result{}, nil } - var rejected *lpxRejected - var sourceErr error - source, rejected, sourceErr = r.resolveLPXSource(ctx, deployment) - if sourceErr != nil { - return ctrl.Result{}, sourceErr - } - pcs, err = r.observeCurrentLPXPodCliqueSet(ctx, deployment) + // DGD identity and input revision are checked before any workload writes. + dgd, err := getDynamoGraphDeployment(ctx, r.Client, deployment) if err != nil { - pcs = nil return ctrl.Result{}, err } - // Invalid source authority retires existing LPX state before reporting rejection. - if rejected != nil { - retiring, retireErr := r.retireInvalidLPXWorkload(ctx, deployment, pcs, rejected.reason) - if retireErr != nil { - return ctrl.Result{}, retireErr - } - if retiring != nil { - state = lpxResult(retiring) - return projectLPXLifecycleStatus(retiring), nil - } - state = lpxResult(rejected) + + // The parent owns deselection/deletion; a cache miss or delayed handoff is not cleanup authority. + if dgd == nil { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the matching DynamoGraphDeployment") return ctrl.Result{}, nil } - var classification lpxClassification - selected, classification, err = r.reconcileLPXSafetyPreflight(ctx, deployment, source, pcs, nil) - // An uncached successor must reach the download gate before snapshot resolution can succeed. - if err != nil && !errors.Is(err, lpx.ErrBuildSnapshotAcquisition) { - return ctrl.Result{}, err - } - if classification != nil { - state = lpxResult(classification) - return projectLPXLifecycleStatus(classification), nil - } - ready, err := r.reconcileModelDownloads(ctx, deployment, source, selected == nil) + + // Resolve the owned Grove hierarchy from one cache observation. + pcs, err := getPodCliqueSet(ctx, r.Client, deployment) if err != nil { return ctrl.Result{}, err } - if !ready { - state.Reason, state.Message = modelDownloadPendingReason, modelDownloadPendingMessage - return ctrl.Result{RequeueAfter: modelDownloadRequeueAfter}, nil + + if pcs != nil && !pcs.DeletionTimestamp.IsZero() { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for PodCliqueSet garbage collection") + return ctrl.Result{}, nil } - if selected == nil { - selected, classification, err = r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source, pcs) + + var ( + pcsg *grovev1alpha1.PodCliqueScalingGroup + requests []lpxv1alpha1.LPUPipelineRequest + ) + + if pcs != nil { + // Foreground PCS replacement waits for its blocking LPRs; discover them only by owner UID. + requests, err = r.getPipelineRequests(ctx, pcs) if err != nil { return ctrl.Result{}, err } - if classification != nil { - state = lpxResult(classification) - return projectLPXLifecycleStatus(classification), nil + + pcsg, err = getPodCliqueScalingGroup(ctx, r.Client, pcs) + if err != nil { + return ctrl.Result{}, err } } - state, result, err = r.reconcileWorkload(ctx, deployment, source, selected, pcs) - return result, err -} -type lpxReconcileCompletion struct { - unavailableReason string - state reconcileOutcome - source *v1beta1.DynamoGraphDeployment - pcs *grovev1alpha1.PodCliqueSet - selected *lpxMaterializing - result ctrl.Result - reconcileErr error + return r.reconcileWorkload(ctx, deployment, dgd, pcs, pcsg, requests) } -// completeReconcile projects deadlines and the final lifecycle outcome after -// every reconcile path has finished mutating its in-memory status. -func (r *graphReconciler) completeReconcile( +// reconcileWorkload consumes owned observations; DGD and deployment are non-nil. +// A nil pcs means initial creation (pcsg is nil and requests is empty). A nil pcsg +// means no current, non-deleting group is observed, not zero capacity. A non-nil +// pcsg is owned by pcs; only its live Spec.Replicas controls existing engine capacity. +// On error, a nonzero RequeueAfter requests a bounded deadline retry; Reconcile +// records the error and persists status before returning that retry to controller-runtime. +func (r *graphReconciler) reconcileWorkload( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, - previous *v1alpha1.LPXGraphDeploymentStatus, - completion lpxReconcileCompletion, -) (ctrl.Result, error) { - state, result, err := completion.state, completion.result, completion.reconcileErr - deadlineAt := time.Time{} - - // Keep active-request deadlines running while unrelated request cleanup converges. - deadlineExceeded := false - recordDeadlineFailure := false - requestScopedRetirement := state.Reason == lpxRetiringReason && state.retirementScope == lpxRequestRetirement - if completion.unavailableReason == "" && completion.source != nil && - (completion.selected != nil || len(deployment.Status.ExpiredRequestUIDs) > 0) && - (state.Reason != lpxRetiringReason || requestScopedRetirement) { - // Recorded cleanup survives errors preparing the next workload; only new clocks need selection. - var desired []lpxModelMaterializing - if completion.selected != nil { - desired = completion.selected.requests - } - classification, wake, deadlineErr := r.reconcileLPXRequestDeadlines( - ctx, deployment, completion.source, completion.pcs, desired, - ) - deadlineAt, err = wake, errors.Join(err, deadlineErr) - if classification != nil { - state = lpxResult(classification) - // Preserve independent work such as polling an unfinished model download. - if deadlineResult := projectLPXLifecycleStatus(classification); deadlineResult.RequeueAfter > 0 { - result = deadlineResult - } - deadlineExceeded, recordDeadlineFailure = lpxDeadlineFlags(classification) - } + dgd *v1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + pcsg *grovev1alpha1.PodCliqueScalingGroup, + requests []lpxv1alpha1.LPUPipelineRequest, +) (result ctrl.Result, err error) { + if result, err := r.reconcileModelDownloads(ctx, deployment, dgd); err != nil || result.RequeueAfter > 0 { + return result, err } + + workload, err := lpx.ResolveSelectedWorkload(ctx, dgd, r.modelRegistry) if err != nil { - if !deadlineExceeded { - state.State, state.Reason, state.Message = v1beta1.DGDStateFailed, "LPXReconciliationFailed", truncateLPXMessage(err.Error()) - } - } else if completion.unavailableReason == "" { - deployment.Status.ObservedGeneration = deployment.Generation - } - ready, failed := metav1.ConditionFalse, metav1.ConditionFalse - if state.State == v1beta1.DGDStateSuccessful { - ready = metav1.ConditionTrue - } - if state.State == v1beta1.DGDStateFailed { - failed = metav1.ConditionTrue - } - // Every authored component observes the complete shared runtime gate. - for name, component := range deployment.Status.Components { - component.Ready = ready == metav1.ConditionTrue - deployment.Status.Components[name] = component - } - for condition, value := range map[string]metav1.ConditionStatus{"Ready": ready, "Failed": failed} { - meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{Type: condition, Status: value, ObservedGeneration: deployment.Generation, Reason: state.Reason, Message: state.Message}) - } - projectLPXSchedulingFailureCondition(deployment, previous, state, recordDeadlineFailure) - if !apiequality.Semantic.DeepEqual(previous, &deployment.Status) { - if statusErr := r.Status().Update(ctx, deployment); statusErr != nil { - err = errors.Join(err, fmt.Errorf("persist LPX lifecycle status: %w", statusErr)) - } - } - if ready == metav1.ConditionTrue && deployment.Status.ModelDownload != nil && deployment.Status.ModelDownload.LastCheckedAt != nil && result.RequeueAfter == 0 { - result.RequeueAfter = max(modelDownloadRequeueAfter, time.Until(deployment.Status.ModelDownload.LastCheckedAt.Add(modelDownloadRefreshInterval))) + return ctrl.Result{}, err } - return completeLPXDeadline(ctx, deadlineAt, result, err) -} -// reconcileWorkload publishes and observes the selected engine after source, -// deadline, placement, and model-download gates have passed. deployment, source -// and selected are non-nil and supplied by that preflight. pcs is nil when its -// stable name was not found before preflight. -func (r *graphReconciler) reconcileWorkload( - ctx context.Context, - deployment *v1alpha1.LPXGraphDeployment, - source *v1beta1.DynamoGraphDeployment, - selected *lpxMaterializing, - pcs *grovev1alpha1.PodCliqueSet, -) (reconcileOutcome, ctrl.Result, error) { - state := reconcileOutcome{State: v1beta1.DGDStatePending, Reason: "LPXPending", Message: "Waiting for the current LPX engine"} - desired, resources, err := renderPodCliqueSet(ctx, source, r.Config, r.runtimeConfig, r.DockerSecretRetriever, selected.workload, selected.plan, deployment) + // Validate the complete replacement before either immutable change can delete the running PCS. + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) if err != nil { - return state, ctrl.Result{}, err - } - if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { - return state, ctrl.Result{}, err - } - // Replica scale-down keeps the immutable PCS shape, so lower the live Grove - // group before deleting stale requests. A workload-shape change replaces the - // PCS instead and must not try to mutate its old group. - scaled := false - if pcs != nil && pcs.DeletionTimestamp.IsZero() && pcs.Annotations[lpx.WorkloadDigestAnnotation] == selected.workloadDigest { - if err := commoncontroller.CheckControllerOwnership(pcs, deployment, r.Scheme()); err != nil { - return state, ctrl.Result{}, err - } - scaled, err = r.reconcileLPXScalingGroupScale(ctx, source, pcs, selected.plan.LPXScalingGroup, true) - if err != nil { - return state, ctrl.Result{}, err - } + return ctrl.Result{}, err } - currents, classification, err := r.reconcileLPXPublicationFence(ctx, deployment, selected, pcs) + desiredPCS, resources, err := r.renderPodCliqueSet(ctx, deployment, dgd, workload, plan) if err != nil { - return state, ctrl.Result{}, err - } - if classification != nil { - state = lpxResult(classification) - return state, projectLPXLifecycleStatus(classification), nil + return ctrl.Result{}, err } - // The existing PCS template only seeds its PCSG; preserve that seed while - // the live group scales independently, including to zero. - if err := preserveLPXScalingGroupReplicaSeed(desired, pcs); err != nil { - return state, ctrl.Result{}, err + + // Model composition changes require replacement: Grove makes clique membership immutable. + // Replica-only changes, including top-level PCS replica drift, use normal mutable sync. + if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != workload.Digest().String() { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") + return ctrl.Result{}, r.deletePodCliqueSet(ctx, pcs) } - for _, resource := range resources { - if err := r.syncLPXResource(ctx, deployment, resource); err != nil { - return state, ctrl.Result{}, err + + // Explicit DGD replicas authorize scale writes; omission leaves the live count externally managed. + explicitReplicas := lpx.ServingComponent(dgd).Replicas + if explicitReplicas == nil && pcs != nil { + // Do not derive an externally managed request set from an unknown replica count. + if pcsg == nil { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the LPX scaling group") + return ctrl.Result{}, nil + } + + plan.Replicas = pcsg.Spec.Replicas + if err := plan.ValidateReplicaCount(); err != nil { + return ctrl.Result{}, err } } - changed, synced, err := commoncontroller.SyncObservedResource( - ctx, - r, - deployment, - pcs, - desired, - commoncontroller.WithPreservedListOrder(), - ) - if err != nil { - return state, ctrl.Result{}, err + + // Preserve matching scheduler receipts and collect new LPRs in publication order. + desiredRequests, missingRequests, intentChanged := resolvePipelineRequests(deployment, requests, workload, plan) + if intentChanged { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") + return ctrl.Result{}, r.deletePodCliqueSet(ctx, pcs) } - if err := r.reconcileEndpoint(ctx, deployment, source); err != nil { - return state, ctrl.Result{}, err + + // Schedule deadline wakes without consuming errors needed by the status boundary. + expiredRequests, deadlineAt := pipelineRequestDeadlines(desiredRequests, pipelineRequestDeadlineSeconds(dgd)) + defer func() { + result = requeueForPipelineRequestDeadline(deadlineAt, result, err) + }() + + // Apply explicit scale-in before deadline failures can block publication. + if err := r.scaleDownPodCliqueScalingGroup(ctx, pcsg, explicitReplicas); err != nil { + return ctrl.Result{}, err } - // Apply scale-out, or scale-down not already persisted before request cleanup. - if !scaled { - if _, err := r.reconcileLPXScalingGroupScale(ctx, source, synced, selected.plan.LPXScalingGroup, false); err != nil { - return state, ctrl.Result{}, err + + // Deadline failures stop publication even when no timer or cleanup is needed. + if len(expiredRequests) > 0 || isSchedulingFailedConditionCurrent(deployment) { + // Explicit or external scale-in has already lowered capacity; failure must not prevent its LPR cleanup. + if pcsg != nil { + if err := r.deletePipelineRequests(ctx, pipelineRequestsPendingDeletion(requests, desiredRequests)); err != nil { + return ctrl.Result{}, err + } + + // Do not delete those names again during deadline cleanup or mutate the expired requests' backing slice. + requests = slices.DeleteFunc(slices.Clone(requests), func(request lpxv1alpha1.LPUPipelineRequest) bool { + _, desired := desiredRequests[request.Name] + return !desired + }) } + return r.reconcilePipelineRequestDeadline(ctx, deployment, pcsg, requests, expiredRequests, explicitReplicas != nil) } - readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, source, deployment, synced) - if err != nil { - return state, ctrl.Result{}, err + + clearPreviousSchedulingFailure(deployment) + + // Existing LPRs require an observed group so scale-down precedes their deletion. + // Without LPRs, initial PCS synchronization can proceed before Grove creates the group. + if pcsg == nil && len(requests) > 0 { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the LPX scaling group") + return ctrl.Result{}, nil } - deployment.Status.Components = readiness.ComponentStatuses - state.Reason, state.Message = readiness.Classification, readiness.Message - if readiness.Ready && !changed { - if err := r.deleteStaleLPXConfigMaps(ctx, deployment, resources); err != nil { - return state, ctrl.Result{}, err - } - state.State = v1beta1.DGDStateSuccessful + + // Deletion is terminal for this pass; LPR events resume publication after cleanup. + if removed := pipelineRequestsPendingDeletion(requests, desiredRequests); len(removed) > 0 { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for removed LPX requests to finish deletion") + return ctrl.Result{}, r.deletePipelineRequests(ctx, removed) } - classification, err = r.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, selected, currents) + + modified, _, err := commoncontroller.SyncObservedResource(ctx, r, deployment, pcs, desiredPCS, commoncontroller.WithPreservedListOrder(), commoncontroller.WithMetadataSync()) if err != nil { - return state, ctrl.Result{}, err + if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the PodCliqueSet cache observation") + return ctrl.Result{}, nil + } + + return ctrl.Result{}, err } - state = overlayLPXResult(state, classification) - return state, projectLPXLifecycleStatus(classification), nil -} -// preserveLPXScalingGroupReplicaSeed mutates nonnil desired to retain the -// observed PCS template's scale-owned replica seed. observed is nil only when -// the first PCS has not been created yet. -func preserveLPXScalingGroupReplicaSeed( - desired *grovev1alpha1.PodCliqueSet, - observed *grovev1alpha1.PodCliqueSet, -) error { - if observed == nil { - return nil + if err := r.reconcileRuntimeResources(ctx, deployment, dgd, resources); err != nil { + return ctrl.Result{}, err } - // LPX renders and owns exactly one complete-engine scaling group. - desiredConfigs := desired.Spec.Template.PodCliqueScalingGroupConfigs - observedConfigs := observed.Spec.Template.PodCliqueScalingGroupConfigs - if len(desiredConfigs) != 1 || len(observedConfigs) != 1 { - return fmt.Errorf("LPX PodCliqueSet %q requires exactly one scaling-group template", desired.Name) + // Publication requires an observed PCS spec and its current, non-deleting group. + if modified || pcsg == nil { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for Grove to observe the workload") + return ctrl.Result{}, nil } - if desiredConfigs[0].Name != observedConfigs[0].Name { - return fmt.Errorf("LPX PodCliqueSet %q scaling-group template changed from %q to %q", - desired.Name, observedConfigs[0].Name, desiredConfigs[0].Name) + + // Scale-out proceeds after deleting old names, without waiting for Pods. + if err := r.scalePodCliqueScalingGroup(ctx, pcsg, explicitReplicas); err != nil { + return ctrl.Result{}, err } - // Copy the pointer value so desired never aliases the observed object. - desiredConfigs[0].Replicas = nil - if observedConfigs[0].Replicas != nil { - replicas := *observedConfigs[0].Replicas - desiredConfigs[0].Replicas = &replicas + // Publication is terminal for this pass; readiness needs watched scheduler receipts. + if len(missingRequests) > 0 { + return ctrl.Result{}, r.reconcilePipelineRequests(ctx, deployment, pcs, missingRequests) } - return nil -} -// reconcileLPXScalingGroupScale applies explicit component replicas to the -// Grove group. Grove seeds this group once, so later replica changes use its -// scale subresource without rewriting the PCS template. pcs is nil when the -// desired PodCliqueSet was not found, in which case there is no group to scale. -func (r *graphReconciler) reconcileLPXScalingGroupScale( - ctx context.Context, - source *v1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - groupName string, - scaleDownOnly bool, -) (bool, error) { - replicas := lpx.ServingComponent(source).Replicas - if replicas == nil || pcs == nil { - return false, nil - } - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: groupName}, group); err != nil { - if apierrors.IsNotFound(err) { - return false, nil + defer func() { + if err == nil { + err = r.deleteUnusedConfigMaps(ctx, deployment, resources) } - return false, err - } - if !metav1.IsControlledBy(group, pcs) { - return false, fmt.Errorf("LPX scaling group lacks the current PCS owner") - } - if !group.DeletionTimestamp.IsZero() { - return false, nil - } - if group.Spec.Replicas == *replicas || scaleDownOnly && group.Spec.Replicas < *replicas { - return false, nil - } - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, - } - if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { - return false, err - } - return true, nil -} + }() -// projectLPXLifecycleStatus transfers reconcile-owned results to the same child before returning. -func projectLPXLifecycleStatus(classification lpxClassification) ctrl.Result { - if deadline, ok := classification.(*lpxDeadlineExceeded); ok { - return ctrl.Result{RequeueAfter: deadline.requeueAfter} - } - if _, retiring := classification.(*lpxRetiring); retiring { - return ctrl.Result{RequeueAfter: lpxRetirementRequeueAfter} - } - return ctrl.Result{} + return r.reconcileReadiness(ctx, deployment, dgd, pcs, pcsg, desiredRequests) } -func lpxDeadlineFlags(classification lpxClassification) (bool, bool) { - deadline, ok := classification.(*lpxDeadlineExceeded) - if !ok { - return false, false +// scaleDownPodCliqueScalingGroup lowers capacity before asynchronous request cleanup. +// A nil pcsg defers scaling until the group is observed; a non-nil pcsg is owned +// and not deleting. Nil replicas leaves capacity externally managed. Scale-out +// must wait until terminating request names disappear. +func (r *graphReconciler) scaleDownPodCliqueScalingGroup(ctx context.Context, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas *int32) error { + if pcsg == nil || replicas == nil || *replicas >= pcsg.Spec.Replicas { + return nil } - return true, deadline.recordFailure + return r.scalePodCliqueScalingGroup(ctx, pcsg, replicas) } -// Publication is an identity-sensitive write boundary: an uncached read fences -// a source/child replacement or input edit, not an attempt to repair informer lag. -func (r *graphReconciler) validateLPXPublicationSource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return err +// scalePodCliqueScalingGroup updates the non-nil, already-owned PCSG using its +// observed resource version. Nil replicas leaves capacity to external autoscaling. +// A successful write updates the supplied observation for the rest of this pass. +func (r *graphReconciler) scalePodCliqueScalingGroup(ctx context.Context, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas *int32) error { + if replicas == nil || pcsg.Spec.Replicas == *replicas { + return nil } - source := &v1beta1.DynamoGraphDeployment{} - sourceKey, err := lpxSourceKey(deployment) - if err != nil { - return err + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: pcsg.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, } - if err := r.apiReader.Get(ctx, sourceKey, source); err != nil { - return err + if err := r.SubResource("scale").Update(ctx, pcsg, client.WithSubResourceBody(scale)); err != nil { + return fmt.Errorf("scale LPX PodCliqueScalingGroup %q: %w", pcsg.Name, err) } - return dynamo.ValidateLPXSource(deployment, source) + pcsg.Spec.Replicas = *replicas + return nil } -// validateLPXDeploymentAuthority fences writes against a replaced or newer child, -// independently of whether its source remains valid for publication. -func (r *graphReconciler) validateLPXDeploymentAuthority(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment) error { - current := &v1alpha1.LPXGraphDeployment{} - if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(deployment), current); err != nil { - return err - } - if current.UID != deployment.UID || current.ResourceVersion != deployment.ResourceVersion || !current.DeletionTimestamp.IsZero() { - return fmt.Errorf("LPXGraphDeployment authority changed before LPX resource mutation") +// reconcileRuntimeResources synchronizes native rendered resources and the +// serving endpoint. deployment and DGD are non-nil. +func (r *graphReconciler) reconcileRuntimeResources(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, resources []client.Object) error { + for _, resource := range resources { + if _, _, err := commoncontroller.SyncResource(ctx, r, deployment, func(context.Context) (client.Object, bool, error) { + return resource, false, nil + }); err != nil { + return err + } } - return nil + return r.reconcileEndpoint(ctx, deployment, dgd) } -func (r *graphReconciler) syncLPXResource(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, desired client.Object) error { - // Observe the resource once and reject foreign ownership before preparing a write. - live := desired.DeepCopyObject().(client.Object) - if err := r.Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil { +func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment) error { + // One serving endpoint belongs to the engine, independently of mutable component names. + pcsName := dynamo.PCSNameForLPX(deployment) + serviceName := pcsName + "-serve" + backend := commoncontroller.GetDiscoveryBackend(r.config.Discovery.Backend, dgd.Annotations) + + observed := &corev1.Service{} + if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: serviceName}, observed); err != nil { if !apierrors.IsNotFound(err) { return err } - live = nil - } else if !metav1.IsControlledBy(live, deployment) { - return fmt.Errorf("refusing to adopt LPX resource %s/%s without its exact owner", live.GetNamespace(), live.GetName()) + observed = nil } - // Keep the endpoint's API-allocated addresses when replacing its rendered spec. - if service, ok := desired.(*corev1.Service); ok && live != nil { - observed := live.(*corev1.Service) + // Switching discovery backends removes only this deployment's endpoint. + if backend != configv1alpha1.DiscoveryBackendKubernetes { + if observed == nil { + return nil + } + if err := commoncontroller.CheckControllerOwnership(observed, deployment, r.Scheme()); err != nil { + return err + } + return client.IgnoreNotFound(r.Delete(ctx, observed, client.Preconditions{UID: &observed.UID, ResourceVersion: &observed.ResourceVersion})) + } + + component := lpx.ServingComponent(dgd) + service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ + ServiceName: serviceName, Namespace: deployment.Namespace, + ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, + DynamoNamespace: dgd.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, + Labels: dynamo.GetDGDComponentResourceLabels(dgd, component.ComponentName, component), + Annotations: dynamo.GetDGDComponentResourceAnnotations(dgd, component.ComponentName, component), + }) + if err != nil { + return err + } + // Keep cleanup and pod selection scoped to this materialization. + service.Labels[deploymentUIDLabel] = string(deployment.UID) + service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue + service.Spec.Selector[grovecommon.LabelPartOfKey] = pcsName + + // Preserve API-allocated addresses and IP families across selector updates. + if observed != nil { service.Spec.ClusterIP = observed.Spec.ClusterIP service.Spec.ClusterIPs = observed.Spec.ClusterIPs service.Spec.IPFamilies = observed.Spec.IPFamilies service.Spec.IPFamilyPolicy = observed.Spec.IPFamilyPolicy } + _, _, err = commoncontroller.SyncObservedResource(ctx, r, deployment, observed, service, commoncontroller.WithMetadataSync()) + return err +} - // Prepare LPX-owned metadata while preserving shared synchronization's bookkeeping. - metadataChanged := false - if live != nil { - labels, annotations := desired.GetLabels(), maps.Clone(desired.GetAnnotations()) - if annotations == nil { - annotations = make(map[string]string) - } - for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { - annotations[key] = live.GetAnnotations()[key] - } - metadataChanged = !maps.Equal(labels, live.GetLabels()) || !maps.Equal(annotations, live.GetAnnotations()) - if metadataChanged { - live.SetLabels(maps.Clone(labels)) - live.SetAnnotations(annotations) - } +// reconcileReadiness combines scheduler receipts with Grove runtime readiness. +// All pointer inputs are non-nil and refer to the same observed workload. +func (r *graphReconciler) reconcileReadiness( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + pcsg *grovev1alpha1.PodCliqueScalingGroup, + requests map[string]*lpxv1alpha1.LPUPipelineRequest, +) (ctrl.Result, error) { + readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, dgd, pcs, pcsg) + if err != nil { + return ctrl.Result{}, err } - // Content updates carry prepared metadata; only metadata-only drift needs another path. - modified, synced, err := commoncontroller.SyncObservedResource(ctx, r, deployment, live, desired) - if err != nil || modified || !metadataChanged { - return err + deployment.Status.Components = readiness.ComponentStatuses + + if !setPipelineRequestReadyCondition(deployment, requests) { + return ctrl.Result{}, nil } - return r.Update(ctx, synced) + + if !readiness.Ready { + setReadyCondition(deployment, v1beta1.DGDStatePending, readiness.Message) + return ctrl.Result{}, nil + } + + setReadyCondition(deployment, v1beta1.DGDStateSuccessful, readiness.Message) + + delay := modelDownloadRefreshInterval + if download := deployment.Status.ModelDownload; download != nil && download.LastCheckedAt != nil { + delay = max(modelDownloadRequeueAfter, time.Until(download.LastCheckedAt.Add(modelDownloadRefreshInterval))) + } + + return ctrl.Result{RequeueAfter: delay}, nil +} + +// deletePodCliqueSet deletes the already-validated, non-nil PCS. +func (r *graphReconciler) deletePodCliqueSet(ctx context.Context, pcs *grovev1alpha1.PodCliqueSet) error { + return client.IgnoreNotFound( + r.Delete( + ctx, + pcs, + // Foreground GC so that blocking LPRs and Grove dependents can be removed. + client.PropagationPolicy(metav1.DeletePropagationForeground), + client.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, + ), + ) } -func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, resources []client.Object) error { +// deleteUnusedConfigMaps runs only after readiness so existing pods keep their +// immutable configuration during replacement. Maps belong to the non-nil LPXGD, +// not the PCS, and therefore survive PCS garbage collection. +func (r *graphReconciler) deleteUnusedConfigMaps(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, resources []client.Object) error { + // Keep old runtime configuration until the replacement workload is Ready. + if !meta.IsStatusConditionTrue(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) { + return nil + } + // Retain every ConfigMap rendered for the current workload. desiredNames := make(map[string]struct{}, len(resources)) for _, resource := range resources { @@ -570,34 +472,21 @@ func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployme } // Discover obsolete ConfigMaps rooted in this exact LPX child. - stale := make([]client.Object, 0) configMaps := &corev1.ConfigMapList{} - if err := r.apiReader.List(ctx, configMaps, + if err := r.List(ctx, configMaps, client.InNamespace(deployment.Namespace), - client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)}, + client.MatchingLabels{deploymentUIDLabel: string(deployment.UID)}, ); err != nil { return err } + + // Preconditions prevent stale observations from deleting replacements. for index := range configMaps.Items { configMap := &configMaps.Items[index] - if metav1.IsControlledBy(configMap, deployment) { - if _, desired := desiredNames[configMap.GetName()]; !desired { - stale = append(stale, configMap) - } + if !metav1.IsControlledBy(configMap, deployment) || !configMap.DeletionTimestamp.IsZero() { + continue } - } - if len(stale) == 0 { - return nil - } - - // Revalidate publication authority before deleting the discovered objects. - if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { - return err - } - - // Preconditions prevent stale observations from deleting replacements. - for _, configMap := range stale { - if !configMap.GetDeletionTimestamp().IsZero() { + if _, desired := desiredNames[configMap.Name]; desired { continue } uid, resourceVersion := configMap.GetUID(), configMap.GetResourceVersion() @@ -610,38 +499,14 @@ func (r *graphReconciler) deleteStaleLPXConfigMaps(ctx context.Context, deployme return nil } -func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { - // One serving endpoint belongs to the engine, independently of mutable component names. - pcsName := dynamo.PCSNameForLPX(deployment) - serviceName := pcsName + "-serve" - - if !commoncontroller.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, source.Annotations) { - service := &corev1.Service{} - key := client.ObjectKey{Namespace: deployment.Namespace, Name: serviceName} - if err := r.Get(ctx, key, service); err != nil { - return client.IgnoreNotFound(err) - } - if !metav1.IsControlledBy(service, deployment) { - return fmt.Errorf("refusing to delete a foreign LPX endpoint %q", service.Name) - } - return client.IgnoreNotFound(r.Delete(ctx, service, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &service.UID, ResourceVersion: &service.ResourceVersion}})) +// integrationDisabledMessage returns an empty string when integration is enabled. +// runtimeConfig is non-nil; child status remains available when disabled. +func integrationDisabledMessage(runtimeConfig *commoncontroller.RuntimeConfig) string { + if !runtimeConfig.Gate.Enabled(features.Grove) { + return "Grove is disabled" } - - // Source component metadata is needed only when publishing the serving endpoint. - component := lpx.ServingComponent(source) - service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ - ServiceName: serviceName, Namespace: deployment.Namespace, - ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, - DynamoNamespace: source.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, - Labels: dynamo.GetDGDComponentResourceLabels(source, component.ComponentName, component), - Annotations: dynamo.GetDGDComponentResourceAnnotations(source, component.ComponentName, component), - }) - if err != nil { - return err + if !runtimeConfig.Gate.Enabled(features.LPX) { + return "LPX integration is disabled" } - // Keep cleanup and pod selection scoped to this materialization. - service.Labels[lpxOwnerUIDLabel] = string(deployment.UID) - service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue - service.Spec.Selector[grovecommon.LabelPartOfKey] = pcsName - return r.syncLPXResource(ctx, deployment, service) + return "" } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index ff4d5c8ebedb..c0d9f9af4f89 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -6,7 +6,11 @@ package lpx import ( "context" "errors" + "fmt" + "net/url" + "os" "slices" + "strconv" "strings" "testing" "time" @@ -18,590 +22,423 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/yaml" ) -func TestLPXReplicaUpdatesUseScaleSubresource(t *testing.T) { - conflict := apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), "group", errors.New("stale version")) +func TestLPXWorkloadErrorDoesNotAcknowledgeGeneration(t *testing.T) { for _, tc := range []struct { - name string - replicas *int32 - seed int32 - owner string - err error - wantUpdates int + name string + deadline bool + failStatus bool + workloadSucceeds bool }{ - {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, - {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, - {name: "foreign scale in", replicas: ptr.To(int32(2)), owner: "foreign"}, - {name: "ownerless scale in", replicas: ptr.To(int32(2)), owner: "absent"}, - {name: "wrong kind scale in", replicas: ptr.To(int32(2)), owner: "wrong kind"}, - {name: "unchanged", replicas: ptr.To(int32(9))}, - {name: "omitted scale out", seed: 3}, - {name: "omitted scale in", seed: 12}, - {name: "conflict", replicas: ptr.To(int32(12)), err: conflict, wantUpdates: 1}, + {name: "ordinary error"}, + {name: "bounded deadline retry", deadline: true}, + {name: "status failure keeps both errors", deadline: true, failStatus: true}, + {name: "status failure after a pending pass", deadline: true, failStatus: true, workloadSucceeds: true}, } { t.Run(tc.name, func(t *testing.T) { - t.Log("Observe a Grove group with nine replicas") - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = tc.replicas - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - pcs := findLPXTestPodCliqueSet(t, objects) - switch tc.owner { - case "foreign": - pcs.OwnerReferences[0].UID = "foreign-owner" - case "absent": - pcs.OwnerReferences = nil - case "wrong kind": - pcs.OwnerReferences[0].Kind = consts.ResourceTypeDynamoGraphDeployment - } - if tc.replicas == nil { - pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas = ptr.To(tc.seed) + t.Log("Retain a pending request while processing a new child generation") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + child.Generation, child.Status.ObservedGeneration = 2, 1 + if tc.deadline { + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](60)} } - hash, err := commoncontroller.GetSpecHash(pcs, commoncontroller.WithPreservedListOrder()) - require.NoError(t, err) - metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) - metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, commoncontroller.NvidiaAnnotationGenerationKey, "1") - group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) - group.Spec.Replicas = 9 - createLPXTestObjects(t, t.Context(), r.Client, objects...) - pcs = observedLPXTestPodCliqueSet(t, t.Context(), r, child, selected) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) - beforePCS := pcs.DeepCopy() - - t.Log("Rebuild scheduler intent from explicit replicas or the current native scale") - selected, classification, err := r.prepareLPXMaterializing(t.Context(), child, source, pcs) - require.NoError(t, err) - require.Nil(t, classification) - require.Equal(t, ptr.Deref(tc.replicas, int32(9)), selected.plan.Replicas) - require.Len(t, selected.requests, int(selected.plan.Replicas)) - updates := 0 + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now(), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, pending)) + + t.Log("Fail runtime synchronization after the scheduling deadline is derived") + readErr, statusErr := errors.New("ConfigMap read unavailable"), errors.New("status write unavailable") + failReads, failStatus := !tc.workloadSucceeds, tc.failStatus + reads := 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*corev1.ConfigMap); ok && failReads { + reads++ + return readErr + } + return delegated.Get(ctx, key, object, opts...) + }, SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - require.Equal(t, "scale", subresource) - require.Equal(t, client.ObjectKeyFromObject(group), client.ObjectKeyFromObject(object)) - options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) - scale := options.SubResourceBody.(*autoscalingv1.Scale) - require.Equal(t, group.ResourceVersion, scale.ResourceVersion) - require.Equal(t, *tc.replicas, scale.Spec.Replicas) - updates++ - if tc.err != nil { - return tc.err + if subresource == "status" && failStatus { + return statusErr } return delegated.SubResource(subresource).Update(ctx, object, opts...) }, }) - - t.Log("Only explicit changes write scale, preserving the observed resource-version precondition") - _, _, err = r.reconcileWorkload(t.Context(), child, source, selected, pcs) - if tc.owner != "" { - var ownershipConflict *commoncontroller.OwnershipConflictError - require.ErrorAs(t, err, &ownershipConflict) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, key) + if !tc.workloadSucceeds { + require.Positive(t, reads) + } + if tc.deadline && !tc.failStatus { + require.NoError(t, err) + require.Equal(t, pipelineRequestDeadlineRetryInterval, result.RequeueAfter) } else { - require.ErrorIs(t, err, tc.err) + if !tc.workloadSucceeds { + require.ErrorIs(t, err, readErr) + } + require.Zero(t, result) + if tc.failStatus { + require.ErrorIs(t, err, statusErr) + } } - require.Equal(t, tc.wantUpdates, updates) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - want := int32(9) - if tc.replicas != nil && err == nil { - want = *tc.replicas + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.EqualValues(t, 1, child.Status.ObservedGeneration) + if !tc.failStatus { + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, ready) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, ready.Reason) + require.EqualValues(t, 2, ready.ObservedGeneration) + require.Contains(t, ready.Message, readErr.Error()) } - require.Equal(t, want, group.Spec.Replicas) - t.Log("Native scale never rewrites the parent PCS seed or its bookkeeping") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) - require.Equal(t, beforePCS, pcs) + t.Log("A successful pending pass acknowledges the generation and preserves its deadline wake") + failReads, failStatus = false, false + result, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.EqualValues(t, 2, child.Status.ObservedGeneration) + if tc.deadline { + require.Positive(t, result.RequeueAfter) + require.LessOrEqual(t, result.RequeueAfter, time.Minute) + } }) } } -func TestLPXTerminatingGroveResourcesRemainRetiring(t *testing.T) { - t.Log("Publish a workload whose PCS and scaling group are both terminating") +func TestPipelineRequestDeadlineContinuesDuringRequestDeletion(t *testing.T) { + t.Log("Publish two pending replicas and expire the tail first") ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) publishSelectedLPXForTest(t, ctx, r, child, selected) - pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - for _, object := range []client.Object{pcs, group} { - object.SetFinalizers([]string{"example.com/cleanup"}) - require.NoError(t, r.Update(ctx, object)) - require.NoError(t, r.Delete(ctx, object)) + pcs := findLPXTestPodCliqueSet(t, objects) + prefix := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + prefix.Status = newTestPipelineRequest(child, pcs, prefix.Name, time.Now(), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, prefix)) + tail := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].Name) + tail.Status = newTestPipelineRequest(child, pcs, tail.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + tail.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, tail)) + + t.Log("Retire the expired tail while its scheduler finalizer holds deletion") + reconcileRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 2 { + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - require.False(t, pcs.DeletionTimestamp.IsZero()) + tail = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, tail.Name) + require.False(t, tail.DeletionTimestamp.IsZero()) - t.Log("Let the publication fence report retirement before attempting scale") - state, result, err := r.reconcileWorkload(ctx, child, source, selected, pcs) + t.Log("Continue enforcing the surviving request's independent deadline") + prefix = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, prefix.Name) + prefix.Status = newTestPipelineRequest(child, pcs, prefix.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, prefix)) + _, err := r.Reconcile(ctx, reconcileRequest) require.NoError(t, err) - require.Equal(t, lpxRetiringReason, state.Reason) - require.Equal(t, lpxRetirementRequeueAfter, result.RequeueAfter) + requirePipelineRequestNotFound(t, ctx, r.Client, child.Namespace, prefix.Name) } -func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { - t.Log("Observe an externally inflated group through a cache that stays stale after scale") +func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { + t.Log("Publish an expired request without a persisted deadline failure") ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) - group.Spec.Replicas = 9 + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) - pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - stale := group.DeepCopy() - scaled, updates := false, 0 + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, pending)) + + t.Log("Reject the status write and forbid cleanup until the failure becomes durable") + statusErr := errors.New("status write unavailable") + failStatus := true + groupMissing := true + statusWrites, scaleWrites, deletes := 0, 0, 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if scaled && key == client.ObjectKeyFromObject(stale) { - if observed, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { - stale.DeepCopyInto(observed) - return nil - } + if _, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && groupMissing { + return apierrors.NewNotFound(consts.PodCliqueScalingGroupGVR.GroupResource(), key.Name) } return delegated.Get(ctx, key, object, opts...) }, SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - updates++ - if err := delegated.SubResource(subresource).Update(ctx, object, opts...); err != nil { - return err + if subresource == "status" { + statusWrites++ + if failStatus { + return statusErr + } + } else if subresource == "scale" { + scaleWrites++ } - scaled = true - return nil + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + deletes++ + return delegated.Delete(ctx, object, opts...) }, }) - - t.Log("Continue reconciliation without issuing a second stale-resource-version write") - _, _, err := r.reconcileWorkload(ctx, child, source, selected, pcs) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, request) + require.ErrorIs(t, err, statusErr) + require.Zero(t, result, "status errors use controller-runtime backoff, not the expiry requeue") + require.Equal(t, 1, statusWrites) + require.Zero(t, scaleWrites) + require.Zero(t, deletes) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + + t.Log("Record failure even without a PCSG observation, but leave the LPR intact") + failStatus = false + result, err = r.Reconcile(ctx, request) require.NoError(t, err) - require.Equal(t, 1, updates) + require.Equal(t, time.Nanosecond, result.RequeueAfter, "status-only updates need an explicit follow-up") + require.Equal(t, 2, statusWrites) + require.Zero(t, scaleWrites) + require.Zero(t, deletes) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + failed := meta.FindStatusCondition(child.Status.Conditions, "Ready").DeepCopy() + + t.Log("Persisted failure cannot authorize cleanup until the live group is observed") + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result, "the PCSG watch resumes cleanup") + require.Zero(t, scaleWrites) + require.Zero(t, deletes) + require.True(t, apiequality.Semantic.DeepEqual(pending, getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name))) + + t.Log("Observing both the persisted failure and the group allows expired-replica cleanup") + groupMissing = false + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result, "request and group events drive cleanup") + require.Equal(t, 1, scaleWrites) + require.Equal(t, 1, deletes) + requirePipelineRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.Equal(t, failed, meta.FindStatusCondition(child.Status.Conditions, "Ready")) + + t.Log("A sticky failure neither polls nor restores the expired capacity") + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result) + require.Equal(t, 1, scaleWrites) + require.Equal(t, 1, deletes) + requirePipelineRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) } -func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { +func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { + t.Log("Expire an interior engine while a higher ordinal is still Bound") ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - - stale := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) - stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} - require.NoError(t, r.Update(ctx, stale)) - - liveSource := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), liveSource)) - lpx.ServingComponent(liveSource).Replicas = ptr.To(int32(1)) - liveSource.Generation++ - require.NoError(t, r.Update(ctx, liveSource)) - liveChild := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) - liveChild.Generation++ - var err error - liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveSource, "") - require.NoError(t, err) - require.NoError(t, r.Update(ctx, liveChild)) - - t.Log("Persist the lower Grove scale before request retirement can block reconciliation") - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - _, err = r.Reconcile(ctx, request) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + requests, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Equal(t, int32(1), group.Spec.Replicas) - t.Log("Let request finalization proceed asynchronously after the lower scale is persisted") - stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) - require.False(t, stale.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), &grovev1alpha1.PodCliqueSet{})) + for index := range requests { + request := &requests[index] + phase := lpxv1alpha1.RequestPhasePending + if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 1 { + phase = lpxv1alpha1.RequestPhaseBound + } + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now().Add(-time.Minute), phase).Status + require.NoError(t, r.Update(ctx, request)) + } - t.Log("Reverse the scale-down while the old request still protects its pods") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), liveSource)) - lpx.ServingComponent(liveSource).Replicas = ptr.To(int32(2)) - liveSource.Generation++ - require.NoError(t, r.Update(ctx, liveSource)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) - liveChild.Generation++ - liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveSource, "") + t.Log("Persist the failure once, then leave the hole unchanged without polling") + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, key) require.NoError(t, err) - require.NoError(t, r.Update(ctx, liveChild)) + require.Equal(t, time.Nanosecond, result.RequeueAfter) for range 2 { - _, err = r.Reconcile(ctx, request) + result, err = r.Reconcile(ctx, key) require.NoError(t, err) + require.Zero(t, result) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas, "Grove must not recreate pods protected by the retiring request") - } - - t.Log("Allow scale-up and a fresh request once the scheduler finishes cleanup") - stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) - retiredUID := stale.UID - stale.Finalizers = nil - require.NoError(t, r.Update(ctx, stale)) - for range 4 { - _, err = r.Reconcile(ctx, request) + require.Equal(t, int32(2), group.Spec.Replicas) + current, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) + require.True(t, apiequality.Semantic.DeepEqual(requests, current), "interior failures must not mutate requests") } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - replacement := getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) - require.NotEqual(t, retiredUID, replacement.UID) - require.True(t, replacement.DeletionTimestamp.IsZero()) -} - -func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { - t.Log("Publish one engine whose capacity is owned by the native Grove scaling group") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = nil - r, initial := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, initial) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, initial) - pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, initial) - pcsUID := pcs.UID - templateReplicas := ptr.Deref(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0) - require.Positive(t, templateReplicas) - request := getLPXRequest(t, ctx, r.Client, child.Namespace, initial.requests[0].requestName) - requestUID := request.UID - - t.Log("Scale the native Grove group to zero and derive empty scheduler intent") - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: initial.plan.LPXScalingGroup}, group)) - group.Spec.Replicas = 0 - require.NoError(t, r.Update(ctx, group)) - zero, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) - require.NoError(t, err) - require.Nil(t, classification) - require.Zero(t, zero.plan.Replicas) - require.Empty(t, zero.requests) - t.Log("Delete the stale LPR before reconciling the stable PCS template") - _, _, err = r.reconcileWorkload(ctx, child, source, zero, pcs) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) - _, _, err = r.reconcileWorkload(ctx, child, source, zero, pcs) + t.Log("A scheduler update that expires the higher ordinal unblocks complete-suffix cleanup") + higher := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].Name) + higher.Status.Phase = lpxv1alpha1.RequestPhasePending + require.NoError(t, r.Update(ctx, higher)) + result, err = r.Reconcile(ctx, key) require.NoError(t, err) + require.Zero(t, result) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Zero(t, group.Spec.Replicas) - storedPCS := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) - require.Equal(t, pcsUID, storedPCS.UID) - require.Equal(t, templateReplicas, ptr.Deref(storedPCS.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0)) - - t.Log("Scale the native group back up and publish a fresh request through the same PCS") - group.Spec.Replicas = 1 - require.NoError(t, r.Update(ctx, group)) - scaledUp, classification, err := r.prepareLPXMaterializing(ctx, child, source, storedPCS) - require.NoError(t, err) - require.Nil(t, classification) - require.Equal(t, int32(1), scaledUp.plan.Replicas) - require.Len(t, scaledUp.requests, 1) - _, _, err = r.reconcileWorkload(ctx, child, source, scaledUp, storedPCS) + current, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) - replacement := getLPXRequest(t, ctx, r.Client, child.Namespace, scaledUp.requests[0].requestName) - require.NotEqual(t, requestUID, replacement.UID) - require.Equal(t, pcsUID, metav1.GetControllerOf(replacement).UID) + require.Empty(t, current) } -func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { - ctx := t.Context() - child, source, registry := newLPXSpecDecodeTestDGD(t) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) +func TestLPXDeletesOnlyStaleOwnedRuntimeConfigMaps(t *testing.T) { + t.Log("Create current, stale, and foreign runtime ConfigMaps") + dgd := loadTestDGD(t, lpx.PipelineSingle, "test-build") + child := newLPXTestDeployment(t, dgd) + owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} + current := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-current", Namespace: child.Namespace, UID: "current", + Labels: map[string]string{deploymentUIDLabel: string(child.UID)}, OwnerReferences: owner, + }} + stale := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-stale", Namespace: child.Namespace, UID: "stale", + Labels: map[string]string{deploymentUIDLabel: string(child.UID)}, OwnerReferences: owner, + }} + foreign := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "runtime-foreign", Namespace: child.Namespace, UID: "foreign", + Labels: map[string]string{deploymentUIDLabel: string(child.UID)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, v1beta1.DynamoGraphDeploymentGVK)}, + }} + r := newLPXTestReconciler(t, nil, child, dgd, current, stale, foreign) - scaledSource := source.DeepCopy() - require.Len(t, scaledSource.Spec.Components, 2) - scaledSource.Spec.Components[1].Replicas = ptr.To(int32(1)) - scaled, rejected := requirePreparedLPX(t, r, ctx, child, scaledSource) - require.Nil(t, rejected) - require.Len(t, scaled.requests, 2) + t.Log("Keep the previous runtime configuration while the replacement is not Ready") + require.NoError(t, r.deleteUnusedConfigMaps(t.Context(), child, []client.Object{current})) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(stale), &corev1.ConfigMap{})) - desiredNames := make(map[string]struct{}, len(scaled.requests)) - for _, request := range scaled.requests { - desiredNames[request.requestName] = struct{}{} - } - var stale *lpxv1alpha1.LPUPipelineRequest - for _, request := range selected.requests { - if _, retained := desiredNames[request.requestName]; retained { - continue - } - stale = getLPXRequest(t, ctx, r.Client, child.Namespace, request.requestName) - break - } - require.NotNil(t, stale) - stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} - require.NoError(t, r.Update(ctx, stale)) + t.Log("Delete only the stale ConfigMap owned by the current LPX child") + setReadyCondition(child, v1beta1.DGDStateSuccessful, "Replacement is Ready") + require.NoError(t, r.deleteUnusedConfigMaps(t.Context(), child, []client.Object{current})) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), pcs)) - pcs.Finalizers = []string{"test.example/observe-retirement"} - require.NoError(t, r.Update(ctx, pcs)) - retiredTemplate := selected.plan.Agents[1].TemplateName - originalCliqueCount := len(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + t.Log("Preserve the current and foreign ConfigMaps") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(current), &corev1.ConfigMap{})) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), client.ObjectKeyFromObject(stale), &corev1.ConfigMap{}))) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(foreign), &corev1.ConfigMap{})) +} - t.Log("Retire the immutable PCS without mutating Grove membership or directly deleting its LPRs") - pcsDeleteObserved := false - requestDeleteObserved := false - wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - switch object.(type) { - case *grovev1alpha1.PodCliqueSet: - pcsDeleteObserved = true - case *lpxv1alpha1.LPUPipelineRequest: - requestDeleteObserved = true +func TestLPXDownloadsBeforePublication(t *testing.T) { + for _, scenario := range []struct { + name, componentName string + sharedDraft bool + messages []string + }{ + {name: "disabled Grove", messages: []string{"Grove is disabled"}}, + {name: "long serving name", componentName: strings.Repeat("serving-", 7) + "engine"}, + {name: "reordered shared draft with long serving name", componentName: strings.Repeat("serving-", 7) + "engine", sharedDraft: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Reconcile authored LPX intent against cold remote builds") + dgd := loadTestDGD(t, lpx.PipelineSingle, modelDownloadTestBuildID) + if scenario.sharedDraft { + dgd = loadTestSpecDecodeDGD(t) + dgd.Spec.Components[0].LPX.BuildID = modelDownloadTestBuildID + dgd.Spec.Components[1].LPX.BuildID = modelDownloadTestSecondBuildID + dgd.Spec.Components[1].ComponentName = "shared-draft-name-is-not-materialized" + dgd.Spec.Components = []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend}, + dgd.Spec.Components[1], dgd.Spec.Components[0], + } } - return delegated.Delete(ctx, object, opts...) - }, - }) - r.Client, r.apiReader = wrapped, wrapped - _, classification, err := r.reconcileLPXPublicationFence(ctx, child, scaled, pcs) - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - require.True(t, pcsDeleteObserved) - require.False(t, requestDeleteObserved) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) - require.False(t, pcs.DeletionTimestamp.IsZero()) - require.Contains(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames, retiredTemplate) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Contains(t, group.Spec.CliqueNames, retiredTemplate) - stale = getLPXRequest(t, ctx, r.Client, stale.Namespace, stale.Name) - require.True(t, stale.DeletionTimestamp.IsZero()) - - t.Log("Fence replacement until owner garbage collection removes every old LPR") - pcs.Finalizers = nil - require.NoError(t, r.Update(ctx, pcs)) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{}))) - _, classification, err = r.reconcileLPXPublicationFence(ctx, child, scaled, nil) - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - - requests, err := r.listLPXRequestCandidates(ctx, child) - require.NoError(t, err) - for index := range requests.Items { - request := &requests.Items[index] - request.Finalizers = nil - require.NoError(t, r.Update(ctx, request)) - require.NoError(t, r.Delete(ctx, request)) - } - _, classification, err = r.reconcileLPXPublicationFence(ctx, child, scaled, nil) - require.NoError(t, err) - require.Nil(t, classification) - - t.Log("Create the replacement PCS with the scaled-in immutable shape") - replacement := renderLPXTestPodCliqueSet(t, ctx, r, child, scaledSource, scaled) - changed, synced, err := commoncontroller.SyncObservedResource( - ctx, r, child, nil, replacement, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, changed) - require.Equal(t, replacement.Spec, synced.Spec) - require.Equal(t, scaled.workloadDigest, synced.Annotations[lpx.WorkloadDigestAnnotation]) - require.Less(t, len(synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames), originalCliqueCount) -} - -func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { - for _, phase := range []lpxv1alpha1.RequestPhase{lpxv1alpha1.RequestPhasePending, lpxv1alpha1.RequestPhaseBound} { - t.Logf("Scheduler phase after recorded expiry: %s", phase) - t.Log("Observe a completed engine and a newer pending request in their shared PCS") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - pcs := findLPXTestPodCliqueSet(t, objects) - serving := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound).Status - require.NoError(t, r.Update(ctx, serving)) - pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) - pending.Status = deadlineTestRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status - pending.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} - require.NoError(t, r.Update(ctx, pending)) - - t.Log("Persist failure before changing Grove or deleting scheduler intent") - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - group := findLPXTestScalingGroup(t, objects, selected.plan.LPXScalingGroup) - persistErr := errors.New("status unavailable") - original := r.Client - r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ - SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - if subresource == "status" { - return persistErr - } - return delegated.SubResource(subresource).Update(ctx, object, opts...) - }, - }) - _, err := r.Reconcile(ctx, request) - require.ErrorIs(t, err, persistErr) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - require.Empty(t, child.Status.ExpiredRequestUIDs) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - require.True(t, getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name).DeletionTimestamp.IsZero()) - r.Client = original - _, err = r.Reconcile(ctx, request) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) - require.True(t, pending.DeletionTimestamp.IsZero()) - if phase == lpxv1alpha1.RequestPhaseBound { - deleteErr := apierrors.NewConflict(lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource(), pending.Name, errors.New("scheduler updated status")) - r.Client = interceptor.NewClient(original.(client.WithWatch), interceptor.Funcs{ - Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { return deleteErr }, - }) - _, err = r.Reconcile(ctx, request) - require.ErrorIs(t, err, deleteErr) - r.Client = original - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - require.Len(t, child.Status.ExpiredRequestUIDs, 1) - require.Equal(t, pending.UID, child.Status.ExpiredRequestUIDs[0]) - } - pending.Status.Phase = phase - require.NoError(t, r.Update(ctx, pending)) - conductorReplicas := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas - if phase == lpxv1alpha1.RequestPhaseBound { - t.Log("An edit after recorded expiry cannot rescue the old request") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) - source.Spec.Scheduling = nil - lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - require.NoError(t, r.Update(ctx, source)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - child.Generation++ - require.NoError(t, r.Update(ctx, child)) - } - - t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") - for range 3 { - _, err = r.Reconcile(ctx, request) + component := lpx.ServingComponent(dgd) + if scenario.componentName != "" { + component.ComponentName = scenario.componentName + } + registry, err := lpx.NewModelRegistry("", nil) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) - require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - } - t.Log("A later edit still waits for the failed request's scheduler finalizer") - pending = getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) - require.False(t, pending.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - if phase == lpxv1alpha1.RequestPhasePending { - child.Generation++ - require.NoError(t, r.Update(ctx, child)) - } - _, err = r.Reconcile(ctx, request) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) + observedRegistry := &fakeModelDownloadRegistry{ModelRegistry: registry} + child := newLPXTestDeployment(t, dgd) + r := newLPXTestReconciler(t, observedRegistry, child, dgd) + if scenario.name == "disabled Grove" { + r.runtimeConfig.Gate.Grove = false + r.enabled = false + } - t.Log("After cleanup the later edit can retry without replacing the serving engine") - if phase == lpxv1alpha1.RequestPhaseBound { - lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = conductorReplicas - require.NoError(t, r.Update(ctx, source)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - child.Generation++ - require.NoError(t, r.Update(ctx, child)) - } - retiredUID := pending.UID - pending.Finalizers = nil - require.NoError(t, r.Update(ctx, pending)) - for range 5 { - _, err = r.Reconcile(ctx, request) + t.Log("Block downloads when Grove is disabled and wait for cold builds otherwise") + before := dgd.DeepCopy() + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) - } - replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) - require.NotEqual(t, retiredUID, replacement.UID) - require.True(t, replacement.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") - } -} - -func TestLPXDeadlineContinuesDuringRequestRetirement(t *testing.T) { - t.Log("Publish two pending replicas and expire the tail first") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - pcs := findLPXTestPodCliqueSet(t, objects) - prefix := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - prefix.Status = deadlineTestRequest(child, pcs, prefix.Name, time.Now(), lpxv1alpha1.RequestPhasePending).Status - require.NoError(t, r.Update(ctx, prefix)) - tail := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) - tail.Status = deadlineTestRequest(child, pcs, tail.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status - tail.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} - require.NoError(t, r.Update(ctx, tail)) - - t.Log("Retire the expired tail while its scheduler finalizer holds deletion") - reconcileRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - for range 2 { - _, err := r.Reconcile(ctx, reconcileRequest) - require.NoError(t, err) - } - tail = getLPXRequest(t, ctx, r.Client, child.Namespace, tail.Name) - require.False(t, tail.DeletionTimestamp.IsZero()) - - t.Log("Continue enforcing the surviving request's independent deadline") - prefix = getLPXRequest(t, ctx, r.Client, child.Namespace, prefix.Name) - prefix.Status = deadlineTestRequest(child, pcs, prefix.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status - require.NoError(t, r.Update(ctx, prefix)) - for range 2 { - _, err := r.Reconcile(ctx, reconcileRequest) - require.NoError(t, err) + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + failed := meta.FindStatusCondition(stored.Status.Conditions, "Ready") + require.NotNil(t, failed) + if len(scenario.messages) == 0 { + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, v1alpha1.LPXReadyReasonPending, failed.Reason) + wantCalls := []string{modelDownloadTestBuildID} + if scenario.sharedDraft { + wantCalls = append(wantCalls, modelDownloadTestSecondBuildID) + } + require.Equal(t, wantCalls, observedRegistry.calls) + } else { + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, failed.Reason) + for _, message := range scenario.messages { + require.Contains(t, failed.Message, message) + } + require.Empty(t, observedRegistry.calls) + } + require.Zero(t, observedRegistry.acquireBuildSnapshotCalls) + pcs := &grovev1alpha1.PodCliqueSetList{} + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), pcs)) + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, pcs.Items) + require.Empty(t, requests.Items) + require.Equal(t, before, dgd) + }) } - requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, prefix.Name) } func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { t.Log("Publish a request, then advance the deployment before that request expires") ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) publishSelectedLPXForTest(t, ctx, r, child, selected) - pending := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) + pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) pcs := findLPXTestPodCliqueSet(t, objects) - pending.Status = deadlineTestRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status require.NoError(t, r.Update(ctx, pending)) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) child.Generation++ @@ -617,14 +454,15 @@ func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { t.Log("After the failure fence is durable, scale down and remove the expired request") _, err = r.Reconcile(ctx, reconcileRequest) require.NoError(t, err) - requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + requirePipelineRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") + failed := meta.FindStatusCondition(child.Status.Conditions, "Ready") require.NotNil(t, failed) - require.Equal(t, metav1.ConditionTrue, failed.Status) - require.Equal(t, lpxSchedulingDeadlineExceededReason, failed.Reason) + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, failed.Reason) require.Equal(t, failureGeneration, failed.ObservedGeneration) - schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, lpxSchedulingFailedCondition) + require.Len(t, child.Status.Conditions, 2) + schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition) require.NotNil(t, schedulingFailed) require.Equal(t, metav1.ConditionTrue, schedulingFailed.Status) require.Equal(t, failureGeneration, schedulingFailed.ObservedGeneration) @@ -633,7 +471,7 @@ func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { for range 3 { _, err = r.Reconcile(ctx, reconcileRequest) require.NoError(t, err) - requireLPXRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) + requirePipelineRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) } t.Log("An edit after the failure permits a fresh request") @@ -644,126 +482,466 @@ func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { _, err = r.Reconcile(ctx, reconcileRequest) require.NoError(t, err) } - replacement := getLPXRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + replacement := getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) require.NotEqual(t, pending.UID, replacement.UID) } -func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { - t.Log("Publish two requests and remove one without a terminal deadline decision") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) - publishSelectedLPXForTest(t, ctx, r, child, selected) +func TestLPXEndpointLifecycle(t *testing.T) { + const uppercaseComponentName = "LPX" + t.Log("Configure an uppercase LPX component and independently named materialization using Kubernetes discovery") + dgd := loadTestDGD(t, lpx.PipelineSingle, "test-build") + dgd.Spec.Components[0].ComponentName = uppercaseComponentName + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + dgd.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} + dgd.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} + dgd.Spec.Labels = map[string]string{"example.com/policy": "enabled"} + child := newLPXTestDeployment(t, dgd) + child.Name = "independent-materialization" + r := newLPXTestReconciler(t, nil, child, dgd) - t.Log("Advance the deployment and remove one request before it fails") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Generation++ - require.NoError(t, r.Update(ctx, child)) + t.Log("Keep the ordinary model Service under the source DGD's ownership") + modelService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.GenerateServiceName("test/model"), Namespace: dgd.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, v1beta1.DynamoGraphDeploymentGVK)}, + Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: dgd.Name}, + }, + Spec: corev1.ServiceSpec{ClusterIP: corev1.ClusterIPNone, + Selector: map[string]string{consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("test/model")}}, + } + require.NoError(t, r.Create(t.Context(), modelService)) + modelKey := client.ObjectKeyFromObject(modelService) + require.NoError(t, r.Get(t.Context(), modelKey, modelService)) + beforeModelService := modelService.DeepCopy() - missing := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].requestName) - previousUID := missing.UID - require.NoError(t, r.Delete(ctx, missing)) + t.Log("Publish the LPX-owned endpoint with its serving-role selector") + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + service := &corev1.Service{} + key := client.ObjectKey{Namespace: dgd.Namespace, Name: dynamo.PCSNameForLPX(child) + "-serve"} + require.NoError(t, r.Get(t.Context(), key, service)) + require.True(t, metav1.IsControlledBy(service, child)) + require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) + require.Equal(t, dynamo.PCSNameForLPX(child), service.Spec.Selector[grovecommon.LabelPartOfKey]) - t.Log("Reconciliation recreates deterministic scheduler intent without inheriting a scheduling clock") - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - replacement := getLPXRequest(t, ctx, r.Client, missing.Namespace, missing.Name) - require.NotEqual(t, previousUID, replacement.UID) - require.Nil(t, replacement.Status) + t.Log("Converge propagated endpoint metadata, including removed keys") + for _, value := range []string{"changed", ""} { + if value == "" { + dgd.Spec.Labels, dgd.Spec.Annotations = nil, nil + } else { + dgd.Spec.Labels["example.com/policy"] = value + dgd.Spec.Annotations["example.com/model-discovery"] = value + } + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + updated := &corev1.Service{} + require.NoError(t, r.Get(t.Context(), key, updated)) + require.Equal(t, value, updated.Labels["example.com/policy"]) + require.Equal(t, value, updated.Annotations["example.com/model-discovery"]) + if value == "" { + require.NotContains(t, updated.Labels, "example.com/policy") + require.NotContains(t, updated.Annotations, "example.com/model-discovery") + } + require.Equal(t, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled], updated.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) + require.Equal(t, service.Spec, updated.Spec) + require.Equal(t, service.OwnerReferences, updated.OwnerReferences) + for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { + require.NotEmpty(t, updated.Annotations[key]) + require.Equal(t, service.Annotations[key], updated.Annotations[key]) + } + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + unchanged := &corev1.Service{} + require.NoError(t, r.Get(t.Context(), key, unchanged)) + require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) + } + + t.Log("Preserve API-allocated Service fields when the endpoint selector changes") + require.NoError(t, r.Get(t.Context(), key, service)) + service.Spec.ClusterIP = "10.96.0.10" + service.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} + service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + service.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyPreferDualStack) + require.NoError(t, r.Update(t.Context(), service)) + allocated := service.DeepCopy() + for _, global := range []bool{true, false} { + dgd.Spec.Components[0].GlobalDynamoNamespace = global + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + require.NoError(t, r.Get(t.Context(), key, service)) + allocated.Spec.Selector[consts.KubeLabelDynamoNamespace] = dgd.GetDynamoNamespaceForComponent(&dgd.Spec.Components[0]) + require.Equal(t, allocated.Spec, service.Spec) + require.Equal(t, allocated.OwnerReferences, service.OwnerReferences) + + t.Log("Reconcile the unchanged endpoint without another write") + version := service.ResourceVersion + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + require.NoError(t, r.Get(t.Context(), key, service)) + require.Equal(t, version, service.ResourceVersion) + } + + t.Log("Remove the LPX-owned discovery endpoint when switching back to non-Kubernetes discovery") + delete(dgd.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) + require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) + + t.Log("Discovery cleanup leaves the ordinary DGD model service unchanged") + require.NoError(t, r.Get(t.Context(), modelKey, modelService)) + require.Equal(t, beforeModelService, modelService) } func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { t.Log("Publish a speculative engine using its frozen child identity") - child, source, registry := newLPXSpecDecodeTestDGD(t) - source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + child, dgd, registry := newLPXSpecDecodeTestDGD(t) + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:old"}}}}, }, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "prefill:old"}}}}, }) - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) publishSelectedLPXForTest(t, t.Context(), r, child, selected) beforeRequests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(t.Context(), beforeRequests)) require.NotEmpty(t, beforeRequests.Items) - beforePCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, selected) + beforePCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, dgd, selected) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) beforeChild := child.DeepCopy() t.Log("Reorder engines, update the frontend and enable an ordinary checkpoint without changing LPX") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - components := source.Spec.Components + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + components := dgd.Spec.Components components[0], components[1] = components[1], components[0] - source.GetComponentByName("frontend").PodTemplate.Spec.Containers[0].Image = "frontend:next" - source.GetComponentByName("prefill").Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ + dgd.GetComponentByName("frontend").PodTemplate.Spec.Containers[0].Image = "frontend:next" + dgd.GetComponentByName("prefill").Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ Enabled: true, CheckpointRef: ptr.To("prefill-checkpoint"), }} - source.Generation++ - require.NoError(t, r.Update(t.Context(), source)) - beforeSource := source.DeepCopy() - require.Empty(t, lpx.ValidateSelectedIntent(source)) - require.NoError(t, dynamo.ValidateLPXSource(child, source)) - afterSelected, rejected := requirePreparedLPX(t, r, t.Context(), child, source) - require.Nil(t, rejected) - afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, source, afterSelected) + dgd.Generation++ + require.NoError(t, r.Update(t.Context(), dgd)) + beforeDGD := dgd.DeepCopy() + observedDGD, err := getDynamoGraphDeployment(t.Context(), r.Client, child) + require.NoError(t, err) + require.Equal(t, dgd, observedDGD) + afterSelected := resolveLPXTestWorkload(t, r.modelRegistry, t.Context(), child, dgd) + afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, dgd, afterSelected) require.Equal(t, beforePCS, afterPCS) require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) - _, err := reconcileSelectedLPXForTest(t, t.Context(), r, child, afterSelected) - require.NoError(t, err) + _, missing, changed := resolvePipelineRequests(child, beforeRequests.Items, afterSelected.workload, afterSelected.plan) + require.False(t, changed) + require.Empty(t, missing) afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(t.Context(), afterRequests)) require.Equal(t, beforeRequests, afterRequests) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) require.Equal(t, beforeChild, child) - require.Equal(t, beforeSource, source) + require.Equal(t, beforeDGD, dgd) } -func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { - t.Log("A request-list failure prevents deriving any active deadline") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - child := newLPXTestDeployment(t, source) - source.Spec.Scheduling = deadlineTestScheduling() - r := newLPXTestReconciler(t, nil, child, source) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - readError := errors.New("temporary request observation failure") - r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, requests := list.(*lpxv1alpha1.LPUPipelineRequestList); requests { - return readError - } - return delegated.List(ctx, list, opts...) - }, - }) - result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.ErrorIs(t, err, readError) - require.Zero(t, result) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - require.NotEqual(t, child.Generation, child.Status.ObservedGeneration) - require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Failed").Message, readError.Error()) -} +func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { + t.Run("deadline", func(t *testing.T) { + t.Log("Observe a completed engine and a newer pending request in their shared PCS") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + serving := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + serving.Status = newTestPipelineRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(ctx, serving)) + pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].Name) + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + pending.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, pending)) + + t.Log("Persist failure before changing Grove or deleting scheduler intent") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + _, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + pending = getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.True(t, pending.DeletionTimestamp.IsZero()) + + t.Log("Retire only the failed scale-out and keep Grove below the authored scale while cleanup is pending") + for range 3 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) + require.True(t, apiequality.Semantic.DeepEqual(serving, getTestPipelineRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + } + t.Log("A later edit still waits for the failed request's scheduler finalizer") + pending = getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.False(t, pending.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + + t.Log("After cleanup the later edit can retry without replacing the serving engine") + retiredUID := pending.UID + pending.Finalizers = nil + require.NoError(t, r.Update(ctx, pending)) + for range 5 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + } + replacement := getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) + require.NotEqual(t, retiredUID, replacement.UID) + require.True(t, replacement.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + require.True(t, apiequality.Semantic.DeepEqual(serving, getTestPipelineRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") + }) +} + +func TestLPXInvalidReplacementPreservesExistingWorkload(t *testing.T) { + t.Log("Publish a speculative engine with two draft models") + ctx := t.Context() + child, dgd, registry := newLPXSpecDecodeTestDGD(t) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) + beforeRequests, err := r.getPipelineRequests(ctx, pcs) + require.NoError(t, err) + + t.Log("Change immutable composition and remove required runtime storage in the same edit") + draft := dgd.GetComponentByName("draft") + draft.Replicas = ptr.To(int32(1)) + agent := draft.ComponentRole(v1beta1.ComponentRoleLPXAgent) + mounts := agent.PodTemplate.Spec.Containers[0].VolumeMounts + agent.PodTemplate.Spec.Containers[0].VolumeMounts = nil + replacement, err := lpx.ResolveSelectedWorkload(ctx, dgd, registry) + require.NoError(t, err) + require.NotEqual(t, selected.workload.Digest(), replacement.Digest()) + updateTestDGD(t, r, child, dgd) + + t.Log("Reject the replacement without deleting the serving PCS or its requests") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 2 { + _, err := r.Reconcile(ctx, request) + require.ErrorContains(t, err, "model storage volume mount") + observed, err := getPodCliqueSet(ctx, r.Client, child) + require.NoError(t, err) + require.Equal(t, pcs, observed) + requests, err := r.getPipelineRequests(ctx, pcs) + require.NoError(t, err) + require.Equal(t, beforeRequests, requests) + } + + t.Log("Authorize replacement once the same desired composition can render successfully") + agent.PodTemplate.Spec.Containers[0].VolumeMounts = mounts + updateTestDGD(t, r, child, dgd) + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{}))) +} + +func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { + t.Log("Publish four replicas with an expired interior request and a finalizer on the healthy tail") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(4)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + before := make([]*lpxv1alpha1.LPUPipelineRequest, len(selected.requests)) + for index, request := range selected.requests { + observed := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, request.Name) + phase := lpxv1alpha1.RequestPhaseBound + if index == 1 { + phase = lpxv1alpha1.RequestPhasePending + } + observed.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now().Add(-time.Hour), phase).Status + if index == 3 { + observed.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + } + require.NoError(t, r.Update(ctx, observed)) + before[index] = observed + } + + t.Log("Apply a four-to-three replica edit despite the retained interior expiry") + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(3)) + updateTestDGD(t, r, child, dgd) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 3 { + _, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.EqualValues(t, 3, group.Spec.Replicas) + tail := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, before[3].Name) + require.False(t, tail.DeletionTimestamp.IsZero()) + for _, original := range before[:3] { + observed := getTestPipelineRequest(t, ctx, r.Client, original.Namespace, original.Name) + require.True(t, apiequality.Semantic.DeepEqual(original, observed)) + } + } + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + + t.Log("Expire the remaining tail and clean its suffix while the removed request still awaits finalization") + remainingTail := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, before[2].Name) + remainingTail.Status = newTestPipelineRequest(child, pcs, remainingTail.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, remainingTail)) + _, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.EqualValues(t, 1, group.Spec.Replicas) + for _, original := range before[1:3] { + requirePipelineRequestNotFound(t, ctx, r.Client, original.Namespace, original.Name) + } + observed := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, before[0].Name) + require.True(t, apiequality.Semantic.DeepEqual(before[0], observed)) + tail := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, before[3].Name) + require.False(t, tail.DeletionTimestamp.IsZero()) + observedPCS, err := getPodCliqueSet(ctx, r.Client, child) + require.NoError(t, err) + require.Equal(t, pcs.UID, observedPCS.UID) +} + +func TestLPXExternalScaleRejectsInvalidReplicaCount(t *testing.T) { + t.Log("Observe externally managed capacity above the materialization limit") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(dgd).Replicas = nil + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + group.Spec.Replicas = 2497 + createLPXTestObjects(t, ctx, r.Client, objects...) + + t.Log("Reject the observed replica count before publishing requests and leave external capacity unchanged") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.ErrorContains(t, err, "between 0 and 2496") + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(ctx, requests)) + require.Empty(t, requests.Items) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.EqualValues(t, 2497, group.Spec.Replicas) +} + +func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { + for _, tc := range []struct { + name string + replicas int32 + expired int + retained []int + }{ + {name: "scale to zero", replicas: 0, expired: 0}, + {name: "remove failed suffix", replicas: 2, expired: 2, retained: []int{0, 1}}, + {name: "retain an interior failure", replicas: 2, expired: 0, retained: []int{0, 1}}, + {name: "scale-in and remaining expired suffix", replicas: 1, expired: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Publish four externally managed replicas with one expired request") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = nil + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + selected.plan.Replicas = 4 + _, missing, changed := resolvePipelineRequests(child, nil, selected.workload, selected.plan) + require.False(t, changed) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + pcs := findLPXTestPodCliqueSet(t, objects) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + require.NoError(t, r.reconcilePipelineRequests(ctx, child, pcs, missing)) + before := make([]*lpxv1alpha1.LPUPipelineRequest, len(missing)) + for index, request := range missing { + observed := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, request.Name) + phase := lpxv1alpha1.RequestPhaseBound + if index == tc.expired { + phase = lpxv1alpha1.RequestPhasePending + } + observed.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now().Add(-time.Hour), phase).Status + require.NoError(t, r.Update(ctx, observed)) + before[index] = observed + } + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.Equal(t, time.Nanosecond, result.RequeueAfter) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + failure := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition).DeepCopy() + require.Equal(t, metav1.ConditionTrue, failure.Status) + + t.Log("Lower capacity externally while keeping the same failed input revision") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + group.Spec.Replicas = tc.replicas + require.NoError(t, r.Update(ctx, group)) + beforeGroup := group.DeepCopy() + deleted := make(map[client.ObjectKey]bool) + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + key := client.ObjectKeyFromObject(object) + require.False(t, deleted[key], "scale-in and deadline cleanup must not repeat a deletion in one pass") + deleted[key] = true + return delegated.Delete(ctx, object, opts...) + }, + }) + for range 3 { + clear(deleted) + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result) + } + + t.Log("Finish scale-in and any eligible deadline suffix without scaling or replacing retained requests") + for index, original := range before { + if slices.Contains(tc.retained, index) { + observed := getTestPipelineRequest(t, ctx, r.Client, original.Namespace, original.Name) + require.True(t, apiequality.Semantic.DeepEqual(original, observed)) + } else { + requirePipelineRequestNotFound(t, ctx, r.Client, original.Namespace, original.Name) + } + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, beforeGroup, group) + require.NoError(t, r.Get(ctx, request.NamespacedName, child)) + require.Equal(t, failure, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) + + t.Log("A later external scale-out does not bypass the failed generation's publication gate") + group.Spec.Replicas = 4 + require.NoError(t, r.Update(ctx, group)) + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + for index := int(tc.replicas); index < len(before); index++ { + requirePipelineRequestNotFound(t, ctx, r.Client, before[index].Namespace, before[index].Name) + } + }) + } +} func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { for _, scenario := range []struct { - name, reason, message string - preserve bool + name, message string + preserve bool }{ - {name: "intent", reason: "LPXRejected", message: "providerOverride"}, - {name: "selected workload", reason: "LPXRejected", message: "replica count"}, - {name: "render", reason: "LPXReconciliationFailed", message: "model storage volume mount"}, - {name: "invalid source", reason: "LPXReconciliationFailed", message: "source"}, - {name: "transient snapshot", reason: "LPXReconciliationFailed", message: "temporary snapshot timeout", preserve: true}, - {name: "inconsistent snapshot", reason: "LPXReconciliationFailed", message: "immutable LPX build snapshot"}, + {name: "render", message: "model storage volume mount"}, + {name: "invalid source", message: "source"}, + {name: "transient snapshot", message: "temporary snapshot timeout", preserve: true}, + {name: "inconsistent snapshot", message: "immutable LPX build snapshot"}, } { t.Run(scenario.name, func(t *testing.T) { t.Log("Stage a PCS and discovery endpoint before the API supplies a PCS UID") - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - r := newLPXTestReconciler(t, registry, child, source) + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + r := newLPXTestReconciler(t, registry, child, dgd) request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} _, err := r.Reconcile(t.Context(), request) require.NoError(t, err) @@ -784,14 +962,10 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { beforePCS, beforeEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() t.Log("Introduce a current terminal input failure, or a transient snapshot outage") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - component := lpx.ServingComponent(source) + component := lpx.ServingComponent(dgd) switch scenario.name { - case "intent": - source.Spec.ProviderOverride = &v1beta1.ProviderOverride{Target: "PodCliqueSet"} - case "selected workload": - component.Replicas = ptr.To(int32(-1)) case "render": component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].VolumeMounts = nil case "invalid source": @@ -803,17 +977,17 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { } r.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: cause} } - source.Generation++ - require.NoError(t, r.Update(t.Context(), source)) + dgd.Generation++ + require.NoError(t, r.Update(t.Context(), dgd)) child.Generation++ - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") require.NoError(t, err) require.NoError(t, r.Update(t.Context(), child)) t.Log("Report the actionable failure without destroying existing resources") for range 3 { _, err = r.Reconcile(t.Context(), request) - if scenario.reason == "LPXReconciliationFailed" && err != nil { + if err != nil { require.ErrorContains(t, err, scenario.message) } else { require.NoError(t, err) @@ -822,10 +996,10 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").Message, scenario.message) } require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - failure := meta.FindStatusCondition(child.Status.Conditions, "Failed") + failure := meta.FindStatusCondition(child.Status.Conditions, "Ready") require.NotNil(t, failure) - require.Equal(t, metav1.ConditionTrue, failure.Status) - require.Equal(t, scenario.reason, failure.Reason) + require.Equal(t, metav1.ConditionFalse, failure.Status) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, failure.Reason) require.Contains(t, failure.Message, scenario.message) require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) @@ -837,244 +1011,23 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { } } -func TestLPXTerminalCleanupPreservesForeignObjectsAndNewerAuthority(t *testing.T) { - const replacedEndpoint = "replaced endpoint" - for _, scenario := range []string{"cleanup", "new child generation", replacedEndpoint, "updated PCS", "pending finalizer"} { - t.Run(scenario, func(t *testing.T) { - t.Log("Seed obsolete owned staging beside ordinary and foreign objects at current target names") - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - child := newLPXTestDeployment(t, source) - owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} - labels := map[string]string{lpxOwnerUIDLabel: string(child.UID)} - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-pcs", Namespace: child.Namespace, UID: "old-pcs", Labels: labels, OwnerReferences: owner, - }, Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}} - endpoint := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-endpoint", Namespace: child.Namespace, UID: "old-endpoint", Labels: labels, OwnerReferences: owner, - }} - runtimeConfig := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "obsolete-runtime", Namespace: child.Namespace, UID: "old-runtime", Labels: labels, OwnerReferences: owner, - }} - ordinary := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ - Name: "ordinary-model", Namespace: child.Namespace, UID: "ordinary-service", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, - }} - foreignPCS, foreignEndpoint := pcs.DeepCopy(), endpoint.DeepCopy() - foreignRuntime := runtimeConfig.DeepCopy() - foreignPCS.UID, foreignEndpoint.UID = "foreign-pcs", "foreign-endpoint" - foreignRuntime.UID = "foreign-runtime" - foreignPCS.Name = dynamo.PCSNameForLPX(child) - foreignEndpoint.Name = foreignPCS.Name + "-serve" - foreignRuntime.Name = foreignPCS.Name + "-runtime" - foreignPCS.OwnerReferences, foreignEndpoint.OwnerReferences = ordinary.OwnerReferences, ordinary.OwnerReferences - foreignRuntime.OwnerReferences = ordinary.OwnerReferences - if scenario == "pending finalizer" { - pcs.Finalizers = []string{"example.com/staged-cleanup"} - } - r := newLPXTestReconciler(t, nil, child, source, pcs, endpoint, runtimeConfig, ordinary, foreignPCS, foreignEndpoint, foreignRuntime) - for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint, foreignRuntime} { - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) - } - - t.Log("Race observed child or resource authority immediately before terminal deletion") - deletes := 0 - wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if err := delegated.List(ctx, list, opts...); err != nil { - return err - } - if _, cliques := list.(*grovev1alpha1.PodCliqueSetList); cliques && scenario == "new child generation" { - current := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(child), current)) - current.Generation++ - return delegated.Update(ctx, current) - } - return nil - }, - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - deletes++ - preconditions := (&client.DeleteOptions{}).ApplyOptions(opts).Preconditions - require.NotNil(t, preconditions) - require.Equal(t, object.GetUID(), *preconditions.UID) - require.Equal(t, object.GetResourceVersion(), *preconditions.ResourceVersion) - if scenario == replacedEndpoint && object.GetName() == endpoint.Name || scenario == "updated PCS" && object.GetName() == pcs.Name { - current := object.DeepCopyObject().(client.Object) - require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(object), current)) - if scenario == replacedEndpoint { - current.SetUID("replacement-endpoint") - } - current.SetLabels(map[string]string{"example.com/newer": "true"}) - require.NoError(t, delegated.Update(ctx, current)) - } - return delegated.Delete(ctx, object, opts...) - }, - }) - r.Client, r.apiReader = wrapped, wrapped - retiring, err := r.retireInvalidLPXWorkload(t.Context(), child, pcs, "invalid runtime") - switch scenario { - case "new child generation": - require.ErrorContains(t, err, "authority changed") - require.Zero(t, deletes) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(endpoint), &corev1.Service{})) - case replacedEndpoint, "updated PCS": - require.True(t, apierrors.IsConflict(err), "object resource-version precondition must reject the stale deletion: %v", err) - object := client.Object(pcs) - if scenario == replacedEndpoint { - object = endpoint - } - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) - require.Equal(t, "true", object.GetLabels()["example.com/newer"]) - default: - require.NoError(t, err) - require.NotNil(t, retiring) - if scenario == "pending finalizer" { - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) - require.False(t, pcs.DeletionTimestamp.IsZero()) - retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, pcs, "invalid runtime") - require.NoError(t, err) - require.NotNil(t, retiring) - pcs.Finalizers = nil - require.NoError(t, r.Update(t.Context(), pcs)) - } - retiring, err = r.retireInvalidLPXWorkload(t.Context(), child, nil, "invalid runtime") - require.NoError(t, err) - require.Nil(t, retiring) - require.Equal(t, 3, deletes, "cleanup must not repeat deletes after they are accepted") - } - - t.Log("Ordinary DGD resources and foreign same-name objects remain byte-for-byte unchanged") - for _, object := range []client.Object{ordinary, foreignPCS, foreignEndpoint, foreignRuntime} { - before := object.DeepCopyObject().(client.Object) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) - require.Equal(t, before, object) - } - }) - } -} - -func TestLPXEndpointLifecycle(t *testing.T) { - const uppercaseComponentName = "LPX" - t.Log("Configure an uppercase LPX component and independently named materialization using Kubernetes discovery") - source := newLPXTestSource(lpx.PipelineSingle, "test-build") - source.Spec.Components[0].ComponentName = uppercaseComponentName - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - source.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} - source.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} - source.Spec.Labels = map[string]string{"example.com/policy": "enabled"} - child := newLPXTestDeployment(t, source) - child.Name = "independent-materialization" - r := newLPXTestReconciler(t, nil, child, source) - - t.Log("Keep the ordinary model Service under the source DGD's ownership") - modelService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.GenerateServiceName("test/model"), Namespace: source.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, - Labels: map[string]string{consts.KubeLabelDynamoGraphDeploymentName: source.Name}, - }, - Spec: corev1.ServiceSpec{ClusterIP: corev1.ClusterIPNone, - Selector: map[string]string{consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("test/model")}}, - } - require.NoError(t, r.Create(t.Context(), modelService)) - modelKey := client.ObjectKeyFromObject(modelService) - require.NoError(t, r.Get(t.Context(), modelKey, modelService)) - beforeModelService := modelService.DeepCopy() - - t.Log("Publish the LPX-owned endpoint with its serving-role selector") - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - service := &corev1.Service{} - key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForLPX(child) + "-serve"} - require.NoError(t, r.Get(t.Context(), key, service)) - require.True(t, metav1.IsControlledBy(service, child)) - require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) - require.Equal(t, dynamo.PCSNameForLPX(child), service.Spec.Selector[grovecommon.LabelPartOfKey]) - - t.Log("Converge propagated endpoint metadata without losing identity, content or sync bookkeeping") - for _, value := range []string{"changed", ""} { - if value == "" { - source.Spec.Labels, source.Spec.Annotations = nil, nil - } else { - source.Spec.Labels["example.com/policy"] = value - source.Spec.Annotations["example.com/model-discovery"] = value - } - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - updated := &corev1.Service{} - require.NoError(t, r.Get(t.Context(), key, updated)) - require.Equal(t, value, updated.Labels["example.com/policy"]) - require.Equal(t, value, updated.Annotations["example.com/model-discovery"]) - require.Equal(t, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled], updated.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) - if value == "" { - require.NotContains(t, updated.Labels, "example.com/policy") - require.NotContains(t, updated.Annotations, "example.com/model-discovery") - } - require.Equal(t, service.Spec, updated.Spec) - require.Equal(t, service.OwnerReferences, updated.OwnerReferences) - for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { - require.NotEmpty(t, updated.Annotations[key]) - require.Equal(t, service.Annotations[key], updated.Annotations[key]) - } - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - unchanged := &corev1.Service{} - require.NoError(t, r.Get(t.Context(), key, unchanged)) - require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) - } - - t.Log("Preserve API-allocated Service fields when the endpoint selector changes") - require.NoError(t, r.Get(t.Context(), key, service)) - service.Spec.ClusterIP = "10.96.0.10" - service.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} - service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} - service.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyPreferDualStack) - require.NoError(t, r.Update(t.Context(), service)) - allocated := service.DeepCopy() - for _, global := range []bool{true, false} { - source.Spec.Components[0].GlobalDynamoNamespace = global - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - require.NoError(t, r.Get(t.Context(), key, service)) - allocated.Spec.Selector[consts.KubeLabelDynamoNamespace] = source.GetDynamoNamespaceForComponent(&source.Spec.Components[0]) - require.Equal(t, allocated.Spec, service.Spec) - require.Equal(t, allocated.OwnerReferences, service.OwnerReferences) - - t.Log("Reconcile the unchanged endpoint without another write") - version := service.ResourceVersion - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - require.NoError(t, r.Get(t.Context(), key, service)) - require.Equal(t, version, service.ResourceVersion) - } - - t.Log("Remove the LPX-owned discovery endpoint when switching back to non-Kubernetes discovery") - delete(source.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) - - t.Log("Retire the recreated discovery endpoint when the workload becomes invalid") - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - require.NoError(t, r.reconcileEndpoint(t.Context(), child, source)) - _, err := r.retireInvalidLPXWorkload(t.Context(), child, nil, "invalid source") - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) - require.NoError(t, r.Get(t.Context(), modelKey, modelService)) - require.Equal(t, beforeModelService, modelService, "LPX cleanup must leave the DGD-owned model Service unchanged") -} - -func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { +func TestLPXMaterializationUsesDGDAndChildIdentity(t *testing.T) { t.Log("Render a materialization with a different name from its source DGD") - _, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - child := newLPXTestDeployment(t, source) + _, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + child := newLPXTestDeployment(t, dgd) child.Name = "independent-materialization" child.UID = "independent-materialization-uid" - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, t.Context(), r.Client, objects...) publishSelectedLPXForTest(t, t.Context(), r, child, selected) t.Log("Route Grove objects to the materialization while retaining the source provenance") pcs := objects[0].(*grovev1alpha1.PodCliqueSet) - require.Equal(t, source.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, dgd.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) for _, object := range objects[1:] { - require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(child)}}, mapLPXChildToRequests(t.Context(), object)) + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(child)}}, mapChildToLPXGraphDeployment(t.Context(), object)) } t.Log("Publish the runtime configuration and endpoint under the exact LGD owner") @@ -1092,61 +1045,249 @@ func TestLPXMaterializationUsesOwnerSourceAndChildIdentity(t *testing.T) { } t.Log("Keep scheduler-request provenance rooted in the source DGD, not the independently named child") - requests, err := r.listOwnedLPXRequests(t.Context(), child, pcs) + requests, err := r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) require.NotEmpty(t, requests) for _, request := range requests { - require.Equal(t, source.Name, request.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, dgd.Name, request.Labels[consts.KubeLabelDynamoGraphDeploymentName]) } } -func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { - t.Log("Publish a speculative engine with long deployment and component names") - _, source, registry := newLPXSpecDecodeTestDGD(t) - source.Name = "test-models-gpt-oss-20b-lp20-b300" - source.Spec.Components[0].ComponentName = "serving-component-with-name" - source.Spec.Components[1].ComponentName = "draft-component-with-name" - source.Spec.Components[1].Replicas = ptr.To(int32(1)) - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - child := newLPXTestDeployment(t, source) - r := newLPXTestReconciler(t, registry, child, source) - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - _, err := r.Reconcile(t.Context(), request) - require.NoError(t, err) - root := dynamo.PCSNameForLPX(child) - require.True(t, strings.HasPrefix(root, source.Name+"-")) - require.Len(t, root, len(source.Name)+len("-ffff")) - pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: root} - require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) - pcs.UID = "stable-pcs" - require.NoError(t, r.Update(t.Context(), pcs)) - endpoint := &corev1.Service{} - endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: root + "-serve"} - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - endpoint.UID = "stable-endpoint" - require.NoError(t, r.Update(t.Context(), endpoint)) - require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) - beforeGroup := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] - beforeOwners := endpoint.OwnerReferences +func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { + t.Log("Publish two requests and remove one without a terminal deadline decision") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) + publishSelectedLPXForTest(t, ctx, r, child, selected) - for _, change := range []string{"rename serving component", "move conductor"} { - t.Log(change) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - if change == "rename serving component" { - source.Spec.Components[0].ComponentName = "renamed-serving-component" - } else { - conductor := *source.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) - source.Spec.Components[0].Roles = slices.DeleteFunc(source.Spec.Components[0].Roles, func(role v1beta1.ComponentRoleSpec) bool { - return role.Name == v1beta1.ComponentRoleLPXConductor - }) - source.Spec.Components[1].Roles = append(source.Spec.Components[1].Roles, conductor) + t.Log("Advance the deployment and remove one request before it fails") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) + + missing := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].Name) + previousUID := missing.UID + require.NoError(t, r.Delete(ctx, missing)) + + t.Log("Reconcile the PCS for the new generation before publishing the missing request") + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result, "the PCS update drives the next reconcile") + requirePipelineRequestNotFound(t, ctx, r.Client, missing.Namespace, missing.Name) + + t.Log("Reconciliation recreates deterministic scheduler intent without inheriting a scheduling clock") + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + replacement := getTestPipelineRequest(t, ctx, r.Client, missing.Namespace, missing.Name) + require.NotEqual(t, previousUID, replacement.UID) + require.Nil(t, replacement.Status) +} + +func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { + ctx := t.Context() + child, dgd, registry := newLPXSpecDecodeTestDGD(t) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + + scaledDGD := dgd.DeepCopy() + require.Len(t, scaledDGD.Spec.Components, 2) + scaledDGD.Spec.Components[1].Replicas = ptr.To(int32(1)) + scaled := resolveLPXTestWorkload(t, r.modelRegistry, ctx, child, scaledDGD) + require.Len(t, scaled.requests, 2) + + desiredNames := make(map[string]struct{}, len(scaled.requests)) + for _, request := range scaled.requests { + desiredNames[request.Name] = struct{}{} + } + var stale *lpxv1alpha1.LPUPipelineRequest + for _, request := range selected.requests { + if _, retained := desiredNames[request.Name]; retained { + continue + } + stale = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, request.Name) + break + } + require.NotNil(t, stale) + stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, stale)) + + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), pcs)) + pcs.Finalizers = []string{"test.example/observe-retirement"} + require.NoError(t, r.Update(ctx, pcs)) + retiredTemplate := selected.plan.Agents[1].TemplateName + originalCliqueCount := len(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + + t.Log("Retire the immutable PCS without mutating Grove membership or directly deleting its LPRs") + pcsDeleteObserved := false + requestDeleteObserved := false + wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + switch object.(type) { + case *grovev1alpha1.PodCliqueSet: + pcsDeleteObserved = true + case *lpxv1alpha1.LPUPipelineRequest: + requestDeleteObserved = true + } + return delegated.Delete(ctx, object, opts...) + }, + }) + r.Client = wrapped + updateTestDGD(t, r, child, scaledDGD) + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + require.True(t, pcsDeleteObserved) + require.False(t, requestDeleteObserved) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.False(t, pcs.DeletionTimestamp.IsZero()) + require.Contains(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames, retiredTemplate) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + stale = getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.True(t, stale.DeletionTimestamp.IsZero()) + + t.Log("Wait while foreground deletion keeps the PCS present") + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + + t.Log("Simulate foreground GC removing dependent requests before the PCS") + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(ctx, requests)) + for index := range requests.Items { + request := &requests.Items[index] + request.Finalizers = nil + require.NoError(t, r.Update(ctx, request)) + require.NoError(t, r.Delete(ctx, request)) + } + pcs.Finalizers = nil + require.NoError(t, r.Update(ctx, pcs)) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{}))) + + t.Log("Create the replacement PCS with the scaled-in immutable shape") + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + synced, err := getPodCliqueSet(ctx, r.Client, child) + require.NoError(t, err) + require.NotNil(t, synced) + require.Equal(t, []string{"cond", "agt0", "agt1"}, synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) + require.Len(t, synced.Spec.Template.Cliques, 3) + require.Equal(t, scaled.workload.Digest().String(), synced.Annotations[lpx.WorkloadDigestAnnotation]) + require.Less(t, len(synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames), originalCliqueCount) +} + +func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { + t.Log("Publish one engine whose capacity is owned by the native Grove scaling group") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(dgd).Replicas = nil + r, initial := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, initial) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, initial) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, initial) + pcsUID := pcs.UID + templateReplicas := ptr.Deref(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0) + require.Positive(t, templateReplicas) + request := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, initial.requests[0].Name) + requestUID := request.UID + + t.Log("Scale the native Grove group to zero and derive empty scheduler intent") + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: initial.plan.LPXScalingGroup}, group)) + group.Spec.Replicas = 0 + require.NoError(t, r.Update(ctx, group)) + + t.Log("Delete the stale LPR before reconciling the stable PCS template") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Zero(t, group.Spec.Replicas) + storedPCS := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) + require.Equal(t, pcsUID, storedPCS.UID) + require.Equal(t, templateReplicas, ptr.Deref(storedPCS.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0)) + + t.Log("Scale the native group back up and publish a fresh request through the same PCS") + group.Spec.Replicas = 1 + require.NoError(t, r.Update(ctx, group)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + replacement := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, initial.requests[0].Name) + require.NotEqual(t, requestUID, replacement.UID) + require.Equal(t, pcsUID, metav1.GetControllerOf(replacement).UID) +} + +func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { + t.Log("Publish a speculative engine with long deployment and component names") + _, dgd, registry := newLPXSpecDecodeTestDGD(t) + dgd.Name = "test-models-gpt-oss-20b-lp20-b300" + dgd.Spec.Components[0].ComponentName = "serving-component-with-name" + dgd.Spec.Components[1].ComponentName = "draft-component-with-name" + dgd.Spec.Components[1].Replicas = ptr.To(int32(1)) + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + child := newLPXTestDeployment(t, dgd) + r := newLPXTestReconciler(t, registry, child, dgd) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), request) + require.NoError(t, err) + root := dynamo.PCSNameForLPX(child) + require.True(t, strings.HasPrefix(root, dgd.Name+"-")) + require.Len(t, root, len(dgd.Name)+len("-ffff")) + pcs := &grovev1alpha1.PodCliqueSet{} + pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: root} + require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) + pcs.UID = "stable-pcs" + require.NoError(t, r.Update(t.Context(), pcs)) + endpoint := &corev1.Service{} + endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: root + "-serve"} + require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) + endpoint.UID = "stable-endpoint" + require.NoError(t, r.Update(t.Context(), endpoint)) + require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) + beforeGroup := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] + beforeOwners := endpoint.OwnerReferences + + t.Log("Observe Grove's group and publish the original requests") + selected := resolveLPXTestWorkload(t, r.modelRegistry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + require.NoError(t, r.Create(t.Context(), group)) + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + + for _, change := range []string{"rename serving component", "move conductor"} { + t.Log(change) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + if change == "rename serving component" { + dgd.Spec.Components[0].ComponentName = "renamed-serving-component" + } else { + conductor := *dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) + dgd.Spec.Components[0].Roles = slices.DeleteFunc(dgd.Spec.Components[0].Roles, func(role v1beta1.ComponentRoleSpec) bool { + return role.Name == v1beta1.ComponentRoleLPXConductor + }) + dgd.Spec.Components[1].Roles = append(dgd.Spec.Components[1].Roles, conductor) } - source.Generation++ - require.NoError(t, r.Update(t.Context(), source)) + dgd.Generation++ + require.NoError(t, r.Update(t.Context(), dgd)) child.Generation++ - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") require.NoError(t, err) require.NoError(t, r.Update(t.Context(), child)) @@ -1156,16 +1297,13 @@ func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { if change == "move conductor" { require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, &grovev1alpha1.PodCliqueSet{}))) require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - require.Equal(t, lpxRetiringReason, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) require.Equal(t, "stable-endpoint", string(endpoint.UID)) - t.Log("Fence replacement until the retired PCS's requests are garbage collected") - _, err = r.Reconcile(t.Context(), request) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, &grovev1alpha1.PodCliqueSet{}))) - requests, err := r.listLPXRequestCandidates(t.Context(), child) - require.NoError(t, err) + t.Log("Simulate dependent request garbage collection, which the fake client does not perform") + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), requests)) require.NotEmpty(t, requests.Items) for index := range requests.Items { require.Equal(t, "stable-pcs", string(metav1.GetControllerOf(&requests.Items[index]).UID)) @@ -1188,8 +1326,8 @@ func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { require.Equal(t, beforeGroup.Name, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Name) require.Equal(t, beforeGroup.CliqueNames, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) require.Equal(t, beforeOwners, endpoint.OwnerReferences) - require.Equal(t, source.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) - require.Equal(t, lpx.ServingComponent(source).ComponentName, endpoint.Spec.Selector[consts.KubeLabelDynamoComponent]) + require.Equal(t, dgd.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, lpx.ServingComponent(dgd).ComponentName, endpoint.Spec.Selector[consts.KubeLabelDynamoComponent]) require.Equal(t, root, endpoint.Spec.Selector[grovecommon.LabelPartOfKey]) require.Equal(t, consts.KubeLabelValueTrue, endpoint.Spec.Selector[dynamo.LPXServingLabel]) allPCS, allEndpoints := &grovev1alpha1.PodCliqueSetList{}, &corev1.ServiceList{} @@ -1235,247 +1373,1472 @@ func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { } } -func TestLPXResourceSyncConvergesContentAndMetadataWithoutAdoption(t *testing.T) { - t.Log("Publish a runtime ConfigMap under the exact LPX child") - source := newLPXTestSource(lpx.PipelineSingle, "test-build") - child := newLPXTestDeployment(t, source) - r := newLPXTestReconciler(t, nil, child, source) - desired := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "lpx-runtime", Namespace: child.Namespace, - Labels: map[string]string{"example.com/policy": "enabled"}, Annotations: map[string]string{"example.com/policy": "enabled"}}, - Data: map[string]string{"config": "initial runtime"}, - } - require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) - recorded := r.recorder.(*events.FakeRecorder).Events - require.Len(t, recorded, 1) - require.Equal(t, "Normal CreateConfigMap Created ConfigMap "+child.Namespace, <-recorded) - key := client.ObjectKeyFromObject(desired) - before := &corev1.ConfigMap{} - require.NoError(t, r.Get(t.Context(), key, before)) - updates := 0 - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { - updates++ - return delegated.Update(ctx, object, opts...) - }, - }) - - t.Log("Change runtime content and remove all authored metadata while retaining refreshed bookkeeping") - desired.Data["config"] = "updated runtime" - desired.Labels, desired.Annotations = nil, nil - desired.OwnerReferences = nil - require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) - require.Len(t, recorded, 1) - require.Equal(t, "Normal UpdateConfigMap Updated ConfigMap "+child.Namespace, <-recorded) - require.Equal(t, 1, updates, "content and metadata must converge in one resource update") - updated := &corev1.ConfigMap{} - require.NoError(t, r.Get(t.Context(), key, updated)) - require.Equal(t, desired.Data, updated.Data) - require.Empty(t, updated.Labels) - require.Len(t, updated.Annotations, 2) - hash, err := commoncontroller.GetSpecHash(desired) - require.NoError(t, err) - require.Equal(t, hash, updated.Annotations[commoncontroller.NvidiaAnnotationHashKey]) - require.NotEqual(t, before.Annotations[commoncontroller.NvidiaAnnotationHashKey], hash) - require.NotEmpty(t, updated.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) - require.Equal(t, before.OwnerReferences, updated.OwnerReferences) - require.NoError(t, r.syncLPXResource(t.Context(), child, desired)) - require.Empty(t, recorded) - unchanged := &corev1.ConfigMap{} - require.NoError(t, r.Get(t.Context(), key, unchanged)) - require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) - - t.Log("A foreign resource at the same key is never adopted or rewritten") - updated.OwnerReferences[0].UID = "foreign-child" - require.NoError(t, r.Update(t.Context(), updated)) - desired.Data["config"] = "must-not-publish" - require.ErrorContains(t, r.syncLPXResource(t.Context(), child, desired), "refusing to adopt LPX resource") - require.NoError(t, r.Get(t.Context(), key, unchanged)) - require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) - require.Equal(t, updated.Data, unchanged.Data) -} - -func TestLPXDeletesOnlyStaleOwnedRuntimeConfigMaps(t *testing.T) { - t.Log("Create current, stale, and foreign runtime ConfigMaps") - source := newLPXTestSource(lpx.PipelineSingle, "test-build") - child := newLPXTestDeployment(t, source) - owner := []metav1.OwnerReference{*metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK)} - current := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "runtime-current", Namespace: child.Namespace, UID: "current", - Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, OwnerReferences: owner, - }} - stale := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "runtime-stale", Namespace: child.Namespace, UID: "stale", - Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, OwnerReferences: owner, - }} - foreign := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "runtime-foreign", Namespace: child.Namespace, UID: "foreign", - Labels: map[string]string{lpxOwnerUIDLabel: string(child.UID)}, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, - }} - r := newLPXTestReconciler(t, nil, child, source, current, stale, foreign) - - t.Log("Delete only the stale ConfigMap owned by the current LPX child") - require.NoError(t, r.deleteStaleLPXConfigMaps(t.Context(), child, []client.Object{current})) +func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { + for _, editDGD := range []bool{true, false} { + name := "child metadata edit" + if editDGD { + name = "source scale-down" + } + t.Run(name, func(t *testing.T) { + t.Log("Materialize Grove for two replicas before publishing scheduler intent") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) - t.Log("Preserve the current and foreign ConfigMaps") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(current), &corev1.ConfigMap{})) - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), client.ObjectKeyFromObject(stale), &corev1.ConfigMap{}))) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(foreign), &corev1.ConfigMap{})) -} + t.Log("Persist the PCS update and re-reconcile before testing LPR publication") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Zero(t, result, "the PCS update drives the next reconcile") + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, requests.Items) -func TestLPXValidatesIntentBeforeDownloadsOrPublication(t *testing.T) { - const componentProvider = "component provider" - for _, scenario := range []struct { - name, componentName string - sharedDraft bool - missingConductor string - messages []string - }{ - {name: "invalid intent", messages: []string{ - "spec.providerOverride: Forbidden:", - "spec.components[0].providerOverride: Forbidden:", - `LPX agent component requires a "main" runtime container`, - "does not support checkpointing", - "spec.topologyConstraint: Forbidden:", - "spec.components[0].topologyConstraint: Forbidden:", - }}, - {name: "missing conductor role", missingConductor: "role", messages: []string{"LPX components must declare exactly one conductor role"}}, - {name: "missing conductor template", missingConductor: "podTemplate", messages: []string{"LPX conductor requires an explicit podTemplate"}}, - {name: "shared draft missing conductor role", sharedDraft: true, missingConductor: "role", messages: []string{"LPX components must declare exactly one conductor role"}}, - {name: "shared draft missing conductor template", sharedDraft: true, missingConductor: "podTemplate", messages: []string{"LPX conductor requires an explicit podTemplate"}}, - {name: "disabled Grove", messages: []string{"Grove is disabled"}}, - {name: componentProvider, messages: []string{"requires the Grove workload provider"}}, - {name: "long serving name", componentName: strings.Repeat("serving-", 7) + "engine"}, - {name: "reordered shared draft with long serving name", componentName: strings.Repeat("serving-", 7) + "engine", sharedDraft: true}, - } { - t.Run(scenario.name, func(t *testing.T) { - t.Log("Reconcile authored LPX intent against cold remote builds") - source := newLPXTestSource(lpx.PipelineSingle, modelDownloadTestBuildID) - if scenario.sharedDraft { - source = newLPXSpecDecodeTestSource() - source.Spec.Components[0].LPX.BuildID = modelDownloadTestBuildID - source.Spec.Components[1].LPX.BuildID = modelDownloadTestSecondBuildID - source.Spec.Components[1].ComponentName = "shared-draft-name-is-not-materialized" - source.Spec.Components = []v1beta1.DynamoComponentDeploymentSharedSpec{ - {ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend}, - source.Spec.Components[1], source.Spec.Components[0], - } - } - component := lpx.ServingComponent(source) - if scenario.componentName != "" { - component.ComponentName = scenario.componentName - } - switch scenario.missingConductor { - case "role": - component.Roles = slices.DeleteFunc(component.Roles, func(role v1beta1.ComponentRoleSpec) bool { - return role.Name == v1beta1.ComponentRoleLPXConductor - }) - case "podTemplate": - component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate = nil - } - switch scenario.name { - case "invalid intent": - source.Spec.ProviderOverride = &v1beta1.ProviderOverride{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueSet", - } - source.Spec.ProviderOverride.Value.Raw = []byte(`{"spec":{"template":{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}}}`) - component.ProviderOverride = &v1beta1.ProviderOverride{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Target: "PodCliqueTemplateSpec", - } - component.ProviderOverride.Value.Raw = []byte(`{"topologyConstraint":{"topologyName":"fabric","pack":{"required":"rack"}}}`) - component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate = &corev1.PodTemplateSpec{} - component.Experimental = &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{Enabled: true}} - source.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} - component.TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} - case componentProvider: - delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) - source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent + t.Log("Edit live input after the first LPR write without changing the reconcile snapshot") + dgdReads, childReads, creates := 0, 0, 0 + wrapped := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + switch object.(type) { + case *v1beta1.DynamoGraphDeployment: + dgdReads++ + case *v1alpha1.LPXGraphDeployment: + childReads++ + } + return delegated.Get(ctx, key, object, opts...) + }, + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if err := delegated.Create(ctx, object, opts...); err != nil { + return err + } + if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + creates++ + if creates == 1 { + if editDGD { + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(1)) + dgd.Generation++ + return delegated.Update(ctx, dgd) + } + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(child), child)) + child.Annotations[lpx.DGDGenerationAnnotation] = "999" + return delegated.Update(ctx, child) + } + } + return nil + }, + }) + r.Client = wrapped + _, err = r.Reconcile(t.Context(), request) + if editDGD { + require.NoError(t, err) + } else { + require.True(t, apierrors.IsConflict(err), "status writes still reject a changed child: %v", err) } - registry, err := lpx.NewModelRegistry("", nil) + require.Equal(t, 1, dgdReads) + require.Equal(t, 1, childReads) + require.Equal(t, 2, creates, "both requests use the validated two-replica snapshot") + retained := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, selected.requests[0].Name) + + t.Log("A subsequent reconcile waits for the updated handoff before retiring any LPR") + _, err = r.Reconcile(t.Context(), request) require.NoError(t, err) - observedRegistry := &fakeModelDownloadRegistry{ModelRegistry: registry} - child := newLPXTestDeployment(t, source) - r := newLPXTestReconciler(t, observedRegistry, child, source) - if source.Spec.ProviderOverride != nil || component.ProviderOverride != nil { - t.Log("Match the stored provider override after its raw JSON is normalized") - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - revision, err := dynamo.LPXInputRevision(source, "") + require.NoError(t, r.List(t.Context(), requests)) + require.Len(t, requests.Items, 2) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas, "an incomplete handoff must not scale Grove") + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + if editDGD { + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") require.NoError(t, err) - child.Spec.InputRevision = revision + child.Generation++ require.NoError(t, r.Update(t.Context(), child)) } - if scenario.name == "disabled Grove" { - r.runtimeConfig.Gate.Grove = false - } - t.Log("Reject unsupported intent before downloads and let valid names reach the download gate") - before := source.DeepCopy() - _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - if scenario.name == componentProvider { - require.ErrorContains(t, err, "requires the Grove workload provider") + t.Log("Converge the new inputs without replacing the retained replica's request") + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + require.Equal(t, *lpx.ServingComponent(dgd).Replicas, group.Spec.Replicas) + require.NoError(t, r.List(t.Context(), requests)) + require.Len(t, requests.Items, int(group.Spec.Replicas)) + require.Equal(t, retained.UID, getTestPipelineRequest(t, t.Context(), r.Client, retained.Namespace, retained.Name).UID) + }) + } +} + +func TestLPXReconcileWaitsForMatchingDGDRevision(t *testing.T) { + for _, dgdAhead := range []bool{true, false} { + name := "child ahead" + if dgdAhead { + name = "source ahead" + } + t.Run(name, func(t *testing.T) { + t.Log("Observe the source and child at different handoff revisions") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r := newLPXTestReconciler(t, registry, child, dgd) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.Generation++ + if dgdAhead { + require.NoError(t, r.Update(t.Context(), dgd)) } else { + var err error + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(t.Context(), child)) } - stored := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) - failed := meta.FindStatusCondition(stored.Status.Conditions, "Failed") + + t.Log("Wait without failing or publishing any workload from mismatched inputs") + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Zero(t, result, "source and child watches resume the handoff") + require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) + require.Len(t, child.Status.Conditions, 1) + observed := meta.FindStatusCondition(child.Status.Conditions, "Ready") + require.NotNil(t, observed) + require.Equal(t, metav1.ConditionFalse, observed.Status) + require.Equal(t, v1alpha1.LPXReadyReasonPending, observed.Reason) + allPCS := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, r.List(t.Context(), allPCS)) + require.Empty(t, allPCS.Items) + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, requests.Items) + + t.Log("Resume publication when the matching source or child reaches the cache") + if dgdAhead { + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(t.Context(), child)) + } else { + require.NoError(t, r.Update(t.Context(), dgd)) + } + _, err = r.Reconcile(t.Context(), request) + require.NoError(t, err) + require.NoError(t, r.List(t.Context(), allPCS)) + require.Len(t, allPCS.Items, 1) + }) + } +} + +func TestScalePodCliqueScalingGroup(t *testing.T) { + conflict := apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), "group", errors.New("stale version")) + for _, tc := range []struct { + name string + replicas *int32 + err error + wantUpdates int + }{ + {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, + {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, + {name: "scale to zero", replicas: ptr.To(int32(0)), wantUpdates: 1}, + {name: "unchanged", replicas: ptr.To(int32(9))}, + {name: "omitted replicas"}, + {name: "conflict", replicas: ptr.To(int32(12)), err: conflict, wantUpdates: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Use an already-owned group observation without rendering a workload") + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "group", Namespace: "default", ResourceVersion: "7"}, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 9}, + } + updates := 0 + r := &graphReconciler{Client: interceptor.NewClient(newLPXTestClient(t), interceptor.Funcs{ + SubResourceUpdate: func(_ context.Context, _ client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "scale", subresource) + require.Same(t, group, object) + scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) + require.Equal(t, group.ResourceVersion, scale.ResourceVersion) + require.Equal(t, *tc.replicas, scale.Spec.Replicas) + updates++ + return tc.err + }, + })} + + t.Log("Only explicit changes write scale; successful writes update the in-pass observation") + err := r.scalePodCliqueScalingGroup(t.Context(), group, tc.replicas) + require.ErrorIs(t, err, tc.err) + require.Equal(t, tc.wantUpdates, updates) + want := int32(9) + if tc.replicas != nil && tc.err == nil { + want = *tc.replicas + } + require.Equal(t, want, group.Spec.Replicas) + }) + } +} + +func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { + t.Log("A request-list failure prevents deriving any active deadline") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, dgd, desired)...) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + readError := errors.New("temporary request observation failure") + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, requests := list.(*lpxv1alpha1.LPUPipelineRequestList); requests { + return readError + } + return delegated.List(ctx, list, opts...) + }, + }) + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.ErrorIs(t, err, readError) + require.Zero(t, result) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.NotEqual(t, child.Generation, child.Status.ObservedGeneration) + require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").Message, readError.Error()) +} + +func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { + t.Log("Observe an externally inflated group through a cache that stays stale after scale") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + group.Spec.Replicas = 9 + createLPXTestObjects(t, ctx, r.Client, objects...) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + stale := group.DeepCopy() + scaled, updates := false, 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if scaled && key == client.ObjectKeyFromObject(stale) { + if observed, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { + stale.DeepCopyInto(observed) + return nil + } + } + return delegated.Get(ctx, key, object, opts...) + }, + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if subresource != lpxTestScaleSubresource { + return delegated.SubResource(subresource).Update(ctx, object, opts...) + } + updates++ + if err := delegated.SubResource(subresource).Update(ctx, object, opts...); err != nil { + return err + } + scaled = true + return nil + }, + }) + + t.Log("Continue reconciliation without issuing a second stale-resource-version write") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Equal(t, 1, updates) +} + +func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + + stale := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[1].Name) + stale.Finalizers = []string{"scheduling.lpu.nvidia.com/plan-protection"} + require.NoError(t, r.Update(ctx, stale)) + + liveDGD := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), liveDGD)) + lpx.ServingComponent(liveDGD).Replicas = ptr.To(int32(1)) + liveDGD.Generation++ + require.NoError(t, r.Update(ctx, liveDGD)) + liveChild := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) + liveChild.Generation++ + var err error + liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveDGD, "") + require.NoError(t, err) + require.NoError(t, r.Update(ctx, liveChild)) + + t.Log("A missing PCSG observation blocks scale-in without deleting existing requests") + base := r.Client + r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { + return apierrors.NewNotFound(consts.PodCliqueScalingGroupGVR.GroupResource(), key.Name) + } + return delegated.Get(ctx, key, object, opts...) + }, + }) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result, "the PCSG watch resumes scale-in") + require.Equal(t, stale, getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name)) + r.Client = base + + t.Log("Persist the lower Grove scale before request retirement can block reconciliation") + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result, "request deletion progresses through watches") + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Equal(t, int32(1), group.Spec.Replicas) + t.Log("Let request finalization proceed asynchronously after the lower scale is persisted") + stale = getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.False(t, stale.DeletionTimestamp.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), &grovev1alpha1.PodCliqueSet{})) + + t.Log("Reverse the scale-down while the old request still protects its pods") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), liveDGD)) + lpx.ServingComponent(liveDGD).Replicas = ptr.To(int32(2)) + liveDGD.Generation++ + require.NoError(t, r.Update(ctx, liveDGD)) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) + liveChild.Generation++ + liveChild.Spec.InputRevision, err = dynamo.LPXInputRevision(liveDGD, "") + require.NoError(t, err) + require.NoError(t, r.Update(ctx, liveChild)) + for range 2 { + result, err = r.Reconcile(ctx, request) + require.NoError(t, err) + require.Zero(t, result, "terminating current names stop scale-out without polling") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(1), group.Spec.Replicas, "Grove must not recreate pods protected by the retiring request") + } + + t.Log("Allow scale-up and a fresh request once the scheduler finishes cleanup") + stale = getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + retiredUID := stale.UID + stale.Finalizers = nil + require.NoError(t, r.Update(ctx, stale)) + for range 4 { + _, err = r.Reconcile(ctx, request) + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(2), group.Spec.Replicas) + replacement := getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) + require.NotEqual(t, retiredUID, replacement.UID) + require.True(t, replacement.DeletionTimestamp.IsZero()) +} + +func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { + for _, tc := range []struct { + name string + minimum *int32 + }{ + {name: "default minimum"}, + {name: "explicit minimum", minimum: ptr.To(int32(2))}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Start without Grove resources and create the owned PCS at its immutable minimum") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + component := lpx.ServingComponent(dgd) + component.Replicas, component.MinAvailable = ptr.To(int32(3)), tc.minimum + r := newLPXTestReconciler(t, registry, child, dgd) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Zero(t, result, "Grove watches resume reconciliation") + pcs, err := getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + require.NotNil(t, pcs) + require.True(t, metav1.IsControlledBy(pcs, child)) + seed := ptr.Deref(tc.minimum, 1) + groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] + require.Equal(t, seed, *groupTemplate.Replicas) + + t.Log("Wait without publishing requests while Grove has not created its scaling group") + result, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Zero(t, result) + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Empty(t, requests) + + t.Log("Observe Grove's group initialized from the PCS template") + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, groupTemplate.Name), + Namespace: pcs.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: seed}, + } + require.NoError(t, r.Create(t.Context(), group)) + + t.Log("Publish requests in ordinal order only after live capacity reaches the explicit desired count") + var published []int64 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Equal(t, int32(3), group.Spec.Replicas) + require.True(t, metav1.IsControlledBy(request, pcs)) + require.True(t, ptr.Deref(metav1.GetControllerOf(request).BlockOwnerDeletion, false)) + published = append(published, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex) + } + return delegated.Create(ctx, object, opts...) + }, + }) + result, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Zero(t, result, "Request creation events resume reconciliation") + require.Equal(t, []int64{0, 1, 2}, published) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) + + t.Log("A later observation neither recreates requests nor copies live replicas into the template") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Equal(t, []int64{0, 1, 2}, published) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.Equal(t, seed, *pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas) + }) + } +} + +func TestLPXTerminatingGroveResourcesBlockPublication(t *testing.T) { + t.Log("Publish a workload whose PCS and scaling group are both terminating") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := observedLPXTestPodCliqueSet(t, ctx, r, child, selected) + pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, pending)) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + for _, object := range []client.Object{pcs, group} { + object.SetFinalizers([]string{"example.com/cleanup"}) + require.NoError(t, r.Update(ctx, object)) + require.NoError(t, r.Delete(ctx, object)) + } + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + require.False(t, pcs.DeletionTimestamp.IsZero()) + + t.Log("Let the publication fence report retirement before attempting scale") + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + require.Zero(t, result, "PCS deletion is observed through its watch") + + t.Log("Whole-workload retirement does not expire or delete surviving requests") + for range 2 { + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + require.Nil(t, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) + require.True(t, getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name).DeletionTimestamp.IsZero()) + } +} + +const lpxTestScaleSubresource = "scale" + +// updateTestDGD emulates the DGD controller publishing a new child revision. +func updateTestDGD(t *testing.T, r *graphReconciler, child *v1alpha1.LPXGraphDeployment, desired *v1beta1.DynamoGraphDeployment) { + t.Helper() + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(desired), dgd)) + dgd.Spec = *desired.Spec.DeepCopy() + dgd.Annotations = desired.DeepCopy().Annotations + dgd.Generation++ + require.NoError(t, r.Update(t.Context(), dgd)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + child.Generation++ + var err error + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, child.Annotations[dynamo.LPXRestartAnnotation]) + require.NoError(t, err) + require.NoError(t, r.Update(t.Context(), child)) +} + +func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { + t.Log("Materialize two draft instances and one target in their single shared scaling group") + deployment, dgd, registry := newLPXSpecDecodeTestDGD(t) + prepare, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), deployment, dgd) + objects := lpxMaterializedObjects(t, prepare, deployment, dgd, selected) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + } + } + pcs := findLPXTestPodCliqueSet(t, objects) + + t.Log("Observe each draft condition against an independent copy of the ready baseline") + for _, scenario := range []string{"ready", "partial", "missing", "foreign", "old revision", "unobserved"} { + t.Run(scenario, func(t *testing.T) { + t.Log("Seed isolated Grove observations and fetch the first draft") + kubeClient := newLPXTestClient(t) + createLPXTestObjects(t, t.Context(), kubeClient, objects...) + firstDraft := getResource[*grovev1alpha1.PodClique](t, objects, selected.plan.Agents[0].CliqueName).DeepCopy() + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(firstDraft), firstDraft)) + require.Equal(t, "draft", firstDraft.Labels[lpx.StageLabel]) + + t.Log("Make only the first draft incomplete while retaining a fully ready second draft and target") + switch scenario { + case "partial": + firstDraft.Status.ReadyReplicas-- + case "foreign": + firstDraft.OwnerReferences = []metav1.OwnerReference{{UID: "foreign", Controller: ptr.To(true)}} + case "old revision": + firstDraft.Status.CurrentPodCliqueSetGenerationHash = ptr.To("old") + case "unobserved": + firstDraft.Status.ObservedGeneration = ptr.To(firstDraft.Generation - 1) + } + if scenario == "missing" { + require.NoError(t, kubeClient.Delete(t.Context(), firstDraft)) + } else { + require.NoError(t, kubeClient.Update(t.Context(), firstDraft)) + } + + t.Log("Project logical draft counts and make both authored components await the complete pair") + readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, dgd, pcs, getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup)) + require.NoError(t, err) + require.Len(t, readiness.ComponentStatuses, 2) + draft, target := readiness.ComponentStatuses["draft"], readiness.ComponentStatuses["lpx"] + require.Equal(t, v1beta1.ComponentKindPodClique, draft.ComponentKind) + require.Len(t, draft.ComponentNames, 2) + require.Equal(t, int32(1), target.Replicas) + if scenario == "ready" { + require.True(t, readiness.Ready) + require.Equal(t, int32(2), draft.Replicas) + require.Equal(t, int32(2), draft.UpdatedReplicas) + require.Equal(t, ptr.To(int32(2)), draft.ReadyReplicas) + require.Equal(t, ptr.To(int32(2)), draft.ScheduledReplicas) + require.Equal(t, ptr.To(int32(1)), target.AvailableReplicas) + } else { + require.False(t, readiness.Ready) + require.Equal(t, ptr.To(int32(1)), draft.ReadyReplicas) + require.Equal(t, ptr.To(int32(0)), target.AvailableReplicas) + } + require.Equal(t, readiness.Ready, draft.Ready) + require.Equal(t, readiness.Ready, target.Ready) + }) + } +} + +func newLPXTestScheme(t testing.TB) *k8sruntime.Scheme { + t.Helper() + scheme := k8sruntime.NewScheme() + for _, add := range []func(*k8sruntime.Scheme) error{ + corev1.AddToScheme, resourcev1.AddToScheme, v1alpha1.AddToScheme, v1beta1.AddToScheme, + grovev1alpha1.AddToScheme, lpxv1alpha1.AddToScheme, + } { + require.NoError(t, add(scheme)) + } + return scheme +} + +type snapshotFailureRegistry struct { + lpx.ModelRegistry + err error +} + +func (r *snapshotFailureRegistry) AcquireBuildSnapshot(context.Context, string) (*lpx.BuildSnapshot, error) { + return nil, r.err +} + +type downloadOrderedLPXRegistry struct { + lpx.ModelRegistry + buildURL url.URL + downloaded bool + calls []string +} + +func (r *downloadOrderedLPXRegistry) AcquireBuildSnapshot( + ctx context.Context, + buildID string, +) (*lpx.BuildSnapshot, error) { + r.calls = append(r.calls, "snapshot") + if !r.downloaded { + return nil, errors.New("LPX snapshot acquired before Model Express download") + } + return r.ModelRegistry.AcquireBuildSnapshot(ctx, buildID) +} + +func (r *downloadOrderedLPXRegistry) BuildURL(string) (*url.URL, error) { + buildURL := r.buildURL + return &buildURL, nil +} + +func (r *downloadOrderedLPXRegistry) EnsureDownloaded(context.Context, url.URL) (bool, error) { + r.calls = append(r.calls, "download") + if len(r.calls) == 1 { + return false, nil + } + r.downloaded = true + return true, nil +} + +func TestSelectedLPXColdCacheDownloadsBeforeSnapshot(t *testing.T) { + t.Log("Build a selected LPX deployment backed by an initially cold Model Express cache") + child, dgd, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) + registry := &downloadOrderedLPXRegistry{ + ModelRegistry: baseRegistry, + buildURL: url.URL{ + Scheme: lpx.BuildSchemeGCS, + Host: "test-bucket", + Path: "/build-v2", + }, + } + reconciler := newLPXTestReconciler(t, registry, child, dgd) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + + t.Log("Check Model Express before acquiring the initial build snapshot") + result, err := reconciler.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Equal(t, modelDownloadRequeueAfter, result.RequeueAfter) + require.Equal(t, []string{"download"}, registry.calls) + require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, "Ready")) + require.NotNil(t, child.Status.ModelDownload) + require.Empty(t, child.Status.ModelDownload.Builds) + sets := &grovev1alpha1.PodCliqueSetList{} + require.NoError(t, reconciler.List(t.Context(), sets)) + require.Empty(t, sets.Items) + + t.Log("Complete the download and reconcile the selected deployment again") + _, err = reconciler.Reconcile(t.Context(), request) + require.NoError(t, err) + require.Equal(t, []string{"download", "download", "snapshot"}, registry.calls) + require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) + require.NotNil(t, child.Status.ModelDownload) + require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) +} + +func TestLPXPublishedWorkloadCleanupAfterFailure(t *testing.T) { + for _, test := range []struct { + name string + snapshotError error + }{ + {name: "inconsistent snapshot", snapshotError: fmt.Errorf("%w: compiler metadata changed during duplicate reads", lpx.ErrBuildSnapshotInconsistent)}, + {name: "transient snapshot", snapshotError: errors.New("temporary object-store timeout")}, + {name: "render failure"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Publish a request backed by the rendered LPX workload") + ctx := t.Context() + deployment, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) + createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, deployment, dgd, desired)...) + condition := publishSelectedLPXForTest(t, ctx, reconciler, deployment, desired) + require.NotNil(t, condition) + require.Equal(t, v1alpha1.LPXReadyReasonPending, condition.Reason) + requestBefore := getTestPipelineRequest(t, ctx, reconciler.Client, deployment.Namespace, desired.requests[0].Name) + key := client.ObjectKeyFromObject(deployment) + pcsKey := client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName} + pcsBefore := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, reconciler.Get(ctx, pcsKey, pcsBefore)) + + t.Log("Fail the actual snapshot dependency or break cross-role model storage") + message := "must use the Conductor model-storage mount path" + if test.snapshotError != nil { + reconciler.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: test.snapshotError} + message = test.snapshotError.Error() + } else { + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) + conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) + for index := range conductor.PodTemplate.Spec.Containers[0].VolumeMounts { + mount := &conductor.PodTemplate.Spec.Containers[0].VolumeMounts[index] + if mount.Name == consts.ModelStorageVolumeName { + mount.MountPath = "/different-model-storage" + } + } + require.NoError(t, reconciler.Update(ctx, dgd)) + require.NoError(t, reconciler.Get(ctx, key, deployment)) + revision, revisionErr := dynamo.LPXInputRevision(dgd, "") + require.NoError(t, revisionErr) + deployment.Spec.InputRevision = revision + require.NoError(t, reconciler.Update(ctx, deployment)) + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + pcs := &grovev1alpha1.PodCliqueSet{} + t.Log("Failed desired input must preserve the exact published request and PCS") + require.Equal(t, requestBefore, getTestPipelineRequest(t, ctx, reconciler.Client, deployment.Namespace, requestBefore.Name)) + require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) + require.Equal(t, pcsBefore, pcs) + + t.Log("Persist the actual actionable failure and retain its wrapped cause") + require.ErrorContains(t, err, message) + if test.snapshotError != nil { + require.ErrorIs(t, err, test.snapshotError) + require.ErrorIs(t, err, lpx.ErrBuildSnapshotAcquisition) + } + require.NoError(t, reconciler.Get(ctx, key, deployment)) + failed := meta.FindStatusCondition(deployment.Status.Conditions, "Ready") require.NotNil(t, failed) - if len(scenario.messages) == 0 { - require.Equal(t, metav1.ConditionFalse, failed.Status) - require.Equal(t, modelDownloadPendingReason, failed.Reason) - wantCalls := []string{modelDownloadTestBuildID} - if scenario.sharedDraft { - wantCalls = append(wantCalls, modelDownloadTestSecondBuildID) + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, deployment.Generation, failed.ObservedGeneration) + require.Equal(t, err.Error(), failed.Message) + }) + } +} + +func TestLPXPodCliqueSetListOrder(t *testing.T) { + t.Log("Create a PCS through the controller with ordered Agent init containers") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + agent := lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXAgent) + agent.PodTemplate.Spec.InitContainers = []corev1.Container{ + {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, + } + r := newLPXTestReconciler(t, registry, child, dgd) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + pcs, err := getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + require.NotNil(t, pcs) + originalUID := pcs.UID + + t.Log("Reverse authored init order and verify the controller opts into order-sensitive synchronization") + slices.Reverse(agent.PodTemplate.Spec.InitContainers) + updateTestDGD(t, r, child, dgd) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + pcs, err = getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + require.Equal(t, originalUID, pcs.UID) + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { + require.Equal(t, agent.PodTemplate.Spec.InitContainers, clique.Spec.PodSpec.InitContainers) + } + } + + t.Log("Unchanged desired input does not write the PCS again") + before := pcs.DeepCopy() + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + pcs, err = getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + require.Equal(t, before, pcs) +} + +func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { + for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { + t.Run(string(pipeline), func(t *testing.T) { + ctx := t.Context() + deployment, dgd, registry := newLPXTestDGD(t, pipeline) + if pipeline == lpx.PipelineLPX { + dgd.Spec.Components[0].Replicas = ptr.To(int32(1)) + dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) + } + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) + objects := lpxMaterializedObjects(t, reconciler, deployment, dgd, desired) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + require.Equal(t, desired.workload.LPXComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) } - require.Equal(t, wantCalls, observedRegistry.calls) + } + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + pcs := findLPXTestPodCliqueSet(t, objects) + rootReads := 0 + base, ok := reconciler.Client.(client.WithWatch) + require.True(t, ok) + reader := interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + rootReads++ + } + return reader.Get(ctx, key, object, opts...) + }}) + observe := func() dynamo.GroveReadiness { + group, err := getPodCliqueScalingGroup(ctx, reconciler.Client, pcs) + require.NoError(t, err) + ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, dgd, pcs, group) + require.NoError(t, err) + require.Len(t, ready.ComponentStatuses, 1) + require.Zero(t, rootReads, "reuse the synchronized PCS observation") + return ready + } + t.Log("Report one complete engine, never separate GPU or Agent component capacity") + ready := observe() + require.True(t, ready.Ready) + status := ready.ComponentStatuses[desired.workload.LPXComponentName()] + require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, status.ComponentKind) + require.Equal(t, []string{desired.plan.LPXScalingGroup}, status.ComponentNames) + require.Equal(t, int32(1), status.Replicas) + require.Equal(t, int32(1), status.UpdatedReplicas) + require.Equal(t, ptr.To(int32(1)), status.AvailableReplicas) + require.Equal(t, ptr.To(int32(1)), status.ScheduledReplicas) + if desired.plan.CyborgClique != "" { + t.Log("A stale GPU-role scale cannot satisfy complete-engine readiness") + gpu := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique).DeepCopy() + require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(gpu), gpu)) + gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 1, 1, 1 + gpu.Status.ReadyReplicas, gpu.Status.ScheduledReplicas = 1, 1 + require.NoError(t, reconciler.Update(ctx, gpu)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, v1beta1.DGDReadyReasonUpdating, ready.Classification) + t.Log("Partial GPU readiness contributes zero complete engine replicas") + gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 2, 2, 2 + gpu.Status.ScheduledReplicas = 2 + require.NoError(t, reconciler.Update(ctx, gpu)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, v1beta1.DGDReadyReasonPodsNotReady, ready.Classification) + require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.LPXComponentName()].AvailableReplicas) + gpu.Status.ReadyReplicas = 2 + require.NoError(t, reconciler.Update(ctx, gpu)) + require.True(t, observe().Ready) + } + t.Log("Reject stale complete-engine capacity even when every existing role is ready") + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.LPXScalingGroup}, group)) + group.Spec.Replicas = 2 + require.NoError(t, reconciler.Update(ctx, group)) + ready = observe() + require.False(t, ready.Ready) + require.Equal(t, v1beta1.DGDReadyReasonUpdating, ready.Classification) + }) + } +} + +func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { + for _, scenario := range []struct{ name, message string }{ + {"LPX", "LPX integration is disabled"}, + {"Grove", "Grove is disabled"}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Log("Publish a complete workload, discovery service and scheduling request") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) + key := client.ObjectKeyFromObject(child) + for range 4 { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + } + require.NoError(t, r.Get(ctx, key, child)) + before := []client.ObjectList{ + &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}, + &corev1.ServiceList{}, &lpxv1alpha1.LPUPipelineRequestList{}, + } + for _, list := range before { + require.NoError(t, r.List(ctx, list)) + require.Positive(t, meta.LenList(list)) + } + + t.Log("Disable the provider before an input edit and an expired deadline can retire the workload") + if scenario.name == "LPX" { + r.runtimeConfig.Gate.LPX = false } else { - require.Equal(t, metav1.ConditionTrue, failed.Status) - if scenario.name == "disabled Grove" { - require.Equal(t, "LPXUnavailable", failed.Reason) - } else if scenario.name == componentProvider { - require.Equal(t, "LPXReconciliationFailed", failed.Reason) - } else { - require.Equal(t, "LPXRejected", failed.Reason) + r.runtimeConfig.Gate.Grove = false + } + r.enabled = false + r.modelRegistry = nil + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) + originalSpec := dgd.Spec.DeepCopy() + dgd.Spec.Components[0].LPX.BuildID = "edited-while-disabled" + require.NoError(t, r.Update(ctx, dgd)) + for range 2 { + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Zero(t, result) + } + require.NoError(t, r.Get(ctx, key, child)) + failed := meta.FindStatusCondition(child.Status.Conditions, "Ready") + require.NotNil(t, failed) + require.Equal(t, metav1.ConditionFalse, failed.Status) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, failed.Reason) + require.Equal(t, scenario.message, failed.Message) + require.Equal(t, child.Generation, failed.ObservedGeneration) + require.Len(t, child.Status.Conditions, 1) + for _, list := range before { + current := list.DeepCopyObject().(client.ObjectList) + require.NoError(t, r.List(ctx, current)) + require.Equal(t, list, current) + } + + t.Log("Re-enable unchanged intent and resume the original PCS and request identities") + dgd.Spec = *originalSpec + require.NoError(t, r.Update(ctx, dgd)) + r.runtimeConfig.Gate.LPX, r.runtimeConfig.Gate.Grove = true, true + r.enabled = true + r.modelRegistry = registry + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key, child)) + require.NotEqual(t, v1alpha1.LPXReadyReasonFailed, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) + for _, list := range before { + current := list.DeepCopyObject().(client.ObjectList) + require.NoError(t, r.List(ctx, current)) + require.Equal(t, list, current) + } + + t.Log("Deletion remains available while disabled and needs no controller finalizer") + if scenario.name == "LPX" { + r.runtimeConfig.Gate.LPX = false + } else { + r.runtimeConfig.Gate.Grove = false + } + r.modelRegistry = nil + require.Empty(t, child.Finalizers) + require.NoError(t, r.Delete(ctx, child)) + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(r.Get(ctx, key, &v1alpha1.LPXGraphDeployment{}))) + }) + } +} + +// lpxTestWorkload is a fixture DSL, not reconciliation state. +type lpxTestWorkload struct { + workload *lpx.SelectedWorkload + plan *lpx.MaterializationPlan + requests []lpxv1alpha1.LPUPipelineRequest +} + +func resolveLPXTestWorkload(t *testing.T, registry lpx.ModelRegistry, ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, +) *lpxTestWorkload { + t.Helper() + workload, err := lpx.ResolveSelectedWorkload(ctx, dgd, registry) + require.NoError(t, err) + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) + require.NoError(t, err) + _, missing, changed := resolvePipelineRequests(deployment, nil, workload, plan) + require.False(t, changed) + fixture := &lpxTestWorkload{workload: workload, plan: plan} + for _, request := range missing { + fixture.requests = append(fixture.requests, *request) + } + return fixture +} + +func newPreparedLPXTestReconciler( + t *testing.T, + registry lpx.ModelRegistry, + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, +) (*graphReconciler, *lpxTestWorkload) { + t.Helper() + + // Construct the fake client before preparing the selected LPX plan. + reconciler := newLPXTestReconciler(t, registry, deployment, dgd) + desired := resolveLPXTestWorkload(t, reconciler.modelRegistry, ctx, deployment, dgd) + return reconciler, desired +} + +func findLPXTestPodCliqueSet(t *testing.T, objects []client.Object) *grovev1alpha1.PodCliqueSet { + t.Helper() + for _, object := range objects { + if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok { + return pcs + } + } + t.Fatal("PodCliqueSet not found") + return nil +} + +func newLPXTestDGD(t *testing.T, pipeline lpx.Pipeline) (*v1alpha1.LPXGraphDeployment, *v1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { + t.Helper() + compilationMode := manifestcapnpv2.CompilationMode_lpuOnly + if pipeline == lpx.PipelineLPX { + compilationMode = manifestcapnpv2.CompilationMode_lpx + } + const buildID = "build-v2" + registry := newLPXTestRegistryWithPartitionsAndMode(t, buildID, []int{7, 8}, compilationMode) + dgd := loadTestDGD(t, pipeline, buildID) + return newLPXTestDeployment(t, dgd), dgd, registry +} + +func newLPXTestDeployment(t *testing.T, dgd *v1beta1.DynamoGraphDeployment) *v1alpha1.LPXGraphDeployment { + t.Helper() + revision, err := dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + return &v1alpha1.LPXGraphDeployment{ + TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "LPXGraphDeployment"}, + ObjectMeta: metav1.ObjectMeta{Name: dgd.Name, Namespace: dgd.Namespace, UID: types.UID("lpx-" + string(dgd.UID)), Generation: 1, + Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(dgd.Generation, 10)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, v1beta1.DynamoGraphDeploymentGVK)}, + }, + Spec: v1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, + } +} + +func loadTestDGD(t testing.TB, pipeline lpx.Pipeline, buildID string) *v1beta1.DynamoGraphDeployment { + t.Helper() + payload, err := os.ReadFile("testdata/dgd.yaml") + require.NoError(t, err) + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.UnmarshalStrict(payload, dgd)) + dgd.Spec.Components[0].LPX.BuildID = buildID + + // Hybrid pipelines author a GPU conductor while retaining the same Agent template. + if pipeline == lpx.PipelineLPX { + payload, err = os.ReadFile("testdata/hybrid-conductor.yaml") + require.NoError(t, err) + conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) + *conductor = v1beta1.ComponentRoleSpec{} + require.NoError(t, yaml.UnmarshalStrict(payload, conductor)) + } + return dgd +} + +func newLPXSpecDecodeTestDGD(t *testing.T) (*v1alpha1.LPXGraphDeployment, *v1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { + t.Helper() + dgd := loadTestSpecDecodeDGD(t) + root := t.TempDir() + writeLPXTestBuild(t, root, "draft-build", []int{7, 8}, manifestcapnpv2.CompilationMode_lpuOnly) + writeLPXTestBuild(t, root, "target-build", []int{9, 10}, manifestcapnpv2.CompilationMode_lpuOnly) + registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() + registry, err := lpx.NewModelRegistry(registryURL, nil) + require.NoError(t, err) + return newLPXTestDeployment(t, dgd), dgd, registry +} + +func loadTestSpecDecodeDGD(t testing.TB) *v1beta1.DynamoGraphDeployment { + dgd := loadTestDGD(t, lpx.PipelineSingle, "target-build") + target := &dgd.Spec.Components[0] + draft := target.DeepCopy() + draft.ComponentName, draft.LPX.BuildID, draft.Replicas = "draft", "draft-build", ptr.To(int32(2)) + draft.Roles = []v1beta1.ComponentRoleSpec{*draft.ComponentRole(v1beta1.ComponentRoleLPXAgent)} + dgd.Spec.Components = append(dgd.Spec.Components, *draft) + return dgd +} + +// newLPXTestClient shares schemes, indexes and the fake client's missing API-server behavior. +func newLPXTestClient(t testing.TB, objects ...client.Object) client.WithWatch { + t.Helper() + scheme := newLPXTestScheme(t) + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithIndex(&lpxv1alpha1.LPUPipelineRequest{}, pipelineRequestPCSOwnerUIDIndex, pipelineRequestOwnerUID). + WithIndex(&v1alpha1.LPXGraphDeployment{}, dgdControllerOwnerIndex, dgdControllerOwnerKey). + WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}, &v1alpha1.LPXGraphDeployment{}, &corev1.Pod{}). + WithObjects(objects...). + Build() + nextLPRUID := 0 + return interceptor.NewClient(base, interceptor.Funcs{ + // The fake client checks resource versions but does not enforce Delete UID preconditions. + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + options := (&client.DeleteOptions{}).ApplyOptions(opts) + if options.Preconditions != nil && options.Preconditions.UID != nil { + observed := object.DeepCopyObject().(client.Object) + if err := delegated.Get(ctx, client.ObjectKeyFromObject(object), observed); err != nil { + return err } - for _, message := range scenario.messages { - require.Contains(t, failed.Message, message) + if observed.GetUID() != *options.Preconditions.UID { + return apierrors.NewConflict(schema.GroupResource{Resource: object.GetObjectKind().GroupVersionKind().Kind}, object.GetName(), errors.New("UID precondition failed")) } - require.Empty(t, observedRegistry.calls) } - require.Zero(t, observedRegistry.acquireBuildSnapshotCalls) - pcs := &grovev1alpha1.PodCliqueSetList{} - requests := &lpxv1alpha1.LPUPipelineRequestList{} - require.NoError(t, r.List(t.Context(), pcs)) - require.NoError(t, r.List(t.Context(), requests)) - require.Empty(t, pcs.Items) - require.Empty(t, requests.Items) - require.Equal(t, before, source) + return delegated.Delete(ctx, object, opts...) + }, + // The fake client does not implement scale for Grove custom resources. + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && subresource == "scale" { + options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) + scale := options.SubResourceBody.(*autoscalingv1.Scale) + group.ResourceVersion = scale.ResourceVersion + group.Spec.Replicas = scale.Spec.Replicas + return delegated.Update(ctx, group) + } + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok && pcs.UID == "" { + pcs.UID = types.UID(pcs.Name + "-uid") + pcs.Generation = 1 + } + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + if request.UID == "" { + nextLPRUID++ + request.UID = types.UID(fmt.Sprintf("lpr-%s-%d", request.Name, nextLPRUID)) + } + if request.Generation == 0 { + request.Generation = 1 + } + if request.CreationTimestamp.IsZero() { + request.CreationTimestamp = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) + } + } + return delegated.Create(ctx, object, opts...) + }, + }) +} + +func newLPXTestReconciler( + t *testing.T, + registry lpx.ModelRegistry, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + objects ...client.Object, +) *graphReconciler { + t.Helper() + revision, err := dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + deployment.Spec.InputRevision = revision + if deployment.ResourceVersion == "" { + deployment.ResourceVersion = "1" + } + seed := append([]client.Object{deployment.DeepCopy(), dgd.DeepCopy()}, objects...) + recorder := events.NewFakeRecorder(100) + config := &configv1alpha1.OperatorConfiguration{ + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + } + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, DRA: true, LPX: true}} + return &graphReconciler{ + Client: newLPXTestClient(t, seed...), + recorder: recorder, + enabled: true, + runtimeConfig: runtimeConfig, + modelRegistry: registry, + config: config, + } +} + +func lpxMaterializedObjects( + t *testing.T, + reconciler *graphReconciler, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + desired *lpxTestWorkload, +) []client.Object { + t.Helper() + t.Log("Render LPX intent once and emulate the API-server and Grove observations") + pcs := renderLPXTestPodCliqueSet(t, t.Context(), reconciler, deployment, dgd, desired) + pcs.TypeMeta = metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet"} + pcs.UID, pcs.Generation = "pcs-uid", 1 + pcs.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)} + const generationHash = "pcs-generation-hash" + pcs.Status = grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(pcs.Generation), CurrentGenerationHash: ptr.To(generationHash), + } + + // Emulate Grove materialization after the controller applies the desired live scale. + groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].DeepCopy() + group := &grovev1alpha1.PodCliqueScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: desired.plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: "lpu-group-uid", + Generation: 1, Labels: grovecommon.GetDefaultLabelsForPodCliqueSetManagedResources(pcs.Name), Annotations: groupTemplate.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: desired.plan.Replicas, MinAvailable: groupTemplate.MinAvailable, + CliqueNames: groupTemplate.CliqueNames, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + }, + } + group.Labels[grovecommon.LabelPartOfKey] = pcs.Name + group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" + objects := []client.Object{pcs, group} + + // Each Grove replica owns independent cliques. + for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { + parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} + for _, template := range pcs.Spec.Template.Cliques { + rendered := template.DeepCopy() + name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) + clique := &grovev1alpha1.PodClique{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, + Labels: rendered.Labels, Annotations: rendered.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }, + Spec: rendered.Spec, + Status: grovev1alpha1.PodCliqueStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + CurrentPodTemplateHash: ptr.To(rendered.Name + "-pod-template-hash"), + }, + } + clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) + for index, dependency := range clique.Spec.StartsAfter { + clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) + } + objects = append(objects, clique) + } + } + return objects +} + +func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.Client, objects ...client.Object) { + t.Helper() + for _, object := range objects { + require.NoError(t, kubeClient.Create(ctx, object.DeepCopyObject().(client.Object))) + } +} + +func observedLPXTestPodCliqueSet( + t *testing.T, + ctx context.Context, + reconciler *graphReconciler, + deployment *v1alpha1.LPXGraphDeployment, + desired *lpxTestWorkload, +) *grovev1alpha1.PodCliqueSet { + t.Helper() + if desired.plan == nil { + return nil + } + pcs := &grovev1alpha1.PodCliqueSet{} + err := reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs) + if apierrors.IsNotFound(err) { + return nil + } + require.NoError(t, err) + return pcs +} + +func TestLPXQueueEditSynchronizesPCSMetadata(t *testing.T) { + t.Log("Keep PCS queue metadata and clique templates consistent across DGD queue edits") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Labels = map[string]string{consts.KubeLabelKaiSchedulerQueue: "queue-a"} + for _, component := range lpx.Components(dgd) { + for _, role := range component.Roles { + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "queue-a") + } + } + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, selected) + key := client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), req) + require.NoError(t, err) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(t.Context(), key, pcs)) + metav1.SetMetaDataLabel(&pcs.ObjectMeta, "external.example/label", "preserved") + metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, "external.example/annotation", "preserved") + require.NoError(t, r.Update(t.Context(), pcs)) + beforeUID := pcs.UID + + t.Log("Apply new queue metadata and templates in place while preserving untracked metadata") + dgd.Spec.Labels[consts.KubeLabelKaiSchedulerQueue] = "queue-b" + for _, component := range lpx.Components(dgd) { + for _, role := range component.Roles { + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "queue-b") + } + } + updateTestDGD(t, r, child, dgd) + _, err = r.Reconcile(t.Context(), req) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key, pcs)) + require.Equal(t, beforeUID, pcs.UID) + require.Equal(t, "preserved", pcs.Labels["external.example/label"]) + require.Equal(t, "preserved", pcs.Annotations["external.example/annotation"]) + require.Equal(t, "queue-b", pcs.Labels[consts.KubeLabelKaiSchedulerQueue]) + for _, clique := range pcs.Spec.Template.Cliques { + require.Equal(t, "queue-b", clique.Labels[consts.KubeLabelKaiSchedulerQueue]) + } + + t.Log("Remove an omitted managed queue from both PCS metadata and clique templates") + delete(dgd.Spec.Labels, consts.KubeLabelKaiSchedulerQueue) + for _, component := range lpx.Components(dgd) { + for _, role := range component.Roles { + delete(role.PodTemplate.Labels, consts.KubeLabelKaiSchedulerQueue) + } + } + updateTestDGD(t, r, child, dgd) + _, err = r.Reconcile(t.Context(), req) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key, pcs)) + require.NotContains(t, pcs.Labels, consts.KubeLabelKaiSchedulerQueue) + for _, clique := range pcs.Spec.Template.Cliques { + require.NotContains(t, clique.Labels, consts.KubeLabelKaiSchedulerQueue) + } + + t.Log("An unchanged reconciliation does not write the PCS again") + beforeVersion := pcs.ResourceVersion + _, err = r.Reconcile(t.Context(), req) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key, pcs)) + require.Equal(t, beforeVersion, pcs.ResourceVersion) + require.Equal(t, "preserved", pcs.Labels["external.example/label"]) + require.Equal(t, "preserved", pcs.Annotations["external.example/annotation"]) +} + +func TestLPXEditBetweenFailureAndCleanupPreservesRetry(t *testing.T) { + t.Log("Persist a deadline failure before the user edits the input to retry") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + pending := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, selected.requests[0].Name) + pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(t.Context(), pending)) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(t.Context(), req) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), req.NamespacedName, child)) + oldFailure := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition).DeepCopy() + require.True(t, isSchedulingFailedConditionCurrent(child)) + + t.Log("The edited generation uses the old failure to retire the already-expired cycle") + dgd.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To[int64](31) + updateTestDGD(t, r, child, dgd) + _, err = r.Reconcile(t.Context(), req) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), req.NamespacedName, child)) + newFailure := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition) + require.Equal(t, oldFailure, newFailure) + require.Greater(t, child.Generation, newFailure.ObservedGeneration) + requirePipelineRequestNotFound(t, t.Context(), r.Client, pending.Namespace, pending.Name) + t.Log("The same single edit permits republication after cleanup") + for range 3 { + _, err = r.Reconcile(t.Context(), req) + require.NoError(t, err) + } + replacement := getTestPipelineRequest(t, t.Context(), r.Client, pending.Namespace, pending.Name) + require.NotEqual(t, pending.UID, replacement.UID) +} + +func TestLPXCommittedDeadlineCleanupPreservesExternalCapacity(t *testing.T) { + t.Log("Publish an externally scaled request with a committed Pod and scheduler finalizer") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Components[0].Replicas = nil + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + request := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + request.Finalizers = []string{"test.example/scheduler-release"} + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBinding).Status + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: child.Namespace, Name: "committed-agent", UID: "agent-uid"}} + request.Status.Committed = committedLPXTestPod(pod) + require.NoError(t, r.Update(ctx, request)) + require.NoError(t, r.Create(ctx, pod)) + reconcileRequest := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + + t.Log("Persist failure without deleting the request or its Pod") + _, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.True(t, request.DeletionTimestamp.IsZero()) + + t.Log("Request deletion never authorizes direct Pod or PodClique deletion") + failRequestDelete := true + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + switch object.(type) { + case *corev1.Pod, *grovev1alpha1.PodClique: + t.Fatalf("graph reconciliation must not delete %T", object) + } + if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok && failRequestDelete { + return errors.New("request deletion temporarily unavailable") + } + return delegated.Delete(ctx, object, opts...) + }, + }) + _, err = r.Reconcile(ctx, reconcileRequest) + require.ErrorContains(t, err, "request deletion temporarily unavailable") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.True(t, request.DeletionTimestamp.IsZero()) + + t.Log("Start request deletion and leave Pod cleanup to Grove and garbage collection") + failRequestDelete = false + _, err = r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.False(t, request.DeletionTimestamp.IsZero()) + require.NotEmpty(t, request.Finalizers) + + t.Log("Registry failure does not authorize Pod cleanup for a terminating request") + r.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: errors.New("registry unavailable")} + _, err = r.Reconcile(ctx, reconcileRequest) + require.ErrorContains(t, err, "registry unavailable") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Equal(t, selected.plan.Replicas, group.Spec.Replicas) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.NotEmpty(t, request.Finalizers) + + t.Log("Wait for scheduler finalization without escalating cleanup or republishing") + r.modelRegistry = registry + _, err = r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.False(t, request.DeletionTimestamp.IsZero()) + require.NotEmpty(t, request.Finalizers) +} + +func TestLPXCommittedReleasePreservesPods(t *testing.T) { + for _, release := range []bool{false, true} { + t.Run(fmt.Sprintf("release=%t", release), func(t *testing.T) { + t.Log("Observe a committed Degraded request with either repair or durable release authority") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, selected) + request := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) + request.Status = newTestPipelineRequest(child, findLPXTestPodCliqueSet(t, objects), request.Name, time.Now(), lpxv1alpha1.RequestPhaseDegraded).Status + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: child.Namespace, Name: "committed-agent", UID: "agent-uid"}} + request.Status.Committed = committedLPXTestPod(pod) + if release { + request.Status.Committed.Execution.Release = &lpxv1alpha1.ReleaseJournal{Reason: lpxv1alpha1.ReleaseReasonDependencyChanged, RequestedAtGeneration: request.Generation} + } + require.NoError(t, r.Update(ctx, request)) + require.NoError(t, r.Create(ctx, pod)) + + t.Log("Neither repair nor release authorizes graph reconciliation to delete Pods") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) + request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) + require.True(t, request.DeletionTimestamp.IsZero()) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) + require.Equal(t, selected.plan.Replicas, group.Spec.Replicas) }) } } -func TestLPXPublicationFencesLiveSourceAndChildMetadata(t *testing.T) { - t.Log("Prepare an immutable render from the current source and child") - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, source) - require.NoError(t, r.validateLPXPublicationSource(t.Context(), child)) - - t.Log("A source edit racing publication prevents the old render from creating a PCS") - liveSource := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(source), liveSource)) - liveSource.Annotations[consts.KubeAnnotationEnableMetrics] = "false" - require.NoError(t, r.Update(t.Context(), liveSource)) - _, _, err := r.reconcileWorkload(t.Context(), child, source, selected, nil) - require.ErrorContains(t, err, "input revision") - allPCS := &grovev1alpha1.PodCliqueSetList{} - require.NoError(t, r.List(t.Context(), allPCS)) - require.Empty(t, allPCS.Items) - - t.Log("Child publication metadata cannot change behind an unchanged spec generation") - liveChild := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), liveChild)) - liveChild.Annotations[lpx.DGDGenerationAnnotation] = "999" - require.NoError(t, r.Update(t.Context(), liveChild)) - require.ErrorContains(t, r.validateLPXPublicationSource(t.Context(), child), "authority changed") +func committedLPXTestPod(pod *corev1.Pod) *lpxv1alpha1.Committed { + return &lpxv1alpha1.Committed{ + Plan: lpxv1alpha1.CommittedPlan{Placement: lpxv1alpha1.PlanPlacement{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}}, + Execution: lpxv1alpha1.CommittedExecution{AcceptedGeneration: ptr.To[int64](1), NodeLocal: &lpxv1alpha1.NodeLocalExecution{ + PartitionSelections: []lpxv1alpha1.NodeLocalPartitionExecution{{SelectedRows: []lpxv1alpha1.NodeLocalRowExecution{{ + Current: &lpxv1alpha1.CurrentBinding{PodRef: lpxv1alpha1.ObjectReference{Namespace: pod.Namespace, Name: pod.Name, UID: string(pod.UID)}}, + }}}}, + }}, + } } diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement.go b/deploy/operator/internal/controller/lpx/deadline_retirement.go deleted file mode 100644 index bce09d323a7f..000000000000 --- a/deploy/operator/internal/controller/lpx/deadline_retirement.go +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - "fmt" - "math" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - autoscalingv1 "k8s.io/api/autoscaling/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// retireExpiredLPXRequests lowers the one live Grove group before deleting the -// affected engine suffix. pcs is nil after its disappearance, which leaves its -// requests to owner garbage collection. -func (r *graphReconciler) retireExpiredLPXRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - requests []lpxv1alpha1.LPUPipelineRequest, - expired []*lpxv1alpha1.LPUPipelineRequest, -) error { - if len(expired) == 0 { - return nil - } - owner := metav1.GetControllerOf(expired[0]) - if owner == nil || owner.Name == "" || owner.UID == "" { - return fmt.Errorf("LPX request %q lacks an exact PodCliqueSet owner", expired[0].Name) - } - - // A missing, replaced, or deleting PCS leaves its requests to owner GC. - if pcs == nil || pcs.Name != owner.Name || pcs.UID != owner.UID || !pcs.DeletionTimestamp.IsZero() { - return nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return fmt.Errorf("refusing to scale PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) - } - for _, request := range expired { - if !metav1.IsControlledBy(request, pcs) { - return fmt.Errorf("expired LPX request %q does not have the current PodCliqueSet owner", request.Name) - } - } - - groupName, _, err := lpxScalingGroupTarget(expired[0]) - if err != nil { - return err - } - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: groupName}, group); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return err - } - if !metav1.IsControlledBy(group, pcs) { - return fmt.Errorf("LPX scaling group lacks the current PCS owner") - } - if !group.DeletionTimestamp.IsZero() { - return nil - } - replicas, retiring, err := lpxExpiredEngineSuffix(requests, expired, group.Spec.Replicas) - if err != nil { - return err - } - if len(retiring) == 0 { - return nil - } - - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return err - } - // Omitted replicas leave capacity externally managed; expire only scheduler - // intent so a later input revision can reuse the unchanged Grove ordinal. - component := lpx.ServingComponent(source) - if component == nil { - return fmt.Errorf("cannot determine replica ownership without an LPX serving component") - } - if component.Replicas != nil && group.Spec.Replicas > replicas { - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: group.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, - } - if err := r.SubResource("scale").Update(ctx, group, client.WithSubResourceBody(scale)); err != nil { - return err - } - } - - // Grove and the scheduler converge asynchronously after the ordered writes. - return r.deleteStaleLPXRequests(ctx, deployment, retiring) -} - -// lpxExpiredEngineSuffix returns complete engine replicas only when every -// expired replica belongs to one contiguous trailing suffix. It orders the -// expired requests last so transient sibling cleanup leaves durable expiry -// evidence for the next reconciliation. An interior failure blocks all -// scale-down so a healthy higher ordinal is never removed. -func lpxExpiredEngineSuffix( - requests []lpxv1alpha1.LPUPipelineRequest, - expired []*lpxv1alpha1.LPUPipelineRequest, - replicas int32, -) (int32, []*lpxv1alpha1.LPUPipelineRequest, error) { - if len(expired) == 0 { - return replicas, nil, nil - } - groupName, _, err := lpxScalingGroupTarget(expired[0]) - if err != nil { - return 0, nil, err - } - failedReplicas := make(map[int64]struct{}, len(expired)) - for _, request := range expired { - name, replica, err := lpxScalingGroupTarget(request) - if err != nil { - return 0, nil, err - } - if name != groupName { - return 0, nil, fmt.Errorf("expired LPX requests target different scaling groups") - } - failedReplicas[replica] = struct{}{} - } - - targetReplicas := int64(replicas) - for targetReplicas > 0 { - if _, failed := failedReplicas[targetReplicas-1]; !failed { - break - } - targetReplicas-- - } - for replica := range failedReplicas { - if replica < targetReplicas { - return replicas, nil, nil - } - } - - expiredNames := make(map[string]struct{}, len(expired)) - for _, request := range expired { - expiredNames[request.Name] = struct{}{} - } - retiring := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) - failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) - for index := range requests { - request := &requests[index] - name, replica, err := lpxScalingGroupTarget(request) - if err != nil { - return 0, nil, err - } - if name != groupName { - return 0, nil, fmt.Errorf("LPX requests target different scaling groups") - } - if replica < targetReplicas { - continue - } - if _, expired := expiredNames[request.Name]; expired { - failed = append(failed, request) - } else { - retiring = append(retiring, request) - } - } - return int32(targetReplicas), append(retiring, failed...), nil -} - -func lpxScalingGroupTarget(request *lpxv1alpha1.LPUPipelineRequest) (string, int64, error) { - target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef - if target == nil || target.Name == "" || target.ReplicaIndex < 0 || target.ReplicaIndex >= math.MaxInt32 { - return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) - } - return target.Name, target.ReplicaIndex, nil -} diff --git a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go b/deploy/operator/internal/controller/lpx/deadline_retirement_test.go deleted file mode 100644 index 919ebafb0bdb..000000000000 --- a/deploy/operator/internal/controller/lpx/deadline_retirement_test.go +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/require" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" -) - -func TestLPXExpiredScaleOutPreservesServingEngine(t *testing.T) { - t.Log("Publish one bound engine and one expired scale-out request") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - serving := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - serving.Status = deadlineTestRequest(child, pcs, serving.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status - require.NoError(t, r.Update(ctx, serving)) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - - t.Log("Reject ambiguous replica ownership without changing the live engine") - invalid := source.DeepCopy() - invalid.Spec.Components = nil - require.ErrorContains(t, r.retireExpiredLPXRequests(ctx, child, invalid, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}), "without an LPX serving component") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - - t.Log("Lower Grove and delete only the expired scheduler intent in one reconciliation") - err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) - require.True(t, apiequality.Semantic.DeepEqual(serving, getLPXRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") -} - -func TestLPXExpiredModelRetiresItsEngineSiblings(t *testing.T) { - t.Log("Publish the multiple model requests belonging to one speculative engine") - ctx := t.Context() - child, source, registry := newLPXSpecDecodeTestDGD(t) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - require.Greater(t, len(desired.requests), 1) - requests, err := r.listOwnedLPXRequests(ctx, child, findLPXTestPodCliqueSet(t, objects)) - require.NoError(t, err) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[len(desired.requests)-1].requestName) - - t.Log("Map one expired model request to every request sharing its engine suffix") - replicas, retiring, err := lpxExpiredEngineSuffix(requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, 1) - require.NoError(t, err) - require.Zero(t, replicas) - require.Len(t, retiring, len(requests)) -} - -func TestLPXExpiredModelPreservesExpiryProofUntilSiblingCleanupStarts(t *testing.T) { - t.Log("Publish multiple model requests for one engine and expire one of them") - ctx := t.Context() - child, source, registry := newLPXSpecDecodeTestDGD(t) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - require.Greater(t, len(requests), 1) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - var sibling *lpxv1alpha1.LPUPipelineRequest - for index := range requests { - if requests[index].Name != expired.Name { - sibling = &requests[index] - break - } - } - require.NotNil(t, sibling) - - t.Log("Fail the first sibling deletion and retain the expired request for a retry") - deleteErr := errors.New("transient sibling deletion failure") - base := r.Client - r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok && request.Name == sibling.Name { - return deleteErr - } - return delegated.Delete(ctx, object, opts...) - }, - }) - err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) - require.ErrorIs(t, err, deleteErr) - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - - t.Log("Retry from the retained expiry proof and remove the complete engine request set") - r.Client = base - requests, err = r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - expired = getLPXRequest(t, ctx, r.Client, child.Namespace, expired.Name) - require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) - for index := range desired.requests { - requireLPXRequestNotFound(t, ctx, r.Client, child.Namespace, desired.requests[index].requestName) - } -} - -func TestLPXExpiredInteriorReplicaDoesNotScaleDown(t *testing.T) { - t.Log("Publish three engine replicas and expire only the middle ordinal") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(3)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[1].requestName) - child.Status.ExpiredRequestUIDs = []types.UID{expired.UID} - require.NoError(t, r.Status().Update(ctx, child)) - - t.Log("Wait for watched changes without polling or changing the live engines") - result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.Zero(t, result) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - require.Equal(t, []types.UID{expired.UID}, child.Status.ExpiredRequestUIDs) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(3), group.Spec.Replicas) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - for _, object := range objects { - if clique, ok := object.(*grovev1alpha1.PodClique); ok { - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(clique), &grovev1alpha1.PodClique{})) - } - } -} - -func TestLPXExpiredInteriorReplicaBlocksTrailingSuffix(t *testing.T) { - t.Log("Publish three engines with both an interior failure and a failed tail") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(3)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - pcs := findLPXTestPodCliqueSet(t, objects) - interior := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - tail := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[2].requestName) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - - t.Log("Do not partially mutate the group while an expired hole remains") - require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{interior, tail})) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(3), group.Spec.Replicas) - for _, request := range []*lpxv1alpha1.LPUPipelineRequest{interior, tail} { - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{})) - } -} - -func TestLPXExpiredMaximalTrailingRunScalesDown(t *testing.T) { - t.Log("Publish four engines and expire the complete trailing pair") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(4)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - pcs := findLPXTestPodCliqueSet(t, objects) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - expired := []*lpxv1alpha1.LPUPipelineRequest{ - getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[2].requestName), - getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[3].requestName), - } - - require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, expired)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) - for index, projection := range desired.requests { - err := r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: projection.requestName}, &lpxv1alpha1.LPUPipelineRequest{}) - if index < 2 { - require.NoError(t, err) - } else { - require.True(t, apierrors.IsNotFound(err)) - } - } -} - -func TestLPXDeadlineRetirementPreservesImplicitReplicaCapacity(t *testing.T) { - t.Log("Publish one externally managed engine and expire its scheduler intent") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = nil - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - - t.Log("Delete the expired LPR without making observed zero capacity the new desired state") - require.NoError(t, r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired})) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) - - t.Log("Continue deriving the implicit desired count from the preserved Grove capacity") - selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) - require.NoError(t, err) - require.Nil(t, classification) - require.Equal(t, int32(1), selected.plan.Replicas) -} - -func TestLPXExpiredRequestWaitsForMissingScalingGroup(t *testing.T) { - t.Log("Keep an expired request while its live PCS has not materialized a scaling group") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - require.NoError(t, r.Delete(ctx, group)) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - - t.Log("Once Grove creates the group, issue scale-down before deleting the LPR") - group.ResourceVersion = "" - group.UID = "replacement-group" - group.DeletionTimestamp = nil - group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - require.NoError(t, r.Create(ctx, group)) - err = r.retireExpiredLPXRequests(ctx, child, source, pcs, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Zero(t, group.Spec.Replicas) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) -} - -func TestLPXExpiredRequestsWaitForOwnerGarbageCollection(t *testing.T) { - t.Log("Keep an expired request observable after its owner PCS disappears") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - expired := getLPXRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].requestName) - requests, err := r.listOwnedLPXRequests(ctx, child, pcs) - require.NoError(t, err) - require.NoError(t, r.Delete(ctx, pcs)) - - err = r.retireExpiredLPXRequests(ctx, child, source, nil, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) -} diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index f7e0816f2dd4..b212a992a7aa 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -10,7 +10,7 @@ import ( "fmt" "slices" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" @@ -27,8 +27,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" ) const ( @@ -85,7 +85,7 @@ func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context // A late runtime table observation must wake disrupted Agents using that exact table. var pods corev1.PodList if err := r.List(ctx, &pods, client.InNamespace(config.GetNamespace()), client.MatchingLabels{ - commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, + consts.KubeLabelDynamoComponentType: consts.ComponentTypeLPX, }); err != nil { log.FromContext(ctx).Error(err, "list LPU eviction triggers for runtime ConfigMap") return nil @@ -96,8 +96,8 @@ func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context for i := range pods.Items { pod := &pods.Items[i] root := pod.Labels[grovecommon.LabelPartOfKey] - hash := pod.Annotations[commonconsts.AnnotationExtraResourcesHash] - if r.isTrigger(pod) && root != "" && hash != "" && dynamolpx.LPUConfigMapName(root, hash) == config.GetName() { + hash := pod.Annotations[consts.AnnotationExtraResourcesHash] + if r.isTrigger(pod) && root != "" && hash != "" && lpx.LPUConfigMapName(root, hash) == config.GetName() { requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) } } @@ -114,11 +114,11 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: owner.Name}, clique); err != nil { return nil, err } - if clique.UID != owner.UID || !dynamolpx.OwnsPodClique(ctx, r.Client, clique) { + if clique.UID != owner.UID || !lpx.OwnsPodClique(ctx, r.Client, clique) { return nil, fmt.Errorf("cannot verify ownership of LPU eviction trigger pod %s/%s", trigger.Namespace, trigger.Name) } - mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[dynamolpx.WorkloadModeAnnotation]) + mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[lpx.WorkloadModeAnnotation]) if mode == "" { return nil, nil } @@ -131,12 +131,12 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: // The Pod's content-addressed runtime table survives native template updates. root := trigger.Labels[grovecommon.LabelPartOfKey] - hash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] + hash := trigger.Annotations[consts.AnnotationExtraResourcesHash] if root == "" || hash == "" { return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap identity", trigger.Namespace, trigger.Name) } var config corev1.ConfigMap - key := client.ObjectKey{Namespace: trigger.Namespace, Name: dynamolpx.LPUConfigMapName(root, hash)} + key := client.ObjectKey{Namespace: trigger.Namespace, Name: lpx.LPUConfigMapName(root, hash)} if err := r.Get(ctx, key, &config); err != nil { return nil, fmt.Errorf("read LPU-GPU eviction runtime ConfigMap: %w", err) } @@ -146,14 +146,14 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor // Bind the table to the verified deployment and the Pod's full content hash. configOwner := metav1.GetControllerOf(&config) - if configOwner == nil || configOwner.APIVersion != nvidiacomv1alpha1.GroupVersion.String() || - configOwner.Kind != nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind || - configOwner.Name != clique.Annotations[dynamolpx.DeploymentNameAnnotation] || - string(configOwner.UID) != clique.Annotations[dynamolpx.DeploymentUIDAnnotation] || - dynamolpx.LPUConfigMapHash(&config) != hash { + if configOwner == nil || configOwner.APIVersion != v1alpha1.GroupVersion.String() || + configOwner.Kind != v1alpha1.LPXGraphDeploymentGVK.Kind || + configOwner.Name != clique.Annotations[lpx.DeploymentNameAnnotation] || + string(configOwner.UID) != clique.Annotations[lpx.DeploymentUIDAnnotation] || + lpx.LPUConfigMapHash(&config) != hash { return nil, fmt.Errorf("cannot authenticate LPU-GPU eviction runtime ConfigMap %s", key) } - triggerPartition, err := dynamolpx.LPUAgentRuntimePartition(trigger, &config) + triggerPartition, err := lpx.LPUAgentRuntimePartition(trigger, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction trigger pod %s/%s: %w", trigger.Namespace, trigger.Name, err) } @@ -161,7 +161,7 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor // Select the full runtime partition before authorizing any sibling deletion. selected := pods[:0] for _, pod := range pods { - partition, err := dynamolpx.LPUAgentRuntimePartition(&pod, &config) + partition, err := lpx.LPUAgentRuntimePartition(&pod, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction candidate pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -178,8 +178,8 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { labels := trigger.GetLabels() - dgdName := labels[commonconsts.KubeLabelDynamoGraphDeploymentName] - component := labels[commonconsts.KubeLabelDynamoComponent] + dgdName := labels[consts.KubeLabelDynamoGraphDeploymentName] + component := labels[consts.KubeLabelDynamoComponent] pcsg := labels[grovecommon.LabelPodCliqueScalingGroup] pcsgReplica := labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] if dgdName == "" || component == "" || pcsg == "" || pcsgReplica == "" { @@ -188,8 +188,8 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev var pods corev1.PodList if err := r.List(ctx, &pods, client.InNamespace(trigger.Namespace), client.MatchingLabels{ - commonconsts.KubeLabelDynamoGraphDeploymentName: dgdName, - commonconsts.KubeLabelDynamoComponent: component, + consts.KubeLabelDynamoGraphDeploymentName: dgdName, + consts.KubeLabelDynamoComponent: component, grovecommon.LabelPodCliqueScalingGroup: pcsg, grovecommon.LabelPodCliqueScalingGroupReplicaIndex: pcsgReplica, }); err != nil { @@ -207,7 +207,7 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev owner.APIVersion != candidateOwner.APIVersion || owner.Kind != candidateOwner.Kind || owner.Name != candidateOwner.Name || owner.UID != candidateOwner.UID || pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || - pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] || + pod.Annotations[consts.AnnotationExtraResourcesHash] != trigger.Annotations[consts.AnnotationExtraResourcesHash] || model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model }), nil } @@ -243,13 +243,13 @@ func (r *lpuEvictionReconciler) deletePods(ctx context.Context, trigger *corev1. } func (r *lpuEvictionReconciler) isLPUAgentPod(pod *corev1.Pod) bool { - componentType := pod.Labels[commonconsts.KubeLabelDynamoComponentType] - return componentType == commonconsts.ComponentTypeLPX && + componentType := pod.Labels[consts.KubeLabelDynamoComponentType] + return componentType == consts.ComponentTypeLPX && pod.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent } func (r *lpuEvictionReconciler) isTrigger(pod *corev1.Pod) bool { - if pod.Spec.SchedulerName != dynamolpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { + if pod.Spec.SchedulerName != lpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { return false } return slices.ContainsFunc(pod.Status.Conditions, func(condition corev1.PodCondition) bool { diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 52dc533f331d..7a461a853316 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -10,7 +10,7 @@ import ( "sort" "testing" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -18,18 +18,16 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" ) @@ -44,18 +42,15 @@ const ( lpuEvictionTestConfigHash = "bf665efcf39a0c79fa97ea2c05664ca7235dd525aced22f47da0633d729d6171" ) -func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = nvidiacomv1alpha1.AddToScheme(scheme) - _ = grovev1alpha1.AddToScheme(scheme) +func newLPUEvictionReconciler(t *testing.T, objs ...client.Object) (*lpuEvictionReconciler, client.Client) { + t.Helper() - deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestLGD, Namespace: lpuEvictionTestNamespace, UID: "test-lgd-uid", }} pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestPCS, Namespace: lpuEvictionTestNamespace, UID: "test-pcs-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)}, }} group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestPCSG, Namespace: lpuEvictionTestNamespace, UID: "test-pcsg-uid", @@ -64,18 +59,18 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ Name: "agents", Namespace: lpuEvictionTestNamespace, UID: "agents-uid", Annotations: map[string]string{ - dynamolpx.DeploymentNameAnnotation: lpuEvictionTestLGD, - dynamolpx.DeploymentUIDAnnotation: "test-lgd-uid", - dynamolpx.WorkloadDigestAnnotation: "test-digest", - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + lpx.DeploymentNameAnnotation: lpuEvictionTestLGD, + lpx.DeploymentUIDAnnotation: "test-lgd-uid", + lpx.WorkloadDigestAnnotation: "test-digest", + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, }} config := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, + Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)}, }, Immutable: ptr.To(true), Data: map[string]string{ @@ -83,14 +78,7 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl "partition_models": lpuEvictionTestModel + "\n" + lpuEvictionTestModel, }, } - cb := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&corev1.Pod{}). - WithObjects(deployment, pcs, group, clique, config) - for _, o := range objs { - cb = cb.WithObjects(o) - } - c := cb.Build() + c := newLPXTestClient(t, append([]client.Object{deployment, pcs, group, clique, config}, objs...)...) return &lpuEvictionReconciler{Client: c, Recorder: events.NewFakeRecorder(16)}, c } @@ -104,19 +92,19 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting OwnerReferences: []metav1.OwnerReference{{APIVersion: "grove.io/v1alpha1", Kind: "PodClique", Name: "agents", UID: "agents-uid", Controller: ptr.To(true)}}, Labels: map[string]string{ grovecommon.LabelPartOfKey: lpuEvictionTestPCS, - commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, - commonconsts.KubeLabelDynamoComponent: lpuEvictionTestComponent, - commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, + consts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, + consts.KubeLabelDynamoComponent: lpuEvictionTestComponent, + consts.KubeLabelDynamoComponentType: consts.ComponentTypeLPX, grovecommon.LabelPodCliqueScalingGroup: lpuEvictionTestPCSG, grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, }, Annotations: map[string]string{ - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), - commonconsts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + lpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + consts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, }, }, - Spec: corev1.PodSpec{SchedulerName: dynamolpx.SchedulerName}, + Spec: corev1.PodSpec{SchedulerName: lpx.SchedulerName}, Status: corev1.PodStatus{Phase: corev1.PodRunning}, } if podIndex != "" { @@ -180,30 +168,28 @@ func lpuEvictionPodNames(pods []corev1.Pod) []string { return names } -func TestLPUEviction_PodsForTrigger(t *testing.T) { - t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { - t.Log("Observe a hybrid Agent without its canonical Grove index") - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, _ := newLPUEvictionReconciler(trigger, sibling) +func TestLPUEvictionRejectsMissingPodIndex(t *testing.T) { + t.Log("Observe a hybrid Agent without its canonical Grove index") + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, _ := newLPUEvictionReconciler(t, trigger, sibling) - t.Log("Reject the incomplete identity before selecting any deletion") - pods, err := r.podsForTrigger(context.Background(), trigger) + t.Log("Reject the incomplete identity before selecting any deletion") + pods, err := r.podsForTrigger(context.Background(), trigger) - require.Error(t, err) - assert.Nil(t, pods) - }) + require.Error(t, err) + assert.Nil(t, pods) } func TestLPUEvictionPreservesNativeReplacement(t *testing.T) { t.Log("Keep the original snapshot while a new native clique coexists") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) replacement := newLPUEvictionPod("agent-1", "0", "1", false, "") replacement.OwnerReferences[0].UID = "replacement-clique" - r, _ := newLPUEvictionReconciler(trigger, replacement) + r, _ := newLPUEvictionReconciler(t, trigger, replacement) pods, err := r.podsForTrigger(t.Context(), trigger) require.NoError(t, err) require.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) @@ -217,7 +203,7 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { t.Run(string(mode), func(t *testing.T) { t.Log("OnDelete can retain old Pods and replace others under the same clique") trigger := newLPUEvictionPod("agent-old-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) sibling := newLPUEvictionPod("agent-old-1", "0", "1", false, "") replacement := newLPUEvictionPod("agent-new-1", "0", "1", false, "") trigger.Labels[grovecommon.LabelPodTemplateHash] = "old-image" @@ -227,7 +213,7 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { sibling.Spec.Containers = trigger.Spec.Containers replacement.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:new"}} require.Equal(t, trigger.OwnerReferences, replacement.OwnerReferences) - r, c := newLPUEvictionReconciler(trigger, sibling, replacement) + r, c := newLPUEvictionReconciler(t, trigger, sibling, replacement) t.Log("Evict the old revision's sibling without deleting the replacement") reconcileLPUEviction(t, r, trigger.Name) @@ -255,7 +241,7 @@ func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { agent := newLPUEvictionPod("agent-1", "0", "1", false, "") otherReplica := newLPUEvictionPod("agent-other-replica", "1", "0", false, "") - r, c := newLPUEvictionReconciler( + r, c := newLPUEvictionReconciler(t, trigger, conductor, agent, @@ -294,11 +280,11 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { t.Log("Observe an ownership snapshot with a stale LGD UID") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) clique := &grovev1alpha1.PodClique{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: "agents"}, clique)) - clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "stale-lgd-uid" + clique.Annotations[lpx.DeploymentUIDAnnotation] = "stale-lgd-uid" require.NoError(t, c.Update(t.Context(), clique)) t.Log("Return a retryable error without deleting the sibling") @@ -307,7 +293,7 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { assert.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) t.Log("Evict the sibling when the owner snapshot catches up") - clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "test-lgd-uid" + clique.Annotations[lpx.DeploymentUIDAnnotation] = "test-lgd-uid" require.NoError(t, c.Update(t.Context(), clique)) reconcileLPUEviction(t, r, trigger.Name) assert.Empty(t, remainingLPUEvictionPods(t, c)) @@ -316,7 +302,7 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { current := &corev1.Pod{} @@ -343,7 +329,7 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" noPartition := newLPUOnlyConductor() otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") t.Log("Consume the real shared-cache Pod projection without deprecated row annotations") @@ -351,7 +337,7 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { podcache.Project(pod) } - r, c := newLPUEvictionReconciler( + r, c := newLPUEvictionReconciler(t, trigger, samePartition, otherPartition, @@ -400,20 +386,20 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Observe a valid sibling before another candidate whose identity cannot authorize deletion") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) if test.missingHash { - delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) + delete(trigger.Annotations, consts.AnnotationExtraResourcesHash) } if test.missingModel { delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) } sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") candidate := newLPUEvictionPod("agent-2", "0", test.candidateIndex, false, "") - r, c := newLPUEvictionReconciler(trigger, sibling, candidate) + r, c := newLPUEvictionReconciler(t, trigger, sibling, candidate) config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} require.NoError(t, c.Get(t.Context(), key, config)) - require.Equal(t, lpuEvictionTestConfigHash, dynamolpx.LPUConfigMapHash(config)) + require.Equal(t, lpuEvictionTestConfigHash, lpx.LPUConfigMapHash(config)) if test.missingConfig { require.NoError(t, c.Delete(t.Context(), config)) } @@ -449,11 +435,11 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { func TestLPUEvictionConfigMapObservationWakesTrigger(t *testing.T) { t.Log("Observe a hybrid disruption before its immutable runtime table reaches the cache") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} require.NoError(t, c.Get(t.Context(), key, config)) require.NoError(t, c.Delete(t.Context(), config)) _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) @@ -478,13 +464,13 @@ func TestLPUEvictionLPUOnlyNeedsNoRuntimePartitionTable(t *testing.T) { t.Run(string(mode), func(t *testing.T) { t.Log("Observe an LPU-only disruption without hybrid runtime partition metadata") trigger := newLPUEvictionPod("agent-0", "0", "", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) sibling := newLPUEvictionPod("agent-1", "0", "", false, "") - delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) - delete(sibling.Annotations, commonconsts.AnnotationExtraResourcesHash) - r, c := newLPUEvictionReconciler(trigger, sibling) + delete(trigger.Annotations, consts.AnnotationExtraResourcesHash) + delete(sibling.Annotations, consts.AnnotationExtraResourcesHash) + r, c := newLPUEvictionReconciler(t, trigger, sibling) config := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), + Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), }} require.NoError(t, c.Delete(t.Context(), config)) @@ -499,7 +485,7 @@ func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t trigger := newLPUEvictionPod("agent-0", "0", "0", false, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) result := reconcileLPUEviction(t, r, "agent-0") assert.Equal(t, ctrl.Result{}, result) @@ -549,16 +535,16 @@ func TestLPUEviction_Noops(t *testing.T) { trigger.Status.Phase = tt.phase trigger.Status.Reason = tt.statusReason if tt.nonLPU { - trigger.Labels[commonconsts.KubeLabelDynamoComponentType] = commonconsts.ComponentTypeWorker + trigger.Labels[consts.KubeLabelDynamoComponentType] = consts.ComponentTypeWorker } if tt.conductor { trigger.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor } if tt.missingMode { - delete(trigger.Annotations, dynamolpx.WorkloadModeAnnotation) + delete(trigger.Annotations, lpx.WorkloadModeAnnotation) } sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) t.Log("Reconcile without deleting the surviving Agent") result := reconcileLPUEviction(t, r, trigger.Name) @@ -573,7 +559,7 @@ func TestLPUEviction_MissingGroveLabelsReturnsError(t *testing.T) { delete(trigger.Labels, grovecommon.LabelPodCliqueScalingGroup) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) + r, c := newLPUEvictionReconciler(t, trigger, sibling) result, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: trigger.Name, Namespace: lpuEvictionTestNamespace}, diff --git a/deploy/operator/internal/controller/lpx/lifecycle.go b/deploy/operator/internal/controller/lpx/lifecycle.go deleted file mode 100644 index c95f705aff22..000000000000 --- a/deploy/operator/internal/controller/lpx/lifecycle.go +++ /dev/null @@ -1,936 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - "crypto/sha256" - "encoding/binary" - "errors" - "fmt" - "hash" - "sort" - "strconv" - "strings" - "time" - "unicode/utf8" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" -) - -const ( - lpxDeploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation - lpxOwnerUIDLabel = lpxDeploymentUIDAnnotation - lpxModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" - lpxPodCliqueSetKind = "PodCliqueSet" - lpxRetirementRequeueAfter = 5 * time.Second - maxDGDConditionMessageSize = 32768 -) - -const ( - lpxRetiringReason string = "LPXRetiring" - lpxSchedulerStatusInvalidReason string = "LPXSchedulerStatusInvalid" -) - -// lpxClassification is deliberately private and closed: reconciliation can -// report only one legal protocol state, never a freely assembled collection -// of correlated booleans. -type lpxClassification interface { - lpxClassification() -} - -type lpxRejected struct { - reason string -} - -type lpxMaterializing struct { - workloadDigest string - requests []lpxModelMaterializing - workload *lpx.SelectedWorkload - plan *lpx.MaterializationPlan -} - -type reconcileOutcome struct { - State nvidiacomv1beta1.DGDState - Reason string - Message string - retirementScope lpxRetirementScope -} - -type lpxRetirementScope uint8 - -const ( - lpxWorkloadRetirement lpxRetirementScope = iota - lpxRequestRetirement -) - -// lpxModelMaterializing is the request-local state for one model projection. -// The enclosing lpxMaterializing owns the shared Grove identity. -type lpxModelMaterializing struct { - requestName string - modelProjection *lpx.ModelProjection - replicaIndex int32 - agentCliqueName string -} - -type lpxGroveIdentity struct { - pcs *grovev1alpha1.PodCliqueSet - plan *lpx.MaterializationPlan -} - -type lpxClosed struct { - incomplete string -} - -type lpxOpen struct{} - -type lpxBound struct{} - -type lpxSchedulerObserved lpxv1alpha1.LPUPipelineRequestStatus - -type lpxRetiring struct { - retirementReason string - scope lpxRetirementScope -} - -func (*lpxRejected) lpxClassification() {} -func (*lpxClosed) lpxClassification() {} -func (*lpxOpen) lpxClassification() {} -func (*lpxBound) lpxClassification() {} -func (*lpxSchedulerObserved) lpxClassification() {} -func (*lpxRetiring) lpxClassification() {} - -func lpxResult(classification lpxClassification) reconcileOutcome { - result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} - switch state := classification.(type) { - case *lpxRejected: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = "LPXRejected" - result.Message = state.reason - case *lpxClosed: - result.Reason = "LPXPublicationPending" - result.Message = state.incomplete - case *lpxOpen: - result.Reason = "LPXPublished" - result.Message = "LPX request is published" - case *lpxBound: - result.Reason = "AwaitingLPXRuntimeActivation" - result.Message = "LPX scheduling is bound; runtime activation and serving readiness remain external" - case *lpxSchedulerObserved: - result = lpxSchedulerResult(state) - case *lpxRetiring: - result.Reason = lpxRetiringReason - result.Message = state.retirementReason - result.retirementScope = state.scope - case *lpxDeadlineExceeded: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = lpxSchedulingDeadlineExceededReason - result.Message = "An LPX request exceeded its scheduling deadline" - default: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = "LPXInternalClassificationError" - result.Message = "Unknown LPX reconciliation classification" - } - return result -} - -// lpxSchedulerResult classifies one receipt and formats its diagnostics once. -func lpxSchedulerResult(state *lpxSchedulerObserved) reconcileOutcome { - result := reconcileOutcome{State: nvidiacomv1beta1.DGDStatePending} - var summary string - switch state.Phase { - case lpxv1alpha1.RequestPhasePending: - result.Reason = "LPXSchedulerPending" - summary = "LPX scheduler is waiting to plan the current request" - case lpxv1alpha1.RequestPhaseNoFit: - result.Reason = "LPXNoFit" - summary = "LPX scheduler found no placement for the current request" - case lpxv1alpha1.RequestPhaseUnsupported: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = "LPXUnsupported" - summary = "LPX scheduler cannot support the current request" - case lpxv1alpha1.RequestPhasePlanned: - result.Reason = "LPXPlanned" - summary = "LPX scheduler committed a placement plan for the current request" - case lpxv1alpha1.RequestPhaseReserving: - result.Reason = "LPXReserving" - summary = "LPX scheduler is reserving the planned LPU allocation" - case lpxv1alpha1.RequestPhaseBinding: - result.Reason = "LPXBinding" - summary = "LPX scheduler is binding the planned workload Pods" - case lpxv1alpha1.RequestPhaseDegraded: - result.Reason = "LPXDegraded" - summary = "LPX scheduler is repairing or releasing a degraded placement" - case lpxv1alpha1.RequestPhaseReleasing: - result.Reason = "LPXReleasing" - summary = "LPX scheduler is releasing the current plan" - case lpxv1alpha1.RequestPhaseReleased: - result.Reason = "LPXReleased" - summary = "LPX scheduler released the current plan" - case lpxv1alpha1.RequestPhaseBound: - return lpxResult(&lpxBound{}) - default: - result.State = nvidiacomv1beta1.DGDStateFailed - result.Reason = lpxSchedulerStatusInvalidReason - summary = fmt.Sprintf("LPX scheduler reported unknown phase %q", state.Phase) - } - result.Message = lpxMessageWithDiagnostics(summary, state.Diagnostics) - return result -} - -func lpxMessageWithDiagnostics(summary string, diagnostics []lpxv1alpha1.StatusDiagnostic) string { - if len(diagnostics) == 0 { - return summary - } - var message strings.Builder - message.WriteString(summary) - message.WriteString(". LPX diagnostics: ") - for index, diagnostic := range diagnostics { - if index > 0 { - message.WriteString("; ") - } - message.WriteString(diagnostic.Code) - message.WriteString(" [") - message.WriteString(diagnostic.Subject) - message.WriteString("]: ") - message.WriteString(diagnostic.Detail) - - // Later diagnostics cannot change the retained prefix. - if message.Len() > maxDGDConditionMessageSize { - break - } - } - return truncateLPXMessage(message.String()) -} - -func truncateLPXMessage(message string) string { - if len(message) <= maxDGDConditionMessageSize { - return message - } - const suffix = "…" - last := maxDGDConditionMessageSize - len(suffix) - for last > 0 && !utf8.ValidString(message[:last]) { - last-- - } - return message[:last] + suffix -} - -func overlayLPXResult( - base reconcileOutcome, - classification lpxClassification, -) reconcileOutcome { - if _, ok := classification.(*lpxBound); ok { - return base - } - return lpxResult(classification) -} - -// reconcileLPXKnownIntentFence deletes a publication before any new graph can -// be rendered when selection, DGD generation, model, or DGD incarnation changes. -// pcs is nil when its stable name was not found. -func (r *graphReconciler) reconcileLPXKnownIntentFence( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - requests []lpxv1alpha1.LPUPipelineRequest, -) (lpxClassification, []lpxv1alpha1.LPUPipelineRequest, error) { - var err error - if requests == nil { - requests, err = r.listOwnedLPXRequests(ctx, deployment, pcs) - } - if err != nil { - return nil, nil, err - } - selected := source.HasLPXComponent() - - for index := range requests { - request := &requests[index] - keep := selected && - request.Annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && - request.Annotations[lpx.ExecutionBackendAnnotation] == "" - if keep { - continue - } - reason := "LPX scheduler was deselected" - if selected { - reason = "The DGD incarnation, generation, model, or selected input changed" - } - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, request, reason) - return retiring, nil, retireErr - } - return nil, requests, nil -} - -// prepareLPXMaterializing resolves the selected workload against the observed -// Grove capacity. pcs is nil when the stable PodCliqueSet name was not found. -func (r *graphReconciler) prepareLPXMaterializing( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, -) (*lpxMaterializing, lpxClassification, error) { - workload, err := lpx.ResolveSelectedWorkload(ctx, source, r.modelRegistry) - if err != nil { - if errors.Is(err, lpx.ErrBuildSnapshotAcquisition) { - return nil, nil, err - } - return nil, &lpxRejected{reason: err.Error()}, nil - } - projections := workload.ModelProjections() - plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) - if err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil - } - // Omitted replicas leave the native scaling group in charge of engine count. - if lpx.ServingComponent(source).Replicas == nil { - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: plan.LPXScalingGroup}, group); err != nil { - if !apierrors.IsNotFound(err) { - return nil, nil, err - } - } else { - // Reject an unrelated PCS before consulting any group it happens to own. - if pcs != nil && !metav1.IsControlledBy(pcs, deployment) { - return nil, nil, fmt.Errorf("refusing to use replicas from a foreign LPX PodCliqueSet") - } - // A group outliving its PCS is normal during background Grove cleanup. - if pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(group, pcs) { - return nil, &lpxRetiring{retirementReason: "Waiting for the previous LPX scaling group to finish cleanup"}, nil - } - plan.Replicas = group.Spec.Replicas - if err := plan.ValidateReplicaCount(); err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil - } - } - } - workloadDigest := workload.Digest().String() - requests := make([]lpxModelMaterializing, 0, len(projections)*int(plan.Replicas)) - for replicaIndex := int32(0); replicaIndex < plan.Replicas; replicaIndex++ { - for projectionIndex, projection := range projections { - requestDigest := digestAttemptKey( - deployment.Namespace, - deployment.Name, - deployment.UID, - projection.Model(), - replicaIndex, - ) - requests = append(requests, lpxModelMaterializing{ - requestName: lpxRequestName(deployment.Name, requestDigest), - modelProjection: projection, - replicaIndex: replicaIndex, - agentCliqueName: plan.ForReplica(replicaIndex).Agents[projectionIndex].CliqueName, - }) - } - } - return &lpxMaterializing{ - workloadDigest: workloadDigest, - requests: requests, - workload: workload, - plan: plan, - }, nil, nil -} - -// retireInvalidLPXWorkload also retires staging that never acquired an LPR. -// Healthy publication and spec-write fences intentionally do not use this path. -// pcs is nil when its stable name was not found. -func (r *graphReconciler) retireInvalidLPXWorkload( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - reason string, -) (*lpxRetiring, error) { - selector := client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)} - owned := make([]client.Object, 0) - podCliqueSets := &grovev1alpha1.PodCliqueSetList{} - if err := r.apiReader.List(ctx, podCliqueSets, client.InNamespace(deployment.Namespace), selector); err != nil { - return nil, err - } - for index := range podCliqueSets.Items { - if pcs := &podCliqueSets.Items[index]; metav1.IsControlledBy(pcs, deployment) { - owned = append(owned, pcs) - } - } - - services := &corev1.ServiceList{} - if err := r.apiReader.List(ctx, services, client.InNamespace(deployment.Namespace), selector); err != nil { - return nil, err - } - for index := range services.Items { - if service := &services.Items[index]; metav1.IsControlledBy(service, deployment) { - owned = append(owned, service) - } - } - - configMaps := &corev1.ConfigMapList{} - if err := r.apiReader.List(ctx, configMaps, client.InNamespace(deployment.Namespace), selector); err != nil { - return nil, err - } - for index := range configMaps.Items { - if configMap := &configMaps.Items[index]; metav1.IsControlledBy(configMap, deployment) { - owned = append(owned, configMap) - } - } - - requests, err := r.listOwnedLPXRequests(ctx, deployment, pcs) - if err != nil { - return nil, err - } - if len(owned) == 0 && len(requests) == 0 { - return nil, nil - } - if len(owned) > 0 { - // A newer child seen during listing must not lose its staging to this stale failure. - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return nil, err - } - - propagation := metav1.DeletePropagationBackground - for _, resource := range owned { - if !resource.GetDeletionTimestamp().IsZero() { - continue - } - uid, resourceVersion := resource.GetUID(), resource.GetResourceVersion() - if err := r.Delete(ctx, resource, &client.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}, - PropagationPolicy: &propagation, - }); err != nil && !apierrors.IsNotFound(err) { - return nil, err - } - } - } - - return &lpxRetiring{retirementReason: reason}, nil -} - -func digestAttemptKey( - namespace string, - name string, - uid types.UID, - model string, - replicaIndex int32, -) string { - h := sha256.New() - writeLPXHashField(h, "namespace", namespace) - writeLPXHashField(h, "name", name) - writeLPXHashField(h, "uid", string(uid)) - writeLPXHashField(h, "model", model) - if replicaIndex > 0 { - writeLPXHashField(h, "group-replica", strconv.FormatInt(int64(replicaIndex), 10)) - } - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} - -func writeLPXHashField(h hash.Hash, tag, value string) { - var size [8]byte - for _, field := range []string{tag, value} { - binary.BigEndian.PutUint64(size[:], uint64(len(field))) - _, _ = h.Write(size[:]) - _, _ = h.Write([]byte(field)) - } -} - -// hasCurrentLPXPublicationAnnotations requires nonnil desired and its DGD; a nil object interface does not match. -func (desired *lpxMaterializing) hasCurrentLPXPublicationAnnotations( - object metav1.Object, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) bool { - if object == nil || !object.GetDeletionTimestamp().IsZero() { - return false - } - annotations := object.GetAnnotations() - executionBackend := annotations[lpx.ExecutionBackendAnnotation] - return annotations[lpxDeploymentUIDAnnotation] == string(deployment.UID) && - executionBackend == "" -} - -func lpxRequestName(dgdName, digest string) string { - prefix := strings.ReplaceAll(dgdName, ".", "-") - if len(prefix) > 23 { - prefix = strings.TrimRight(prefix[:23], "-") - } - hexDigest := strings.TrimPrefix(digest, "sha256:") - return fmt.Sprintf("lpx-%s-%s", prefix, hexDigest[:32]) -} - -// reconcileLPXPublicationFence requires a nonnil desired whose requests remain immutable during the call. -// pcs is nil when the desired PodCliqueSet was not found; it cannot be recreated until garbage collection -// removes every request from its previous incarnation. -func (r *graphReconciler) reconcileLPXPublicationFence( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - pcs *grovev1alpha1.PodCliqueSet, -) (map[string]*lpxv1alpha1.LPUPipelineRequest, lpxClassification, error) { - // Retain each desired position for digest lookup and collision precedence. - desiredByName := make(map[string]int, len(desired.requests)) - for index := range desired.requests { - desiredByName[desired.requests[index].requestName] = index - } - currents := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) - foreignCollision := len(desired.requests) - stale := make([]*lpxv1alpha1.LPUPipelineRequest, 0) - requests, err := r.listLPXRequestCandidates(ctx, deployment) - if err != nil { - return nil, nil, err - } - for index := range requests.Items { - request := &requests.Items[index] - desiredIndex, desiredName := desiredByName[request.Name] - ownedByChild := ownsLPXRequest(deployment, pcs, request) - if !ownedByChild { - // Preserve desired-order error precedence without retaining collision state. - if desiredName { - foreignCollision = min(foreignCollision, desiredIndex) - } - continue - } - if desiredName { - currents[request.Name] = request - continue - } - stale = append(stale, request) - } - // A deterministic-name collision must never be adopted or deleted. - if foreignCollision < len(desired.requests) { - return nil, nil, fmt.Errorf("LPX request name %q is occupied by a foreign owner", desired.requests[foreignCollision].requestName) - } - if pcs == nil && len(currents)+len(stale) > 0 { - return currents, &lpxRetiring{retirementReason: "Waiting for garbage collection to remove the previous LPX requests"}, nil - } - if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != desired.workloadDigest { - if err := r.deleteLPXPodCliqueSet(ctx, deployment, pcs.Name, pcs.UID); err != nil { - return nil, nil, err - } - return currents, &lpxRetiring{retirementReason: "The selected LPX workload shape changed"}, nil - } - if pcs != nil && !pcs.DeletionTimestamp.IsZero() { - return currents, &lpxRetiring{retirementReason: "Waiting for the current PodCliqueSet to finish cleanup"}, nil - } - if len(stale) > 0 { - if err := r.deleteStaleLPXRequests(ctx, deployment, stale); err != nil { - return nil, nil, err - } - return currents, &lpxRetiring{ - retirementReason: "One or more engines are no longer requested", - scope: lpxRequestRetirement, - }, nil - } - - // Grove must not recreate pods while a retiring request still protects their names. - for _, request := range desired.requests { - if live := currents[request.requestName]; live != nil && !live.DeletionTimestamp.IsZero() { - return currents, &lpxRetiring{ - retirementReason: "Waiting for the current LPX request to finish cleanup", - scope: lpxRequestRetirement, - }, nil - } - } - - // A recorded deadline failure fences this generation even if the scheduler later reports a terminal phase. - if lpxDeadlineFailureCurrent(deployment) { - return currents, &lpxDeadlineExceeded{}, nil - } - return currents, nil, nil -} - -// listLPXRequestCandidates uses the propagated deployment identity to avoid a -// namespace-wide scan. Ownership is still checked before a returned request is -// observed or changed. -func (r *graphReconciler) listLPXRequestCandidates( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) (*lpxv1alpha1.LPUPipelineRequestList, error) { - list := &lpxv1alpha1.LPUPipelineRequestList{} - err := r.apiReader.List(ctx, list, - client.InNamespace(deployment.Namespace), - client.MatchingLabels{lpxOwnerUIDLabel: string(deployment.UID)}, - ) - return list, err -} - -// observeCurrentLPXPodCliqueSet returns nil when the deployment's stable PCS -// name is absent from the informer cache. -func (r *graphReconciler) observeCurrentLPXPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, -) (*grovev1alpha1.PodCliqueSet, error) { - pcs := &grovev1alpha1.PodCliqueSet{} - err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: dynamo.PCSNameForLPX(deployment)}, pcs) - if apierrors.IsNotFound(err) { - return nil, nil - } - return pcs, err -} - -// ownsLPXRequest follows the request's exact controller chain using the PCS -// already observed by the caller. pcs is nil after that PCS disappears. -func ownsLPXRequest( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - request *lpxv1alpha1.LPUPipelineRequest, -) bool { - owner := metav1.GetControllerOf(request) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != lpxPodCliqueSetKind { - return false - } - if pcs != nil { - return metav1.IsControlledBy(pcs, deployment) && metav1.IsControlledBy(request, pcs) - } - - // Keep observing requests from the missing PCS, not another owner with a copied label. - return owner.Name == dynamo.PCSNameForLPX(deployment) && request.Labels[lpxOwnerUIDLabel] == string(deployment.UID) -} - -// listOwnedLPXRequests filters the deployment-scoped list against the caller's -// PCS observation. pcs is nil after that owner disappears. -func (r *graphReconciler) listOwnedLPXRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, -) ([]lpxv1alpha1.LPUPipelineRequest, error) { - requests, err := r.listLPXRequestCandidates(ctx, deployment) - if err != nil { - return nil, err - } - owned := make([]lpxv1alpha1.LPUPipelineRequest, 0) - for index := range requests.Items { - request := &requests.Items[index] - if ownsLPXRequest(deployment, pcs, request) { - owned = append(owned, *request) - } - } - sort.Slice(owned, func(i, j int) bool { return owned[i].Name < owned[j].Name }) - return owned, nil -} - -// deleteLPXPodCliqueSet starts background cascading deletion of an exact PCS. -// A same-name replacement proves that the old UID is already absent and is -// never adopted or deleted. -func (r *graphReconciler) deleteLPXPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - pcsName string, - pcsUID types.UID, -) error { - pcs := &grovev1alpha1.PodCliqueSet{} - if err := r.apiReader.Get(ctx, types.NamespacedName{Namespace: deployment.Namespace, Name: pcsName}, pcs); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return err - } - if pcs.UID != pcsUID { - return nil - } - if !metav1.IsControlledBy(pcs, deployment) { - return fmt.Errorf("refusing to delete PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) - } - if !pcs.DeletionTimestamp.IsZero() { - return nil - } - if deployment.DeletionTimestamp.IsZero() { - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return err - } - } - propagation := metav1.DeletePropagationBackground - if err := r.Delete(ctx, pcs, &client.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, - PropagationPolicy: &propagation, - }); err != nil && !apierrors.IsNotFound(err) { - return err - } - return nil -} - -// deleteStaleLPXRequests removes scaled-in scheduler intent without disturbing -// the shared PCS. Grove and the LPX scheduler converge their pods independently. -func (r *graphReconciler) deleteStaleLPXRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - requests []*lpxv1alpha1.LPUPipelineRequest, -) error { - if deployment.DeletionTimestamp.IsZero() { - if err := r.validateLPXDeploymentAuthority(ctx, deployment); err != nil { - return err - } - } - for _, request := range requests { - if !request.DeletionTimestamp.IsZero() { - continue - } - if request.UID == "" { - return fmt.Errorf("refusing to delete LPX request %q without an immutable UID", request.Name) - } - uid, resourceVersion := request.UID, request.ResourceVersion - if err := r.Delete(ctx, request, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ - UID: &uid, ResourceVersion: &resourceVersion, - }}); err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("delete stale LPX request %q: %w", request.Name, err) - } - } - return nil -} - -// retireLPXEngineRequest removes the PCS whose immutable request no longer -// matches the selected workload. -func (r *graphReconciler) retireLPXEngineRequest( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - request *lpxv1alpha1.LPUPipelineRequest, - reason string, -) (lpxClassification, error) { - owner := metav1.GetControllerOf(request) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || - owner.Kind != lpxPodCliqueSetKind || owner.UID == "" { - return nil, fmt.Errorf("LPX request %q does not have an exact PodCliqueSet controller owner", request.Name) - } - if err := r.deleteLPXPodCliqueSet(ctx, deployment, owner.Name, owner.UID); err != nil { - return nil, err - } - return &lpxRetiring{retirementReason: reason}, nil -} - -func (r *graphReconciler) reconcileSelectedLPXFromCurrentRequests( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) (lpxClassification, error) { - if currents == nil { - currents = make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) - } - - pcs, classification, err := r.reconcileSelectedLPXPodCliqueSet( - ctx, - deployment, - desired, - currents, - ) - if classification != nil || err != nil { - return classification, err - } - - for index := range desired.requests { - request := &desired.requests[index] - if currents[request.requestName] != nil { - continue - } - groveIdentity := &lpxGroveIdentity{pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} - live, createErr := r.createPublishedLPXRequest(ctx, deployment, request, groveIdentity) - if createErr != nil { - return nil, createErr - } - currents[request.requestName] = live - } - - return classifyLPXCurrentRequests(desired.requests, currents), nil -} - -// classifyLPXCurrentRequests accepts absent requests. -func classifyLPXCurrentRequests( - requests []lpxModelMaterializing, - currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) lpxClassification { - // Stream terminal, side-effect-free classifications in request order. - var firstNonBound lpxClassification - for _, request := range requests { - if live := currents[request.requestName]; live != nil { - classification := classifyPublishedLPX(live) - if state, ok := classification.(*lpxSchedulerObserved); ok && lpxSchedulerResult(state).State == nvidiacomv1beta1.DGDStateFailed { - return classification - } - if _, bound := classification.(*lpxBound); !bound && firstNonBound == nil { - firstNonBound = classification - } - } - } - - if firstNonBound != nil { - return firstNonBound - } - return &lpxBound{} -} - -func (r *graphReconciler) reconcileSelectedLPXPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - currents map[string]*lpxv1alpha1.LPUPipelineRequest, -) (*grovev1alpha1.PodCliqueSet, lpxClassification, error) { - pcs, incomplete, err := r.observeLPXPodCliqueSet(ctx, deployment, desired, len(currents) == 0) - if err != nil { - return nil, nil, err - } - - if pcs == nil || incomplete != "" { - if len(currents) == 0 { - return nil, &lpxClosed{incomplete: incomplete}, nil - } - for _, request := range desired.requests { - if live := currents[request.requestName]; live != nil { - retiring, err := r.retireLPXEngineRequest(ctx, deployment, live, fmt.Sprintf("The published PCS identity became unavailable: %q", incomplete)) - return nil, retiring, err - } - } - return nil, &lpxClosed{incomplete: incomplete}, nil - } - - // The PCS owns each request; LPX resolves the replica-local clique references itself. - for index := range desired.requests { - request := &desired.requests[index] - identity := &lpxGroveIdentity{pcs: pcs, plan: desired.plan.ForReplica(request.replicaIndex)} - if live := currents[request.requestName]; live != nil { - if err := validateCurrentLPXRequest(deployment, request, identity, live); err != nil { - retiring, retireErr := r.retireLPXEngineRequest(ctx, deployment, live, err.Error()) - return nil, retiring, retireErr - } - } - } - return pcs, nil, nil -} - -// classifyPublishedLPX classifies a non-nil published request from its current scheduler receipt. -func classifyPublishedLPX(current *lpxv1alpha1.LPUPipelineRequest) lpxClassification { - if current == nil || current.Status == nil || current.Status.ObservedGeneration == nil || - *current.Status.ObservedGeneration != current.Generation { - return &lpxOpen{} - } - if current.Status.Phase == lpxv1alpha1.RequestPhaseBound { - return &lpxBound{} - } - observed := lpxSchedulerObserved(*current.Status.DeepCopy()) - return &observed -} - -func (r *graphReconciler) createPublishedLPXRequest( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - materializing *lpxModelMaterializing, - grove *lpxGroveIdentity, -) (*lpxv1alpha1.LPUPipelineRequest, error) { - if err := r.validateLPXPublicationSource(ctx, deployment); err != nil { - return nil, err - } - spec := materializing.modelProjection.RequestSpec(grove.plan, materializing.agentCliqueName) - request := &lpxv1alpha1.LPUPipelineRequest{ - TypeMeta: metav1.TypeMeta{ - APIVersion: lpxv1alpha1.GroupVersion.String(), - Kind: "LpuPipelineRequest", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: materializing.requestName, - Namespace: deployment.Namespace, - Labels: map[string]string{ - consts.KubeLabelDynamoGraphDeploymentName: metav1.GetControllerOf(deployment).Name, - lpxOwnerUIDLabel: string(deployment.UID), - }, - Annotations: lpxRequestAnnotations(deployment, materializing), - }, - Spec: spec, - } - if err := controllerutil.SetControllerReference(grove.pcs, request, r.Client.Scheme()); err != nil { - return nil, err - } - if err := r.Create(ctx, request); err != nil { - return nil, fmt.Errorf( - "create LPX request %q for model %q: %w", - request.Name, - materializing.modelProjection.Model(), - err, - ) - } - return request, nil -} - -func lpxRequestAnnotations( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - materializing *lpxModelMaterializing, -) map[string]string { - // Callers validate the exact source controller owner before constructing requests. - sourceOwner := metav1.GetControllerOf(deployment) - return map[string]string{ - lpx.DGDUIDAnnotation: string(sourceOwner.UID), - lpx.DeploymentNameAnnotation: deployment.Name, - lpxDeploymentUIDAnnotation: string(deployment.UID), - lpxModelAnnotation: materializing.modelProjection.Model(), - lpx.WorkloadDigestAnnotation: materializing.modelProjection.Digest().String(), - lpxv1alpha1.CompilerSnapshotDigestAnnotation: materializing.modelProjection.CompilerSnapshotDigest(), - } -} - -func validateCurrentLPXRequest( - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxModelMaterializing, - grove *lpxGroveIdentity, - request *lpxv1alpha1.LPUPipelineRequest, -) error { - if !metav1.IsControlledBy(request, grove.pcs) { - return fmt.Errorf("LPX request %q does not have the exact PCS owner", request.Name) - } - if request.UID == "" { - return fmt.Errorf("LPX request %q has no immutable UID", request.Name) - } - wantAnnotations := lpxRequestAnnotations(deployment, desired) - for key, value := range wantAnnotations { - if request.Annotations[key] != value { - return fmt.Errorf("LPX request %q immutable annotation %q no longer matches the current deployment", request.Name, key) - } - } - wantSpec := desired.modelProjection.RequestSpec(grove.plan, desired.agentCliqueName) - if !apiequality.Semantic.DeepEqual(request.Spec, wantSpec) { - return fmt.Errorf("LPX request %q immutable intent no longer matches the current publication", request.Name) - } - return nil -} - -func (r *graphReconciler) observeLPXPodCliqueSet( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, - noCurrentRequests bool, -) (*grovev1alpha1.PodCliqueSet, string, error) { - pcs := &grovev1alpha1.PodCliqueSet{} - key := types.NamespacedName{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName} - if err := r.apiReader.Get(ctx, key, pcs); err != nil { - if apierrors.IsNotFound(err) { - return nil, "Waiting for the selected Grove PodCliqueSet", nil - } - return nil, "", err - } - if noCurrentRequests && !metav1.IsControlledBy(pcs, deployment) { - return nil, "", fmt.Errorf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", pcs.Name) - } - if pcs.UID == "" || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) { - return nil, "The selected Grove PodCliqueSet lacks the exact LPXGraphDeployment owner identity", nil - } - - incomplete := "" - if !desired.hasCurrentLPXPublicationAnnotations(pcs, deployment) { - incomplete = "The selected Grove PodCliqueSet is stale" - } - if pcs.Spec.Replicas != 1 { - if incomplete == "" { - incomplete = fmt.Sprintf("The selected Grove PodCliqueSet has %d replicas, want 1", pcs.Spec.Replicas) - } - } - return pcs, incomplete, nil -} diff --git a/deploy/operator/internal/controller/lpx/lifecycle_test.go b/deploy/operator/internal/controller/lpx/lifecycle_test.go deleted file mode 100644 index 6771b8a92060..000000000000 --- a/deploy/operator/internal/controller/lpx/lifecycle_test.go +++ /dev/null @@ -1,2146 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - "errors" - "fmt" - "maps" - "net/url" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "testing" - "time" - "unicode/utf8" - - configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" - groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/require" - autoscalingv1 "k8s.io/api/autoscaling/v1" - corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/events" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" -) - -const lpxTestOtherName = "other" - -func newLPXTestScheme(t testing.TB) *k8sruntime.Scheme { - t.Helper() - scheme := k8sruntime.NewScheme() - for _, add := range []func(*k8sruntime.Scheme) error{ - corev1.AddToScheme, resourcev1.AddToScheme, nvidiacomv1alpha1.AddToScheme, nvidiacomv1beta1.AddToScheme, - grovev1alpha1.AddToScheme, lpxv1alpha1.AddToScheme, - } { - require.NoError(t, add(scheme)) - } - return scheme -} - -type snapshotFailureRegistry struct { - lpx.ModelRegistry - err error -} - -type downloadOrderedLPXRegistry struct { - lpx.ModelRegistry - buildURL url.URL - downloaded bool - calls []string -} - -func (r *snapshotFailureRegistry) AcquireBuildSnapshot(context.Context, string) (*lpx.BuildSnapshot, error) { - return nil, r.err -} - -func (r *downloadOrderedLPXRegistry) BuildURL(string) (*url.URL, error) { - buildURL := r.buildURL - return &buildURL, nil -} - -func (r *downloadOrderedLPXRegistry) EnsureDownloaded(context.Context, url.URL) (bool, error) { - firstDownload := !slices.Contains(r.calls, "download") - r.calls = append(r.calls, "download") - if firstDownload { - return false, nil - } - r.downloaded = true - return true, nil -} - -func (r *downloadOrderedLPXRegistry) AcquireBuildSnapshot( - ctx context.Context, - buildID string, -) (*lpx.BuildSnapshot, error) { - r.calls = append(r.calls, "snapshot") - if !r.downloaded { - return nil, errors.New("LPX snapshot acquired before Model Express download") - } - return r.ModelRegistry.AcquireBuildSnapshot(ctx, buildID) -} - -func requirePreparedLPX( - t *testing.T, - reconciler *graphReconciler, - ctx context.Context, - dgd *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, -) (*lpxMaterializing, *lpxRejected) { - t.Helper() - desired, classification, err := reconciler.prepareLPXMaterializing(ctx, dgd, source, nil) - require.NoError(t, err) - if classification == nil { - return desired, nil - } - rejected, ok := classification.(*lpxRejected) - require.True(t, ok, "classification=%T", classification) - return desired, rejected -} - -func newPreparedLPXTestReconciler( - t *testing.T, - registry lpx.ModelRegistry, - ctx context.Context, - dgd *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, -) (*graphReconciler, *lpxMaterializing) { - t.Helper() - - // Construct the fake client before preparing the selected LPX plan. - reconciler := newLPXTestReconciler(t, registry, dgd, source) - desired, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) - require.Nil(t, rejected) - return reconciler, desired -} - -func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { - t.Log("Publish the implicit hybrid runtime without interpreting stale selector annotations") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineLPX) - source.Annotations[consts.KubeAnnotationLPXSchedulerBackend] = "unknown-scheduler" - source.Annotations[consts.KubeAnnotationLPXExecutionBackend] = "unknown-execution" - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - require.NotNil(t, desired) - require.Equal(t, "unknown-scheduler", source.Annotations[consts.KubeAnnotationLPXSchedulerBackend]) - require.Equal(t, "unknown-execution", source.Annotations[consts.KubeAnnotationLPXExecutionBackend]) - require.Empty(t, desired.plan.ConductorTemplate) - require.Empty(t, desired.plan.ConductorClique) - require.NotEmpty(t, desired.plan.CyborgClique) - - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - for _, object := range objects { - require.NotEmpty(t, object.GetName()) - } - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - require.NotContains(t, group.Spec.CliqueNames, "") - cyborg := findLPXTestClique(t, objects, desired.plan.CyborgClique) - require.NotContains(t, cyborg.Spec.StartsAfter, "") - require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.IsType(t, &lpxOpen{}, classification) - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - require.NotNil(t, request.Spec.CyborgPodCliqueRef) - require.Equal(t, desired.plan.CyborgClique, request.Spec.CyborgPodCliqueRef.Name) -} - -func TestSelectedLPXColdCacheDownloadsBeforeSnapshot(t *testing.T) { - t.Log("Build a selected LPX deployment backed by an initially cold Model Express cache") - child, source, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) - registry := &downloadOrderedLPXRegistry{ - ModelRegistry: baseRegistry, - buildURL: url.URL{ - Scheme: lpx.BuildSchemeGCS, - Host: "test-bucket", - Path: "/build-v2", - }, - } - child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "Ready", ObservedGeneration: child.Generation}} - child.Status.ObservedGeneration = child.Generation - child.Status.ModelDownload = &nvidiacomv1beta1.ModelDownloadStatus{ - Builds: []string{registry.buildURL.String()}, - LastCheckedAt: &metav1.Time{Time: time.Now()}, - } - reconciler := newLPXTestReconciler(t, registry, child, source) - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - - t.Log("Force Model Express despite the fresh cached status and avoid acquiring a build snapshot") - result, err := reconciler.Reconcile(t.Context(), request) - require.NoError(t, err) - require.Equal(t, modelDownloadRequeueAfter, result.RequeueAfter) - require.Equal(t, []string{"download"}, registry.calls) - require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) - require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, "Ready")) - require.NotNil(t, child.Status.ModelDownload) - require.Empty(t, child.Status.ModelDownload.Builds) - sets := &grovev1alpha1.PodCliqueSetList{} - require.NoError(t, reconciler.List(t.Context(), sets)) - require.Empty(t, sets.Items) - - t.Log("Complete the download and reconcile the selected deployment again") - _, err = reconciler.Reconcile(t.Context(), request) - require.NoError(t, err) - require.Equal(t, []string{"download", "download", "snapshot"}, registry.calls) - require.NoError(t, reconciler.Get(t.Context(), request.NamespacedName, child)) - require.NotNil(t, child.Status.ModelDownload) - require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) -} - -func TestSelectedLPXSuccessorBuildDownloadsBeforePublication(t *testing.T) { - t.Log("Publish the previous build before switching to an uncached successor") - ctx := t.Context() - child, source, baseRegistry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, baseRegistry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - previous := getLPXRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].requestName) - const nextBuild = "next-build" - registry := &downloadOrderedLPXRegistry{ - ModelRegistry: newLPXTestRegistryWithPartitionsAndMode(t, nextBuild, []int{7, 8}, manifestcapnpv2.CompilationMode_lpuOnly), - buildURL: url.URL{Scheme: lpx.BuildSchemeGCS, Host: "test-bucket", Path: "/" + nextBuild}, - } - r.modelRegistry = registry - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) - lpx.ServingComponent(source).LPX.BuildID = nextBuild - source.Generation++ - require.NoError(t, r.Update(ctx, source)) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Status.ExpiredRequestUIDs = []types.UID{previous.UID} - require.NoError(t, r.Status().Update(ctx, child)) - child.Generation++ - var err error - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - require.NoError(t, r.Update(ctx, child)) - - t.Log("Keep the download poll despite an interior expiry, preserving the existing request") - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - result, err := r.Reconcile(ctx, request) - require.NoError(t, err) - require.Equal(t, modelDownloadRequeueAfter, result.RequeueAfter) - require.Equal(t, []string{"snapshot", "download"}, registry.calls) - require.Equal(t, previous, getLPXRequest(t, ctx, r.Client, previous.Namespace, previous.Name)) - require.NoError(t, r.Get(ctx, request.NamespacedName, child)) - require.Equal(t, child.Generation, child.Status.ObservedGeneration) - require.Equal(t, []types.UID{previous.UID}, child.Status.ExpiredRequestUIDs) - - t.Log("Resolve the successor after download completes on the next reconciliation") - _, err = r.Reconcile(ctx, request) - require.NoError(t, err) - require.Equal(t, []string{"snapshot", "download", "snapshot", "download", "snapshot"}, registry.calls) - require.NoError(t, r.Get(ctx, request.NamespacedName, child)) - require.Equal(t, []string{registry.buildURL.String()}, child.Status.ModelDownload.Builds) - require.NotNil(t, child.Status.ModelDownload.LastCheckedAt) -} - -func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXSpecDecodeTestDGD(t) - - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - require.Len(t, desired.requests, 3) - require.Len(t, desired.plan.Agents, 3) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.IsType(t, &lpxOpen{}, classification) - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd, findLPXTestPodCliqueSet(t, objects)) - require.NoError(t, err) - require.Len(t, requests, 3) - - requestByModel := make(map[string]lpxv1alpha1.LPUPipelineRequest, len(requests)) - for _, request := range requests { - model := request.Annotations[lpxModelAnnotation] - requestByModel[model] = request - require.Equal(t, string(dgd.UID), request.Labels[lpxOwnerUIDLabel]) - require.Equal(t, model, request.Spec.NodeLocal.Model) - require.Equal(t, desired.plan.LPXScalingGroup, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name) - } - for _, projection := range desired.requests { - request, found := requestByModel[projection.modelProjection.Model()] - require.True(t, found) - require.Equal(t, projection.modelProjection.Digest().String(), request.Annotations[lpx.WorkloadDigestAnnotation]) - require.Equal(t, projection.modelProjection.CompilerSnapshotDigest(), request.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) - } - - for index, expected := range desired.plan.Agents { - projection := &desired.requests[index] - clique := findLPXTestClique(t, objects, expected.CliqueName) - require.Equal(t, int32(expected.Replicas), clique.Spec.Replicas) - require.Equal(t, ptr.To(int32(expected.Replicas)), clique.Spec.MinAvailable) - require.Equal(t, projection.modelProjection.Digest().String(), clique.Annotations[lpx.WorkloadDigestAnnotation]) - require.Equal(t, projection.modelProjection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) - require.Equal(t, projection.modelProjection.Model(), clique.Annotations[lpxv1alpha1.PodModelAnnotation]) - require.NotContains(t, clique.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) - require.NotContains(t, clique.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) - } - pcs := findLPXTestPodCliqueSet(t, objects) - require.Equal(t, desired.workloadDigest, pcs.Annotations[lpx.WorkloadDigestAnnotation]) - - targetRequest := requestByModel["target"] - target := targetRequest.DeepCopy() - target.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: lpxv1alpha1.RequestPhaseUnsupported, - ObservedGeneration: ptr.To(target.Generation), - Diagnostics: []lpxv1alpha1.StatusDiagnostic{{ - Code: "TargetUnsupported", Subject: "model/target", Detail: "target projection is unsupported", - }}, - } - require.NoError(t, reconciler.Update(ctx, target)) - classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.NoError(t, err) - require.Equal(t, nvidiacomv1beta1.DGDStateFailed, lpxResult(classification).State) - require.Equal(t, "LPXUnsupported", lpxResult(classification).Reason) - require.Contains(t, lpxResult(classification).Message, "TargetUnsupported") -} - -func TestLPXMaterializationRejectsExcessiveReplicas(t *testing.T) { - for _, nativeScale := range []bool{false, true} { - t.Run(fmt.Sprintf("nativeScale=%t", nativeScale), func(t *testing.T) { - t.Log("Prepare a valid workload and its owned Grove scaling group") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - - t.Log("Exceed the engine limit through either the DGD or native Grove scale") - lpx.ServingComponent(source).Replicas = ptr.To(int32(2497)) - if nativeScale { - lpx.ServingComponent(source).Replicas = nil - group.Spec.Replicas = 2497 - } - createLPXTestObjects(t, ctx, r.Client, pcs, group) - - t.Log("Reject the count before constructing per-engine request state") - selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, pcs) - require.NoError(t, err) - require.Nil(t, selected) - require.Equal(t, &lpxRejected{reason: "LPX replica count must be between 0 and 2496"}, classification) - }) - } -} - -func TestLPXImplicitReplicaObservationWaitsForMatchingPodCliqueSet(t *testing.T) { - t.Log("Render the stable Grove identities for an externally managed replica count") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = nil - base, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, base, child, source, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := findLPXTestScalingGroup(t, objects, desired.plan.LPXScalingGroup) - replacement := pcs.DeepCopy() - replacement.UID = "replacement-pcs" - - for _, test := range []struct { - name string - pcs *grovev1alpha1.PodCliqueSet - }{ - {name: "PCS has already disappeared"}, - {name: "replacement PCS does not own the old group", pcs: replacement}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Keep the old PCSG visible during background owner cleanup") - seed := []client.Object{group.DeepCopy()} - if test.pcs != nil { - seed = append(seed, test.pcs.DeepCopy()) - } - r := newLPXTestReconciler(t, registry, child.DeepCopy(), source.DeepCopy(), seed...) - - t.Log("Classify the transient owner mismatch as retirement, not reconciliation failure") - selected, classification, err := r.prepareLPXMaterializing(ctx, child, source, test.pcs) - require.NoError(t, err) - require.Nil(t, selected) - require.IsType(t, &lpxRetiring{}, classification) - }) - } -} - -func TestLPXPublishedWorkloadFailureRetirement(t *testing.T) { - for _, test := range []struct { - name string - snapshotError error - }{ - {name: "inconsistent snapshot", snapshotError: fmt.Errorf("%w: compiler metadata changed during duplicate reads", lpx.ErrBuildSnapshotInconsistent)}, - {name: "transient snapshot", snapshotError: errors.New("temporary object-store timeout")}, - {name: "render failure"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Publish a request backed by the rendered LPX workload") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.IsType(t, &lpxOpen{}, classification) - requestBefore := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - key := client.ObjectKeyFromObject(dgd) - pcsKey := client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} - pcsBefore := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, pcsKey, pcsBefore)) - - t.Log("Fail the actual snapshot dependency or break cross-role model storage") - message := "must use the Conductor model-storage mount path" - if test.snapshotError != nil { - reconciler.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: test.snapshotError} - message = test.snapshotError.Error() - } else { - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(source), source)) - conductor := source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) - for index := range conductor.PodTemplate.Spec.Containers[0].VolumeMounts { - mount := &conductor.PodTemplate.Spec.Containers[0].VolumeMounts[index] - if mount.Name == consts.ModelStorageVolumeName { - mount.MountPath = "/different-model-storage" - } - } - require.NoError(t, reconciler.Update(ctx, source)) - require.NoError(t, reconciler.Get(ctx, key, dgd)) - revision, revisionErr := dynamo.LPXInputRevision(source, "") - require.NoError(t, revisionErr) - dgd.Spec.InputRevision = revision - require.NoError(t, reconciler.Update(ctx, dgd)) - } - _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - pcs := &grovev1alpha1.PodCliqueSet{} - t.Log("Failed desired input must preserve the exact published request and PCS") - require.Equal(t, requestBefore, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, requestBefore.Name)) - require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) - require.Equal(t, pcsBefore, pcs) - - t.Log("Persist the actual actionable failure and retain its wrapped cause") - require.ErrorContains(t, err, message) - if test.snapshotError != nil { - require.ErrorIs(t, err, test.snapshotError) - require.ErrorIs(t, err, lpx.ErrBuildSnapshotAcquisition) - } - require.NoError(t, reconciler.Get(ctx, key, dgd)) - failed := meta.FindStatusCondition(dgd.Status.Conditions, "Failed") - require.NotNil(t, failed) - require.Equal(t, metav1.ConditionTrue, failed.Status) - require.Equal(t, dgd.Generation, failed.ObservedGeneration) - require.Equal(t, err.Error(), failed.Message) - }) - } -} - -func TestLPXPodCliqueSetMetadataSyncUsesSuppliedObservation(t *testing.T) { - t.Log("Render initial root metadata and seed an exact-owned PCS with native Grove controls") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Labels = map[string]string{"test.example/version": "before", "kai.scheduler/removed": "before"} - source.Spec.Annotations = maps.Clone(source.Spec.Labels) - prepare, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - desired := renderLPXTestPodCliqueSet(t, ctx, prepare, dgd, source, selected) - cached := desired.DeepCopy() - require.NoError(t, ctrl.SetControllerReference(dgd, cached, prepare.Scheme())) - cached.UID = "cached-pcs-uid" - cached.ResourceVersion = "7" - cached.Generation = 4 - hash, err := commoncontroller.GetSpecHash(cached, commoncontroller.WithPreservedListOrder()) - require.NoError(t, err) - metav1.SetMetaDataAnnotation(&cached.ObjectMeta, commoncontroller.NvidiaAnnotationHashKey, hash) - delete(cached.Annotations, commoncontroller.NvidiaAnnotationGenerationKey) - cached.Finalizers = []string{groveconstants.FinalizerPodCliqueSet} - cached.Annotations[groveconstants.AnnotationDisableManagedResourceProtection] = "true" - cached.Annotations[groveconstants.AnnotationReconcileTrigger] = "previous-trigger" - before := cached.DeepCopy() - for _, metadata := range []map[string]string{desired.Labels, desired.Annotations} { - metadata["test.example/version"] = "after" - metadata["kai.scheduler/added"] = "" - delete(metadata, "kai.scheduler/removed") - } - - t.Log("Record writes and reject extra reads of the supplied observation") - reconciler := newLPXTestReconciler(t, registry, dgd, source, cached) - base, ok := reconciler.Client.(client.WithWatch) - require.True(t, ok) - updates := 0 - reconciler.Client = interceptor.NewClient(base, interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - return errors.New("PCS synchronization must use its supplied observation") - }, - Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { - updates++ - return delegated.Update(ctx, object, opts...) - }, - }) - - t.Log("Only bookkeeping changes; root metadata additions, edits and removals are ignored") - modified, synced, err := commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, cached, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, modified) - require.Equal(t, 1, updates) - want := before.DeepCopy() - want.ResourceVersion = synced.ResourceVersion - want.Annotations[commoncontroller.NvidiaAnnotationGenerationKey] = "4" - require.Equal(t, want, synced) - require.Equal(t, before, cached, "the supplied observation must not be mutated") - - t.Log("Root metadata differences alone are a no-op once bookkeeping is current") - updates = 0 - modified, synced, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.False(t, modified) - require.Zero(t, updates) - require.Equal(t, want, synced) - - t.Log("Template metadata still updates through Spec without changing root metadata or native controls") - desired.Spec.Template.Cliques[0].Annotations = map[string]string{"test.example/template": "next"} - modified, synced, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, modified) - require.Equal(t, 1, updates) - want.Spec = desired.Spec - want.ResourceVersion = synced.ResourceVersion - want.Annotations[commoncontroller.NvidiaAnnotationHashKey], err = commoncontroller.GetSpecHash(desired, commoncontroller.WithPreservedListOrder()) - require.NoError(t, err) - want.Annotations[commoncontroller.NvidiaAnnotationGenerationKey] = "5" - require.True(t, apiequality.Semantic.DeepEqual(want, synced)) - - t.Log("A concurrent spec edit conflicts instead of being overwritten from a stale observation") - cached = synced.DeepCopy() - concurrent := synced.DeepCopy() - concurrent.Spec.Replicas++ - concurrent.Generation++ - require.NoError(t, base.Update(ctx, concurrent)) - desired.Spec.Replicas += 2 - updates = 0 - modified, synced, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, cached, desired, commoncontroller.WithPreservedListOrder(), - ) - require.True(t, apierrors.IsConflict(err), "concurrent spec change must trigger a fresh reconciliation: %v", err) - require.Nil(t, synced) - require.False(t, modified) - require.Equal(t, 1, updates) - stored := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(concurrent), stored)) - require.Equal(t, concurrent, stored) -} - -func TestLPXPodCliqueSetListOrder(t *testing.T) { - for _, reorder := range []bool{false, true} { - t.Run(fmt.Sprintf("reorder=%t", reorder), func(t *testing.T) { - t.Log("Create an LPX PCS with two ordered init containers") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - desired := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, selected) - desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers = []corev1.Container{ - {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, - } - modified, observed, err := commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, modified) - - t.Log("Model API-defaulted live fields without changing the applied generation") - observed.Generation = 1 - require.Nil(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.EnableServiceLinks) - observed.Spec.Template.Cliques[0].Spec.PodSpec.EnableServiceLinks = ptr.To(true) - require.NoError(t, reconciler.Update(ctx, observed)) - before := observed.DeepCopy() - if reorder { - slices.Reverse(desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) - } - - t.Log("Apply authored ordering changes but ignore unchanged desired specs") - modified, synced, err := commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, observed, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.Equal(t, reorder, modified) - require.Equal(t, desired.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers, synced.Spec.Template.Cliques[0].Spec.PodSpec.InitContainers) - require.Equal(t, before, observed, "synchronization must not mutate its observation") - if !reorder { - require.Equal(t, before, synced, "API defaults must not cause a spec rewrite") - } - - t.Log("The next reconciliation is a no-op") - modified, _, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, synced, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.False(t, modified) - }) - } -} - -func TestLPXPublicationFencePrecedesPodCliqueSetWrite(t *testing.T) { - t.Log("Prepare a render for publication and observe writes across the authoritative fence") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, selected := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - desired := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, selected) - base, ok := reconciler.Client.(client.WithWatch) - require.True(t, ok) - writes, lists := 0, 0 - reconciler.Client = interceptor.NewClient(base, interceptor.Funcs{ - Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { - writes++ - return delegated.Create(ctx, object, opts...) - }, - Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { - writes++ - return delegated.Update(ctx, object, opts...) - }, - }) - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - fenceErr := errors.New("fence list failed") - failFence := true - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - lists++ - if failFence { - return fenceErr - } - return delegated.List(ctx, list, opts...) - }, - }) - - t.Log("A failed publication observation prevents initial creation") - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, nil) - require.ErrorIs(t, err, fenceErr) - require.Nil(t, classification) - require.Nil(t, currents) - require.Zero(t, writes) - require.Equal(t, 1, lists) - require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(desired), &grovev1alpha1.PodCliqueSet{}))) - - t.Log("Recover the observation and create the exact child-owned PCS") - failFence = false - writes, lists = 0, 0 - currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, selected, nil) - require.NoError(t, err) - require.Nil(t, classification) - require.Empty(t, currents) - modified, synced, err := commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, modified) - require.Equal(t, 1, lists) - require.Equal(t, 1, writes) - require.True(t, apiequality.Semantic.DeepEqual(desired.Spec, synced.Spec)) - require.True(t, metav1.IsControlledBy(synced, dgd)) - require.Equal(t, desired.Labels, synced.Labels) - require.Equal(t, desired.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) - hash, hashErr := commoncontroller.GetSpecHash(desired, commoncontroller.WithPreservedListOrder()) - require.NoError(t, hashErr) - require.Equal(t, hash, synced.Annotations[commoncontroller.NvidiaAnnotationHashKey]) - require.Equal(t, "1", synced.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) - stored := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(desired), stored)) - require.Equal(t, stored.ResourceVersion, synced.ResourceVersion) - - t.Log("A stale missing observation does not adopt an already-created PCS") - modified, _, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, nil, desired.DeepCopy(), commoncontroller.WithPreservedListOrder(), - ) - require.True(t, apierrors.IsAlreadyExists(err), "create race must trigger a fresh reconciliation: %v", err) - require.False(t, modified) - - t.Log("An in-place update does not require listing every scheduler request") - live := synced - desired = live.DeepCopy() - desired.ResourceVersion, desired.UID = "", "" - desired.Spec.Replicas++ - desired.Annotations[lpx.WorkloadDigestAnnotation] = "next" - failFence = true - writes, lists = 0, 0 - modified, synced, err = commoncontroller.SyncObservedResource( - ctx, reconciler, dgd, live, desired, commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.True(t, modified) - require.Equal(t, live.UID, synced.UID) - require.Equal(t, 1, writes) - require.Zero(t, lists) - require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(live), stored)) - require.True(t, apiequality.Semantic.DeepEqual(desired.Spec, stored.Spec)) - require.Equal(t, live.Annotations[lpx.WorkloadDigestAnnotation], stored.Annotations[lpx.WorkloadDigestAnnotation]) -} - -func TestLPXPublicationFenceReusesObservedPodCliqueSetForRequestOwnership(t *testing.T) { - t.Log("Publish two requests owned by the already-observed PodCliqueSet") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(source).Replicas = ptr.To(int32(2)) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - objects := lpxMaterializedObjects(t, r, child, source, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - pcsGets := 0 - r.apiReader = interceptor.NewClient(r.apiReader.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - pcsGets++ - } - return delegated.Get(ctx, key, object, opts...) - }, - }) - - t.Log("Classify every LPR against that observation without per-request API reads") - currents, classification, err := r.reconcileLPXPublicationFence(ctx, child, desired, pcs) - require.NoError(t, err) - require.Nil(t, classification) - require.Len(t, currents, len(desired.requests)) - require.Zero(t, pcsGets) -} - -func TestLPXPublicationWaitsForRequestGarbageCollectionBeforeRecreatingPodCliqueSet(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) - t.Log("An unrelated live PCS also has a request with this deployment's copied UID label") - foreignPCS := pcs.DeepCopy() - foreignPCS.Name, foreignPCS.UID, foreignPCS.ResourceVersion = lpxTestOtherName, "foreign-pcs", "" - foreignPCS.OwnerReferences = nil - require.NoError(t, reconciler.Create(ctx, foreignPCS)) - foreign := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - foreign.Name, foreign.UID, foreign.ResourceVersion = "foreign-request", "foreign-request-uid", "" - foreign.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(foreignPCS, grovev1alpha1.SchemeGroupVersion.WithKind(lpxPodCliqueSetKind))} - require.NoError(t, reconciler.Create(ctx, foreign)) - require.NoError(t, reconciler.Delete(ctx, pcs)) - - t.Log("Keep the replacement PCS fenced while its predecessor's request remains persisted") - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) - require.NoError(t, err) - require.Len(t, currents, 1) - require.IsType(t, &lpxRetiring{}, classification) - - t.Log("Permit idempotent replacement after garbage collection removes the previous request") - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - require.NoError(t, reconciler.Delete(ctx, request)) - currents, classification, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) - require.NoError(t, err) - require.Empty(t, currents) - require.Nil(t, classification) - require.Equal(t, foreign, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, foreign.Name)) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(foreignPCS), &grovev1alpha1.PodCliqueSet{})) -} - -func TestLPXScaleDownDeletesOnlyStaleRequests(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Components[0].Replicas = ptr.To[int32](3) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - for _, projection := range desired.requests { - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, projection.requestName) - request.Status = &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(request.Generation), - } - require.NoError(t, reconciler.Update(ctx, request)) - } - pcs := findLPXTestPodCliqueSet(t, objects) - pcsUID := pcs.UID - retainedUID := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName).UID - - t.Log("Scale from three replicas to one while every request remains pending") - scaledDown := withLPXTestReplicas(t, desired, 1) - _, classification, err := reconciler.reconcileLPXPublicationFence( - ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), - ) - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - - t.Log("Delete every stale LPR concurrently without replacing the shared PCS") - currents, classification, err := reconciler.reconcileLPXPublicationFence( - ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), - ) - require.NoError(t, err) - require.Nil(t, classification) - require.Len(t, currents, 1) - for _, projection := range desired.requests[1:] { - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, projection.requestName) - } - retained := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - require.Equal(t, retainedUID, retained.UID) - storedPCS := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) - require.Equal(t, pcsUID, storedPCS.UID) -} - -func TestLPXScaleUpWaitsForTerminatingScaledDownRequest(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Components[0].Replicas = ptr.To[int32](2) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - pcsUID := pcs.UID - removed := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[1].requestName) - removedUID := removed.UID - removed.Finalizers = []string{"scheduler.example/cleanup"} - require.NoError(t, reconciler.Update(ctx, removed)) - - t.Log("Scale down and begin direct request deletion") - scaledDown := withLPXTestReplicas(t, desired, 1) - _, classification, err := reconciler.reconcileLPXPublicationFence( - ctx, dgd, scaledDown, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, scaledDown), - ) - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - terminating := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) - require.False(t, terminating.DeletionTimestamp.IsZero()) - - t.Log("Scale back up while the same request name is still terminating") - _, classification, err = reconciler.reconcileLPXPublicationFence( - ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), - ) - require.NoError(t, err) - require.IsType(t, &lpxRetiring{}, classification) - storedPCS := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) - require.Equal(t, pcsUID, storedPCS.UID) - - t.Log("Recreate only the request after scheduler cleanup completes") - terminating.Finalizers = nil - require.NoError(t, reconciler.Update(ctx, terminating)) - requireLPXRequestNotFound(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) - classification = publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.IsType(t, &lpxOpen{}, classification) - recreated := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, removed.Name) - require.NotEqual(t, removedUID, recreated.UID) - require.Equal(t, pcsUID, metav1.GetControllerOf(recreated).UID) - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) - require.Equal(t, pcsUID, storedPCS.UID) -} - -func TestLPXElapsedDeadlineDoesNotFenceBoundGeneration(t *testing.T) { - t.Log("Publish a request that reaches Bound before its persisted deadline is observed again") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Status = deadlineTestRequest(dgd, deadlineTestPCS(dgd, metav1.GetControllerOf(request).UID), request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status - require.NoError(t, reconciler.Update(ctx, request)) - - t.Log("Terminal scheduler disposition wins the deadline race and permits ordinary observation") - currents, classification, err := reconciler.reconcileLPXPublicationFence( - ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), - ) - require.NoError(t, err) - require.Len(t, currents, 1) - require.Nil(t, classification) - classification, wake, err := reconciler.reconcileLPXRequestDeadlines( - ctx, dgd, source, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), desired.requests, - ) - require.NoError(t, err) - require.Nil(t, classification) - require.True(t, wake.IsZero()) -} - -func TestLPXRecordedDeadlineFailureFencesLateBoundRequest(t *testing.T) { - t.Log("Record expiry before the scheduler reports a terminal phase") - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Spec.Scheduling = deadlineTestScheduling() - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Status = deadlineTestRequest(dgd, deadlineTestPCS(dgd, metav1.GetControllerOf(request).UID), request.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhaseBound).Status - require.NoError(t, reconciler.Update(ctx, request)) - dgd.Status.Conditions = []metav1.Condition{{ - Type: lpxSchedulingFailedCondition, Status: metav1.ConditionTrue, - Reason: lpxSchedulingDeadlineExceededReason, ObservedGeneration: dgd.Generation, - }} - - t.Log("Keep the recorded failure authoritative for the current generation") - currents, classification, err := reconciler.reconcileLPXPublicationFence( - ctx, dgd, desired, observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired), - ) - require.NoError(t, err) - require.Len(t, currents, 1) - require.IsType(t, &lpxDeadlineExceeded{}, classification) -} - -func TestLPXPreflightPreservesRequestsDuringNativeGroveSync(t *testing.T) { - ctx := t.Context() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - rendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, dgd, source, desired) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - published := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - - pcsKey := types.NamespacedName{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName} - drifted := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - require.NotEmpty(t, drifted.Spec.Template.Cliques) - drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName = "externally-mutated" - require.NoError(t, reconciler.Update(ctx, drifted)) - require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - - _, _, err := commoncontroller.SyncObservedResource( - ctx, - reconciler, - dgd, - drifted, - rendered, - commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) - - t.Log("Correct the spec in place without stopping the existing PCS") - require.NoError(t, reconciler.Get(ctx, pcsKey, drifted)) - require.Equal(t, int32(1), drifted.Spec.Replicas) - require.Equal(t, types.UID("pcs-uid"), drifted.UID) - require.NotEqual(t, "externally-mutated", drifted.Spec.Template.Cliques[0].Spec.PodSpec.PriorityClassName) - - foreignPCS := drifted.DeepCopy() - foreignPCS.ResourceVersion = "" - require.NotEmpty(t, foreignPCS.OwnerReferences) - foreignPCS.OwnerReferences[0].UID = "foreign-owner" - foreignReconciler := newLPXTestReconciler(t, registry, dgd, source, foreignPCS) - _, err = foreignReconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil) - require.EqualError(t, err, fmt.Sprintf("refusing to inspect PodCliqueSet %q without the exact LPXGraphDeployment controller owner", foreignPCS.Name)) - - t.Log("An explicitly stopped PCS still cannot publish another request") - drifted.Spec.Replicas = 0 - require.NoError(t, reconciler.Update(ctx, drifted)) - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - pcsGets := 0 - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - pcsGets++ - } else { - return errors.New("retired Grove observation must not read dependencies") - } - return delegated.Get(ctx, key, object, opts...) - }, - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return errors.New("retired Grove observation must not read dependencies") - }, - }) - classification, err := reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, dgd, desired, nil) - require.NoError(t, err) - closed := requireLPXClosed(t, classification) - require.Equal(t, "The selected Grove PodCliqueSet has 0 replicas, want 1", closed.incomplete) - require.Equal(t, 1, pcsGets) - reconciler.apiReader = reader - - t.Log("Restore the explicit hold in place without rotating the existing request") - _, _, err = commoncontroller.SyncObservedResource( - ctx, - reconciler, - dgd, - drifted, - rendered, - commoncontroller.WithPreservedListOrder(), - ) - require.NoError(t, err) - - recreated := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, pcsKey, recreated)) - require.Equal(t, drifted.UID, recreated.UID) - for _, clique := range recreated.Spec.Template.Cliques { - require.NotEqual(t, "externally-mutated", clique.Spec.PodSpec.PriorityClassName) - } - require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, published.Name)) -} - -func TestGroveSpecSyncPreservesWorkloadAfterTopologyRejection(t *testing.T) { - t.Log("Keep an existing two-partition engine while preparing a three-partition successor") - ctx := t.Context() - oldDGD, oldSource, oldRegistry := newLPXTestDGD(t, lpx.PipelineSingle) - oldPrepareReconciler := newLPXTestReconciler(t, oldRegistry, oldDGD, oldSource) - oldDesired, rejected := requirePreparedLPX(t, oldPrepareReconciler, ctx, oldDGD, oldSource) - require.Nil(t, rejected) - - oldPCS := findLPXTestPodCliqueSet(t, lpxMaterializedObjects(t, oldPrepareReconciler, oldDGD, oldSource, oldDesired)) - published := deadlineTestRequest(oldDGD, oldPCS, oldDesired.requests[0].requestName, time.Now(), lpxv1alpha1.RequestPhaseBound) - - const successorBuildID = "build-v2-successor" - successorRegistry := newLPXTestRegistryWithPartitionsAndMode( - t, - successorBuildID, - []int{7, 8, 9}, - manifestcapnpv2.CompilationMode_lpuOnly, - ) - successorDGD := oldDGD.DeepCopy() - successorSource := oldSource.DeepCopy() - successorDGD.Generation++ - successorSource.Spec.Components[0].LPX.BuildID = successorBuildID - reconciler := newLPXTestReconciler(t, successorRegistry, successorDGD, successorSource, oldPCS, published) - successorDesired, rejected := requirePreparedLPX(t, reconciler, ctx, successorDGD, successorSource) - require.Nil(t, rejected) - - t.Log("The successor changes Agent minAvailable, which pinned Grove rejects as immutable") - pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := types.NamespacedName{ - Namespace: successorDGD.Namespace, - Name: successorDesired.plan.PodCliqueSetName, - } - require.NoError(t, reconciler.Get(ctx, pcsKey, pcs)) - published = getLPXRequest(t, ctx, reconciler.Client, successorDGD.Namespace, published.Name) - successorRendered := renderLPXTestPodCliqueSet(t, ctx, reconciler, successorDGD, successorSource, successorDesired) - for index, clique := range successorRendered.Spec.Template.Cliques { - if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { - require.Equal(t, ptr.To(int32(4)), pcs.Spec.Template.Cliques[index].Spec.MinAvailable) - require.Equal(t, ptr.To(int32(6)), clique.Spec.MinAvailable) - } - } - - t.Log("Return the admission error without replacing or scaling down the existing workload") - admissionErr := apierrors.NewForbidden(grovev1alpha1.SchemeGroupVersion.WithResource("podcliquesets").GroupResource(), pcs.Name, - errors.New("spec.template.cliques.spec.minAvailable: field is immutable")) - reconciler.Client = interceptor.NewClient(reconciler.Client.(client.WithWatch), interceptor.Funcs{ - Update: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.UpdateOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - return admissionErr - } - return delegated.Update(ctx, object, opts...) - }, - }) - changed, synced, err := commoncontroller.SyncObservedResource( - ctx, - reconciler, - successorDGD, - pcs, - successorRendered, - commoncontroller.WithPreservedListOrder(), - ) - require.ErrorIs(t, err, admissionErr) - require.Nil(t, synced) - require.False(t, changed) - - t.Log("The complete stored PCS and published request remain unchanged") - stored := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, pcsKey, stored)) - require.Equal(t, pcs, stored) - require.Equal(t, published, getLPXRequest(t, ctx, reconciler.Client, successorDGD.Namespace, published.Name)) -} - -func TestLPXRetirementOverridesRuntimeReadiness(t *testing.T) { - t.Log("Retirement must replace both successful and pending runtime results") - retiring := &lpxRetiring{retirementReason: "test retirement"} - _, isError := any(retiring).(error) - require.False(t, isError, "expected retirement must stay outside the operational error channel") - for _, base := range []reconcileOutcome{{State: nvidiacomv1beta1.DGDStateSuccessful}, {State: nvidiacomv1beta1.DGDStatePending}} { - require.Equal(t, lpxResult(retiring), overlayLPXResult(base, retiring)) - } -} - -func TestLPXRestartPreservesBoundProofAndFinalization(t *testing.T) { - t.Log("Prepare the LPU-only runtime without adding scheduler configuration to its source") - ctx := context.Background() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - delete(source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - require.Nil(t, source.Spec.Scheduling) - require.NotContains(t, source.Annotations, consts.KubeAnnotationLPXSchedulerBackend) - - t.Log("Materialize the selected runtime") - objects := lpxMaterializedObjects(t, reconciler, dgd, source, desired) - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - - t.Log("Keep an otherwise-ready runtime gated before the exact request is Bound") - classification := publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.IsType(t, &lpxOpen{}, classification) - runtimeReady := reconcileOutcome{ - State: nvidiacomv1beta1.DGDStateSuccessful, - Reason: "all_resources_are_ready", - Message: "All resources are ready", - } - require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) - - t.Log("Recreate the controller and derive the same open attempt from API objects") - reconciler = &graphReconciler{ - Client: reconciler.Client, - recorder: reconciler.recorder, - apiReader: reconciler.apiReader, - runtimeConfig: reconciler.runtimeConfig, - modelRegistry: registry, - Config: reconciler.Config, - DockerSecretRetriever: reconciler.DockerSecretRetriever, - } - rederived, rejected := requirePreparedLPX(t, reconciler, ctx, dgd, source) - require.Nil(t, rejected) - desired = rederived - classification, err := reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - - t.Log("Observe the exact current scheduler receipt after the controller restart") - request := getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Generation = 7 - request.Status = newLPXNodeLocalBoundStatus(7, 3, "sha256:bound-plan") - require.NoError(t, reconciler.Update(ctx, request)) - - classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.NoError(t, err) - bound, ok := classification.(*lpxBound) - require.True(t, ok) - - t.Log("Return an exact node-local Bound receipt to the base Grove runtime result") - require.Equal(t, runtimeReady, overlayLPXResult(runtimeReady, bound)) - runtimePending := reconcileOutcome{ - State: nvidiacomv1beta1.DGDStatePending, - Reason: "some_resources_are_not_ready", - Message: "Some resources are not ready", - } - require.Equal(t, runtimePending, overlayLPXResult(runtimePending, bound)) - - t.Log("Keep runtime readiness gated when the Bound receipt belongs to an older generation") - request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Status.ObservedGeneration = ptr.To(int64(6)) - require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - require.Equal(t, "LPXPublished", overlayLPXResult(runtimeReady, classification).Reason) - - t.Log("Stale failure diagnostics are not attributed to the current request generation") - request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - request.Status.Phase = lpxv1alpha1.RequestPhaseUnsupported - request.Status.Diagnostics = []lpxv1alpha1.StatusDiagnostic{{ - Code: "StaleUnsupported", Subject: "request/old", Detail: "belongs to an older generation", - }} - require.NoError(t, reconciler.Update(ctx, request)) - classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.NoError(t, err) - require.IsType(t, &lpxOpen{}, classification) - require.NotContains(t, lpxResult(classification).Message, "StaleUnsupported") - - t.Log("The owner chain is sufficient for deletion without Dynamo finalizers") - request = getLPXRequest(t, ctx, reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.plan.PodCliqueSetName}, pcs)) - require.True(t, metav1.IsControlledBy(pcs, dgd)) - require.True(t, metav1.IsControlledBy(request, pcs)) - require.Empty(t, request.Finalizers) - require.Empty(t, dgd.Finalizers) -} - -func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { - for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { - t.Run(string(pipeline), func(t *testing.T) { - ctx := t.Context() - deployment, source, registry := newLPXTestDGD(t, pipeline) - if pipeline == lpx.PipelineLPX { - source.Spec.Components[0].Replicas = ptr.To(int32(1)) - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - } - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, source) - objects := lpxMaterializedObjects(t, reconciler, deployment, source, desired) - for _, object := range objects { - switch live := object.(type) { - case *grovev1alpha1.PodCliqueScalingGroup: - live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 - live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 - case *grovev1alpha1.PodClique: - live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas - live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas - require.Equal(t, desired.workload.LPXComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) - } - } - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - pcs := findLPXTestPodCliqueSet(t, objects) - rootReads := 0 - base, ok := reconciler.Client.(client.WithWatch) - require.True(t, ok) - reader := interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - rootReads++ - } - return reader.Get(ctx, key, object, opts...) - }}) - observe := func() dynamo.GroveReadiness { - ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, source, deployment, pcs) - require.NoError(t, err) - require.Len(t, ready.ComponentStatuses, 1) - require.Zero(t, rootReads, "reuse the synchronized PCS observation") - return ready - } - t.Log("Report one complete engine, never separate GPU or Agent component capacity") - ready := observe() - require.True(t, ready.Ready) - status := ready.ComponentStatuses[desired.workload.LPXComponentName()] - require.Equal(t, nvidiacomv1beta1.ComponentKindPodCliqueScalingGroup, status.ComponentKind) - require.Equal(t, []string{desired.plan.LPXScalingGroup}, status.ComponentNames) - require.Equal(t, int32(1), status.Replicas) - require.Equal(t, int32(1), status.UpdatedReplicas) - require.Equal(t, ptr.To(int32(1)), status.AvailableReplicas) - require.Equal(t, ptr.To(int32(1)), status.ScheduledReplicas) - if desired.plan.CyborgClique != "" { - t.Log("A stale GPU-role scale cannot satisfy complete-engine readiness") - gpu := findLPXTestClique(t, objects, desired.plan.CyborgClique).DeepCopy() - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(gpu), gpu)) - gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 1, 1, 1 - gpu.Status.ReadyReplicas, gpu.Status.ScheduledReplicas = 1, 1 - require.NoError(t, reconciler.Update(ctx, gpu)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, nvidiacomv1beta1.DGDReadyReasonUpdating, ready.Classification) - t.Log("Partial GPU readiness contributes zero complete engine replicas") - gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 2, 2, 2 - gpu.Status.ScheduledReplicas = 2 - require.NoError(t, reconciler.Update(ctx, gpu)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, nvidiacomv1beta1.DGDReadyReasonPodsNotReady, ready.Classification) - require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.LPXComponentName()].AvailableReplicas) - gpu.Status.ReadyReplicas = 2 - require.NoError(t, reconciler.Update(ctx, gpu)) - require.True(t, observe().Ready) - } - t.Log("Reject stale complete-engine capacity even when every existing role is ready") - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.LPXScalingGroup}, group)) - group.Spec.Replicas = 2 - require.NoError(t, reconciler.Update(ctx, group)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, nvidiacomv1beta1.DGDReadyReasonUpdating, ready.Classification) - }) - } -} - -func TestLPXClassifiesCurrentSchedulerReceipts(t *testing.T) { - tests := []struct { - name string - phase lpxv1alpha1.RequestPhase - wantState nvidiacomv1beta1.DGDState - wantReason string - wantMessage string - }{ - {name: "pending", phase: lpxv1alpha1.RequestPhasePending, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXSchedulerPending"}, - {name: "no fit", phase: lpxv1alpha1.RequestPhaseNoFit, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXNoFit"}, - {name: "unsupported", phase: lpxv1alpha1.RequestPhaseUnsupported, wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXUnsupported"}, - {name: "planned", phase: lpxv1alpha1.RequestPhasePlanned, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXPlanned"}, - {name: "reserving", phase: lpxv1alpha1.RequestPhaseReserving, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXReserving"}, - {name: "binding", phase: lpxv1alpha1.RequestPhaseBinding, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXBinding"}, - {name: "degraded", phase: lpxv1alpha1.RequestPhaseDegraded, wantState: nvidiacomv1beta1.DGDStatePending, wantReason: "LPXDegraded"}, - { - name: "unknown phase", phase: lpxv1alpha1.RequestPhase("future"), - wantState: nvidiacomv1beta1.DGDStateFailed, wantReason: "LPXSchedulerStatusInvalid", wantMessage: `unknown phase "future"`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Record a scheduler status observation for the current generation") - request := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{Generation: 7}, - Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, - Status: &lpxv1alpha1.LPUPipelineRequestStatus{ - Phase: test.phase, - ObservedGeneration: ptr.To(int64(7)), - Diagnostics: []lpxv1alpha1.StatusDiagnostic{{ - Code: "ExactDiagnostic", Subject: "requests/current", Detail: "the exact scheduler detail", - }}, - }, - } - - t.Log("Classify the current receipt and preserve its phase, journal and diagnostic") - classification := classifyPublishedLPX(request) - observed, ok := classification.(*lpxSchedulerObserved) - require.True(t, ok) - require.Equal(t, test.phase, observed.Phase) - result := lpxResult(observed) - require.Equal(t, test.wantState, result.State) - require.Equal(t, test.wantReason, result.Reason) - require.Contains(t, result.Message, "ExactDiagnostic") - require.Contains(t, result.Message, "requests/current") - require.Contains(t, result.Message, "the exact scheduler detail") - require.Equal(t, 1, strings.Count(result.Message, "LPX diagnostics:")) - require.Equal(t, 1, strings.Count(result.Message, "the exact scheduler detail")) - if test.wantMessage != "" { - require.Contains(t, result.Message, test.wantMessage) - } - - t.Log("The first failure outranks open requests; otherwise the first non-Bound receipt wins") - before := request.DeepCopy() - later := request.DeepCopy() - later.Status.Diagnostics[0].Detail = "a later scheduler receipt" - requests := []lpxModelMaterializing{{requestName: "open"}, {requestName: "observed"}, {requestName: "later"}} - currents := map[string]*lpxv1alpha1.LPUPipelineRequest{"open": {}, "observed": request, "later": later} - var want lpxClassification = &lpxOpen{} - if test.wantState == nvidiacomv1beta1.DGDStateFailed { - want = observed - } - require.Equal(t, want, classifyLPXCurrentRequests(requests, currents)) - slices.Reverse(requests) - require.Equal(t, (*lpxSchedulerObserved)(later.Status), classifyLPXCurrentRequests(requests, currents)) - require.Equal(t, before, request) - }) - } -} - -func TestLPXSchedulerDiagnosticMessageRespectsConditionLimit(t *testing.T) { - diagnostics := []lpxv1alpha1.StatusDiagnostic{{ - Code: strings.Repeat("c", 253), - Subject: strings.Repeat("s", 253), - Detail: strings.Repeat("界", 4096/len("界")), - }} - for range 31 { - diagnostics = append(diagnostics, diagnostics[0]) - } - - message := lpxMessageWithDiagnostics("LPX scheduler diagnostic summary", diagnostics) - - require.LessOrEqual(t, len(message), maxDGDConditionMessageSize) - require.True(t, utf8.ValidString(message)) - require.True(t, strings.HasSuffix(message, "…")) -} - -func TestLPXRequestIdentityAndRetirementFence(t *testing.T) { - type digestInput struct { - namespace string - name string - uid types.UID - model string - } - digest := func(input digestInput, replicaIndex int32) string { - return digestAttemptKey( - input.namespace, - input.name, - input.uid, - input.model, - replicaIndex, - ) - } - base := digestInput{ - namespace: "ns", name: "dgd", uid: "uid-a", model: "default", - } - baseDigest := digest(base, 0) - require.NotEmpty(t, baseDigest) - for name, mutate := range map[string]func(*digestInput){ - "namespace": func(key *digestInput) { key.namespace = "other-ns" }, - "name": func(key *digestInput) { key.name = "other-dgd" }, - "uid": func(key *digestInput) { key.uid = "uid-b" }, - "model": func(key *digestInput) { key.model = "model-b" }, - } { - t.Run(name, func(t *testing.T) { - changed := base - mutate(&changed) - require.NotEqual(t, baseDigest, digest(changed, 0)) - mutate(&changed) - require.NotEmpty(t, digest(changed, 0)) - }) - } - require.Equal(t, baseDigest, digest(base, 0), "change-then-revert must return to the same complete tuple digest") - require.NotEqual(t, baseDigest, digest(base, 1)) - require.LessOrEqual(t, len(lpxRequestName("a-very-long-but-readable-dynamo-graph-deployment-name", baseDigest)), 63) - ctx := context.Background() - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, dgd, source) - - t.Log("Fence the exact PCS owner and immutable deployment annotations") - projection := &desired.requests[0] - identity := &lpxGroveIdentity{pcs: deadlineTestPCS(dgd, "pcs-uid"), plan: desired.plan} - current := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{UID: "request-uid", Annotations: lpxRequestAnnotations(dgd, projection), OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(identity.pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}}, - Spec: projection.modelProjection.RequestSpec(identity.plan, projection.agentCliqueName), - } - require.NoError(t, validateCurrentLPXRequest(dgd, projection, identity, current)) - changed := *identity - changed.pcs = identity.pcs.DeepCopy() - changed.pcs.UID = "replacement-pcs-uid" - require.ErrorContains(t, validateCurrentLPXRequest(dgd, projection, &changed, current), "exact PCS owner") - - createLPXTestObjects(t, ctx, reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - publishSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - - dgd.Generation++ - pcs := observedLPXTestPodCliqueSet(t, ctx, reconciler, dgd, desired) - classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, pcs, nil) - require.NoError(t, err) - require.Nil(t, classification) - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: dgd.Namespace, Name: desired.requests[0].requestName}, &lpxv1alpha1.LPUPipelineRequest{})) - - classification, _, err = reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, pcs, nil) - require.NoError(t, err) - require.Nil(t, classification) -} - -func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { - for _, scenario := range []struct{ name, message string }{ - {"LPX", "LPX integration is disabled"}, - {"Grove", "Grove is disabled"}, - } { - t.Run(scenario.name, func(t *testing.T) { - t.Log("Publish a complete workload, discovery service and scheduling request") - ctx := t.Context() - child, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - source.Spec.Scheduling = deadlineTestScheduling() - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, source) - createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, source, selected)...) - key := client.ObjectKeyFromObject(child) - for range 4 { - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - } - require.NoError(t, r.Get(ctx, key, child)) - before := []client.ObjectList{ - &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}, - &corev1.ServiceList{}, &lpxv1alpha1.LPUPipelineRequestList{}, - } - for _, list := range before { - require.NoError(t, r.List(ctx, list)) - require.Positive(t, meta.LenList(list)) - } - - t.Log("Disable the provider before an input edit and an expired deadline can retire the workload") - if scenario.name == "LPX" { - r.runtimeConfig.Gate.LPX = false - } else { - r.runtimeConfig.Gate.Grove = false - } - r.modelRegistry = nil - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(source), source)) - originalSpec := source.Spec.DeepCopy() - source.Spec.Components[0].LPX.BuildID = "edited-while-disabled" - require.NoError(t, r.Update(ctx, source)) - for range 2 { - result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.Zero(t, result) - } - require.NoError(t, r.Get(ctx, key, child)) - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") - require.NotNil(t, failed) - require.Equal(t, metav1.ConditionTrue, failed.Status) - require.Equal(t, "LPXUnavailable", failed.Reason) - require.Equal(t, scenario.message, failed.Message) - require.Equal(t, child.Generation, failed.ObservedGeneration) - require.Equal(t, metav1.ConditionFalse, meta.FindStatusCondition(child.Status.Conditions, "Ready").Status) - for _, list := range before { - current := list.DeepCopyObject().(client.ObjectList) - require.NoError(t, r.List(ctx, current)) - require.Equal(t, list, current) - } - - t.Log("Re-enable unchanged intent and resume the original PCS and request identities") - source.Spec = *originalSpec - require.NoError(t, r.Update(ctx, source)) - r.runtimeConfig.Gate.LPX, r.runtimeConfig.Gate.Grove = true, true - r.modelRegistry = registry - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, key, child)) - require.NotEqual(t, "LPXUnavailable", meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) - for _, list := range before { - current := list.DeepCopyObject().(client.ObjectList) - require.NoError(t, r.List(ctx, current)) - require.Equal(t, list, current) - } - - t.Log("Deletion remains available while disabled and needs no controller finalizer") - if scenario.name == "LPX" { - r.runtimeConfig.Gate.LPX = false - } else { - r.runtimeConfig.Gate.Grove = false - } - r.modelRegistry = nil - require.Empty(t, child.Finalizers) - require.NoError(t, r.Delete(ctx, child)) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, key, &nvidiacomv1alpha1.LPXGraphDeployment{}))) - }) - } -} - -func TestLPXFinalizationAfterRequestAPIRemoval(t *testing.T) { - resource := lpxv1alpha1.GroupVersion.WithResource("lpupipelinerequests").GroupResource() - for _, scenario := range []struct { - name string - listErr error - }{ - {"removed CRD", apierrors.NewNotFound(resource, "")}, - {"forbidden", apierrors.NewForbidden(resource, "", errors.New("denied"))}, - {"transport error", errors.New("connection reset")}, - } { - t.Run(scenario.name, func(t *testing.T) { - t.Log("Keep a legacy controller finalizer while the request API cannot be read") - ctx := t.Context() - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - child := newLPXTestDeployment(t, source) - child.Finalizers = []string{legacyLPXGraphDeploymentFinalizer} - r := newLPXTestReconciler(t, nil, child, source) - r.apiReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return scenario.listErr - }, - }) - - t.Log("Ordinary reconciliation still fails when requests cannot be listed") - _, err := r.listOwnedLPXRequests(ctx, child, nil) - require.ErrorIs(t, err, scenario.listErr) - - t.Log("Release the legacy finalizer without reading requests; owner garbage collection owns cleanup") - key := client.ObjectKeyFromObject(child) - require.NoError(t, r.Delete(ctx, child)) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, key, child))) - }) - } -} - -func TestLPXControllerPreservesPublishedWorkWhenProviderEditIsInvalid(t *testing.T) { - t.Log("Reject an incompatible provider edit without deleting the existing engine") - dgd, source, registry := newLPXTestDGD(t, lpx.PipelineSingle) - reconciler, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), dgd, source) - createLPXTestObjects(t, t.Context(), reconciler.Client, lpxMaterializedObjects(t, reconciler, dgd, source, desired)...) - publishSelectedLPXForTest(t, t.Context(), reconciler, dgd, desired) - - require.NoError(t, reconciler.Get(t.Context(), client.ObjectKeyFromObject(source), source)) - source.Annotations[consts.KubeAnnotationWorkloadProvider] = consts.WorkloadProviderComponent - require.NoError(t, reconciler.Update(t.Context(), source)) - _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) - require.ErrorContains(t, err, "LPX requires the Grove workload provider") - request := getLPXRequest(t, t.Context(), reconciler.Client, dgd.Namespace, desired.requests[0].requestName) - require.Nil(t, request.DeletionTimestamp) -} - -func TestLPXPublicationFenceRejectsForeignExactNameCollisionFromAuthoritativeList(t *testing.T) { - t.Log("Give foreign requests the desired names in the opposite lexical order") - ctx := t.Context() - source := newLPXTestSource(lpx.PipelineSingle, "build-v2") - dgd := newLPXTestDeployment(t, source) - foreign := &lpxv1alpha1.LPUPipelineRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: "z-foreign-request", - Namespace: dgd.Namespace, - UID: "foreign-collision-uid", - Labels: map[string]string{lpxOwnerUIDLabel: string(dgd.UID)}, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), - Kind: lpxPodCliqueSetKind, - Name: lpxTestOtherName, - UID: "other-uid", - Controller: ptr.To(true), - }}, - }, - } - second := foreign.DeepCopy() - second.Name, second.UID = "a-foreign-request", "second-foreign-collision-uid" - second.OwnerReferences[0].APIVersion = nvidiacomv1beta1.GroupVersion.String() - second.OwnerReferences[0].Kind = nvidiacomv1beta1.DynamoGraphDeploymentGVK.Kind - desired := &lpxMaterializing{requests: []lpxModelMaterializing{ - {requestName: foreign.Name}, {requestName: second.Name}, - }} - reconciler := newLPXTestReconciler(t, nil, dgd, source) - - t.Log("Complete an empty preflight fence before foreign requests occupy the desired names") - currents, retiring, err := reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) - require.NoError(t, err) - require.Empty(t, currents) - require.Nil(t, retiring) - for _, request := range []*lpxv1alpha1.LPUPipelineRequest{foreign, second} { - require.NoError(t, reconciler.Create(ctx, request)) - } - - t.Log("Exclude foreign requests from owned publications and the known-intent retirement fence") - requests, err := reconciler.listOwnedLPXRequests(ctx, dgd, nil) - require.NoError(t, err) - require.Empty(t, requests) - classification, _, err := reconciler.reconcileLPXKnownIntentFence(ctx, dgd, source, nil, nil) - require.NoError(t, err) - require.Nil(t, classification) - - t.Log("Observe labeled exact-name collisions with one scoped authoritative list and no per-request Gets") - reader, ok := reconciler.apiReader.(client.WithWatch) - require.True(t, ok) - requestGets := 0 - requestListCalls := make([]client.ListOptions, 0) - reconciler.apiReader = interceptor.NewClient(reader, interceptor.Funcs{ - Get: func( - ctx context.Context, - delegated client.WithWatch, - key client.ObjectKey, - object client.Object, - opts ...client.GetOption, - ) error { - if _, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { - requestGets++ - } - return delegated.Get(ctx, key, object, opts...) - }, - List: func( - ctx context.Context, - delegated client.WithWatch, - list client.ObjectList, - opts ...client.ListOption, - ) error { - if _, ok := list.(*lpxv1alpha1.LPUPipelineRequestList); ok { - requestListCalls = append(requestListCalls, *(&client.ListOptions{}).ApplyOptions(opts)) - } - return delegated.List(ctx, list, opts...) - }, - }) - - currents, retiring, err = reconciler.reconcileLPXPublicationFence(ctx, dgd, desired, nil) - require.EqualError(t, err, fmt.Sprintf("LPX request name %q is occupied by a foreign owner", foreign.Name)) - require.Nil(t, currents) - require.Nil(t, retiring) - require.Zero(t, requestGets, "exact-name collision detection must not issue one request Get per desired projection") - require.Len(t, requestListCalls, 1) - require.Zero(t, requestListCalls[0].Limit) - require.Empty(t, requestListCalls[0].Continue) - require.Equal(t, dgd.Namespace, requestListCalls[0].Namespace) - require.Equal(t, lpxOwnerUIDLabel+"="+string(dgd.UID), requestListCalls[0].LabelSelector.String()) - - t.Log("Refresh publication state and refuse the foreign requests created after the empty preflight") - classification, err = reconcileSelectedLPXForTest(t, ctx, reconciler, dgd, desired) - require.ErrorContains(t, err, "occupied by a foreign owner") - require.Nil(t, classification) - require.Len(t, requestListCalls, 2, "final publication must refresh the authoritative request snapshot") - for _, request := range []*lpxv1alpha1.LPUPipelineRequest{foreign, second} { - stored := &lpxv1alpha1.LPUPipelineRequest{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(request), stored)) - require.Equal(t, request.UID, stored.UID) - require.Equal(t, request.OwnerReferences, stored.OwnerReferences) - require.True(t, stored.DeletionTimestamp.IsZero()) - } - allRequests := &lpxv1alpha1.LPUPipelineRequestList{} - require.NoError(t, reconciler.List(ctx, allRequests, client.InNamespace(dgd.Namespace))) - require.Len(t, allRequests.Items, 2, "the fresh final snapshot must not publish alongside the collisions") -} - -func requireLPXClosed(t *testing.T, classification lpxClassification) *lpxClosed { - t.Helper() - closed, ok := classification.(*lpxClosed) - require.True(t, ok, "classification=%T", classification) - return closed -} - -func newLPXNodeLocalBoundStatus(generation, _ int64, _ lpxv1alpha1.PlanDigest) *lpxv1alpha1.LPUPipelineRequestStatus { - return &lpxv1alpha1.LPUPipelineRequestStatus{Phase: lpxv1alpha1.RequestPhaseBound, ObservedGeneration: &generation} -} - -func findLPXTestClique(t *testing.T, objects []client.Object, name string) *grovev1alpha1.PodClique { - t.Helper() - for _, object := range objects { - if clique, ok := object.(*grovev1alpha1.PodClique); ok && clique.Name == name { - return clique - } - } - t.Fatalf("PodClique %q not found", name) - return nil -} - -func findLPXTestScalingGroup(t *testing.T, objects []client.Object, name string) *grovev1alpha1.PodCliqueScalingGroup { - t.Helper() - for _, object := range objects { - if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && group.Name == name { - return group - } - } - t.Fatalf("PodCliqueScalingGroup %q not found", name) - return nil -} - -func findLPXTestPodCliqueSet(t *testing.T, objects []client.Object) *grovev1alpha1.PodCliqueSet { - t.Helper() - for _, object := range objects { - if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - return pcs - } - } - t.Fatal("PodCliqueSet not found") - return nil -} - -func renderLPXTestPodCliqueSet( - t *testing.T, ctx context.Context, reconciler *graphReconciler, - dgd *nvidiacomv1alpha1.LPXGraphDeployment, source *nvidiacomv1beta1.DynamoGraphDeployment, desired *lpxMaterializing, -) *grovev1alpha1.PodCliqueSet { - t.Helper() - rendered, _, err := renderPodCliqueSet(ctx, source, reconciler.Config, reconciler.runtimeConfig, - reconciler.DockerSecretRetriever, desired.workload, desired.plan, dgd) - require.NoError(t, err) - return rendered -} - -func newLPXTestDGD(t *testing.T, pipeline lpx.Pipeline) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { - t.Helper() - compilationMode := manifestcapnpv2.CompilationMode_lpuOnly - if pipeline == lpx.PipelineLPX { - compilationMode = manifestcapnpv2.CompilationMode_lpx - } - const buildID = "build-v2" - registry := newLPXTestRegistryWithPartitionsAndMode(t, buildID, []int{7, 8}, compilationMode) - source := newLPXTestSource(pipeline, buildID) - return newLPXTestDeployment(t, source), source, registry -} - -func newLPXTestDeployment(t *testing.T, source *nvidiacomv1beta1.DynamoGraphDeployment) *nvidiacomv1alpha1.LPXGraphDeployment { - t.Helper() - revision, err := dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - return &nvidiacomv1alpha1.LPXGraphDeployment{ - TypeMeta: metav1.TypeMeta{APIVersion: nvidiacomv1alpha1.GroupVersion.String(), Kind: "LPXGraphDeployment"}, - ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: types.UID("lpx-" + string(source.UID)), Generation: 1, - Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - }, - Spec: nvidiacomv1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, - } -} - -func newLPXTestSource(pipeline lpx.Pipeline, buildID string) *nvidiacomv1beta1.DynamoGraphDeployment { - one := int32(1) - source := &nvidiacomv1beta1.DynamoGraphDeployment{ - TypeMeta: metav1.TypeMeta{APIVersion: nvidiacomv1beta1.GroupVersion.String(), Kind: "DynamoGraphDeployment"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "selected-source", Namespace: "test", UID: "source-uid", Generation: 3, - Annotations: map[string]string{consts.KubeAnnotationLPXSchedulerBackend: consts.LPXSchedulerBackend}, - }, - Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ - Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ - ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, Replicas: &one, - LPX: &nvidiacomv1beta1.LPXConfig{BuildID: buildID}, - Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, { - Name: nvidiacomv1beta1.ComponentRoleLPXAgent, - PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: consts.MainContainerName, Image: "lpu-runtime", - VolumeMounts: []corev1.VolumeMount{ - {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, - {Name: "config", MountPath: "/configs"}, - {Name: "host-dev", MountPath: "/dev"}, - {Name: "host-sys", MountPath: "/sys"}, - {Name: "ssh-secret", MountPath: "/ssh-pk", ReadOnly: true}, - {Name: "hugepages", MountPath: "/dev/hugepages"}, - {Name: "single-v2-ssh-key", MountPath: "/tmp/dynamo-lpu-ssh"}, - }, - }}, - Volumes: []corev1.Volume{ - {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathDirectory)}}}, - {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys", Type: ptr.To(corev1.HostPathDirectory)}}}, - {Name: "hugepages", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumHugePages}}}, - {Name: "ssh-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ssh-secret", DefaultMode: ptr.To[int32](0644)}}}, - {Name: "single-v2-ssh-key", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - }, - }}, - }}, - }}, - }, - } - // Each role authors its own startup; the conductor cannot inherit the Agent command. - agentTemplate := source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent).PodTemplate - agentTemplate.Spec.Containers[0].Command = []string{"/bin/quasar-entrypoint"} - conductorTemplate := agentTemplate.DeepCopy() - conductorTemplate.Spec.Containers[0].Command = []string{"/bin/nova"} - source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor).PodTemplate = conductorTemplate - if pipeline == lpx.PipelineLPX { - *source.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) = nvidiacomv1beta1.ComponentRoleSpec{ - Name: nvidiacomv1beta1.ComponentRoleLPXConductor, - Replicas: &one, - PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - ResourceClaims: []corev1.PodResourceClaim{{Name: "gpu", ResourceClaimTemplateName: ptr.To("gpu")}}, - Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "cyborg-runtime", - Resources: corev1.ResourceRequirements{Claims: []corev1.ResourceClaim{{Name: "gpu"}}}, - VolumeMounts: []corev1.VolumeMount{ - {Name: "config", MountPath: "/configs"}, - {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, - }, - }}, - Volumes: []corev1.Volume{ - {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - }, - }}, - } - } - return source -} - -func newLPXSpecDecodeTestDGD(t *testing.T) (*nvidiacomv1alpha1.LPXGraphDeployment, *nvidiacomv1beta1.DynamoGraphDeployment, lpx.ModelRegistry) { - t.Helper() - source := newLPXSpecDecodeTestSource() - root := t.TempDir() - writeLPXTestBuild(t, root, "draft-build", []int{7, 8}, manifestcapnpv2.CompilationMode_lpuOnly) - writeLPXTestBuild(t, root, "target-build", []int{9, 10}, manifestcapnpv2.CompilationMode_lpuOnly) - registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() - registry, err := lpx.NewModelRegistry(registryURL, nil) - require.NoError(t, err) - return newLPXTestDeployment(t, source), source, registry -} - -func newLPXSpecDecodeTestSource() *nvidiacomv1beta1.DynamoGraphDeployment { - source := newLPXTestSource(lpx.PipelineSingle, "target-build") - target := &source.Spec.Components[0] - draft := target.DeepCopy() - draft.ComponentName, draft.LPX.BuildID, draft.Replicas = "draft", "draft-build", ptr.To(int32(2)) - draft.Roles = []nvidiacomv1beta1.ComponentRoleSpec{*draft.ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent)} - source.Spec.Components = append(source.Spec.Components, *draft) - return source -} - -func newLPXTestRegistryWithPartitionsAndMode( - t *testing.T, - buildID string, - partitionIDs []int, - compilationMode manifestcapnpv2.CompilationMode, -) lpx.ModelRegistry { - t.Helper() - root := t.TempDir() - writeLPXTestBuild(t, root, buildID, partitionIDs, compilationMode) - registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() - registry, err := lpx.NewModelRegistry(registryURL, nil) - require.NoError(t, err) - return registry -} - -func writeLPXTestBuild( - t *testing.T, - root string, - buildID string, - partitionIDs []int, - compilationMode manifestcapnpv2.CompilationMode, -) { - t.Helper() - require.NotEmpty(t, partitionIDs) - buildDir := filepath.Join(root, buildID) - require.NoError(t, os.Mkdir(buildDir, 0o700)) - - t.Log("Encode the V2 manifest header and the tokenizer consumed by rendered runtimes") - message, manifest := newTestGraphManifest(t) - - t.Log("Retain the opaque compiler provenance without the removed V1 publication contract") - populateTestGraphBuild(t, manifest, buildID) - - t.Log("Describe the same one-batch runtime with explicit V2 runtime I/O and prop-sync evidence") - deployment, program := newTestGraphProgram(t, manifest, compilationMode, uint32(len(partitionIDs)*2), 8192) - program.SetNumKvCaches(1) - program.SetNumBatchSplitDivisions(1) - chains, err := deployment.NewSelectedPropSyncChains(1) - require.NoError(t, err) - selectedIDs, err := chains.At(0).NewPartitionIds(int32(len(partitionIDs))) - require.NoError(t, err) - for index, partitionID := range partitionIDs { - selectedIDs.Set(index, uint32(partitionID)) - } - - t.Log("Package the LPU partitions and the hybrid CUDA marker in the flat V2 artifact inventory") - artifacts, err := manifest.NewArtifacts() - require.NoError(t, err) - partitionCount := len(partitionIDs) - if compilationMode == manifestcapnpv2.CompilationMode_lpx { - partitionCount++ - } - partitions, err := artifacts.NewPartitions(int32(partitionCount)) - require.NoError(t, err) - for index, partitionID := range partitionIDs { - partition := partitions.At(index) - ref, err := partition.NewPartition() - require.NoError(t, err) - ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) - ref.SetPartitionId(uint32(partitionID)) - detail, err := partition.Detail().NewLpu() - require.NoError(t, err) - require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", partitionID))) - require.NoError(t, detail.SetTopology("URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA")) - detail.SetNumChips(16) - detail.SetDevicesPerNode(8) - } - if compilationMode == manifestcapnpv2.CompilationMode_lpx { - partition := partitions.At(len(partitionIDs)) - ref, err := partition.NewPartition() - require.NoError(t, err) - ref.SetDeviceType(manifestcapnpv2.DeviceType_cuda) - ref.SetPartitionId(uint32(len(partitionIDs))) - _, err = partition.Detail().NewCuda() - require.NoError(t, err) - } - - t.Log("Publish the required V2 compiler manifest") - payload, err := message.Marshal() - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), payload, 0o600)) -} - -func newLPXTestReconciler( - t *testing.T, - registry lpx.ModelRegistry, - dgd *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - objects ...client.Object, -) *graphReconciler { - t.Helper() - revision, err := dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - dgd.Spec.InputRevision = revision - if dgd.ResourceVersion == "" { - dgd.ResourceVersion = "1" - } - scheme := newLPXTestScheme(t) - seed := append([]client.Object{dgd.DeepCopy(), source.DeepCopy()}, objects...) - base := fake.NewClientBuilder(). - WithScheme(scheme). - WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences). - WithStatusSubresource(&nvidiacomv1beta1.DynamoGraphDeployment{}, &nvidiacomv1alpha1.LPXGraphDeployment{}). - WithObjects(seed...). - Build() - nextLPRUID := 0 - wrapped := interceptor.NewClient(base, interceptor.Funcs{ - // The fake client does not implement scale for Grove custom resources. - SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && subresource == "scale" { - options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) - scale := options.SubResourceBody.(*autoscalingv1.Scale) - group.ResourceVersion = scale.ResourceVersion - group.Spec.Replicas = scale.Spec.Replicas - return delegated.Update(ctx, group) - } - return delegated.SubResource(subresource).Update(ctx, object, opts...) - }, - Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { - if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { - if request.UID == "" { - nextLPRUID++ - request.UID = types.UID(fmt.Sprintf("lpr-%s-%d", request.Name, nextLPRUID)) - } - if request.Generation == 0 { - request.Generation = 1 - } - if request.CreationTimestamp.IsZero() { - request.CreationTimestamp = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) - } - } - return delegated.Create(ctx, object, opts...) - }, - }) - recorder := events.NewFakeRecorder(100) - config := &configv1alpha1.OperatorConfiguration{ - LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - } - runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, DRA: true, LPX: true}} - return &graphReconciler{ - Client: wrapped, - recorder: recorder, - apiReader: wrapped, - runtimeConfig: runtimeConfig, - modelRegistry: registry, - Config: config, - } -} - -func lpxMaterializedObjects( - t *testing.T, - reconciler *graphReconciler, - dgd *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - desired *lpxMaterializing, -) []client.Object { - t.Helper() - t.Log("Render LPX intent once and emulate the API-server and Grove observations") - pcs := renderLPXTestPodCliqueSet(t, t.Context(), reconciler, dgd, source, desired) - pcs.TypeMeta = metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet"} - pcs.UID, pcs.Generation = "pcs-uid", 1 - pcs.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(dgd, nvidiacomv1alpha1.LPXGraphDeploymentGVK)} - const generationHash = "pcs-generation-hash" - pcs.Status = grovev1alpha1.PodCliqueSetStatus{ - ObservedGeneration: ptr.To(pcs.Generation), CurrentGenerationHash: ptr.To(generationHash), - } - - // Grove materializes the rendered scaling-group configuration once. - groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].DeepCopy() - group := &grovev1alpha1.PodCliqueScalingGroup{ - TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, - ObjectMeta: metav1.ObjectMeta{ - Name: desired.plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: "lpu-group-uid", - Generation: 1, Labels: grovecommon.GetDefaultLabelsForPodCliqueSetManagedResources(pcs.Name), Annotations: groupTemplate.Annotations, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: ptr.Deref(groupTemplate.Replicas, 1), MinAvailable: groupTemplate.MinAvailable, - CliqueNames: groupTemplate.CliqueNames, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), - }, - } - group.Labels[grovecommon.LabelPartOfKey] = pcs.Name - group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" - objects := []client.Object{pcs, group} - - // Each Grove replica owns independent cliques. - for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { - parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} - for _, template := range pcs.Spec.Template.Cliques { - rendered := template.DeepCopy() - name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) - clique := &grovev1alpha1.PodClique{ - TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, - Labels: rendered.Labels, Annotations: rendered.Annotations, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, - }, - Spec: rendered.Spec, - Status: grovev1alpha1.PodCliqueStatus{ - ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), - CurrentPodTemplateHash: ptr.To(rendered.Name + "-pod-template-hash"), - }, - } - clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name - clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name - clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) - for index, dependency := range clique.Spec.StartsAfter { - clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) - } - objects = append(objects, clique) - } - } - return objects -} - -func withLPXTestReplicas(t *testing.T, desired *lpxMaterializing, replicas int32) *lpxMaterializing { - t.Helper() - require.NotNil(t, desired) - require.NotNil(t, desired.plan) - require.Positive(t, desired.plan.Replicas) - require.GreaterOrEqual(t, replicas, int32(0)) - require.LessOrEqual(t, replicas, desired.plan.Replicas) - require.Zero(t, len(desired.requests)%int(desired.plan.Replicas)) - requestsPerReplica := len(desired.requests) / int(desired.plan.Replicas) - next := *desired - next.requests = slices.Clone(desired.requests[:int(replicas)*requestsPerReplica]) - plan := *desired.plan - plan.Replicas = replicas - next.plan = &plan - return &next -} - -func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.Client, objects ...client.Object) { - t.Helper() - for _, object := range objects { - require.NoError(t, kubeClient.Create(ctx, object.DeepCopyObject().(client.Object))) - } -} - -func observedLPXTestPodCliqueSet( - t *testing.T, - ctx context.Context, - reconciler *graphReconciler, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, -) *grovev1alpha1.PodCliqueSet { - t.Helper() - if desired.plan == nil { - return nil - } - pcs := &grovev1alpha1.PodCliqueSet{} - err := reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.PodCliqueSetName}, pcs) - if apierrors.IsNotFound(err) { - return nil - } - require.NoError(t, err) - return pcs -} - -func reconcileSelectedLPXForTest( - t *testing.T, - ctx context.Context, - reconciler *graphReconciler, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, -) (lpxClassification, error) { - t.Helper() - pcs := observedLPXTestPodCliqueSet(t, ctx, reconciler, deployment, desired) - currents, classification, err := reconciler.reconcileLPXPublicationFence(ctx, deployment, desired, pcs) - if classification != nil || err != nil { - return classification, err - } - return reconciler.reconcileSelectedLPXFromCurrentRequests(ctx, deployment, desired, currents) -} - -func publishSelectedLPXForTest( - t *testing.T, - ctx context.Context, - reconciler *graphReconciler, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - desired *lpxMaterializing, -) lpxClassification { - t.Helper() - classification, err := reconcileSelectedLPXForTest(t, ctx, reconciler, deployment, desired) - require.NoError(t, err) - return classification -} - -func getLPXRequest(t *testing.T, ctx context.Context, kubeClient client.Reader, namespace, name string) *lpxv1alpha1.LPUPipelineRequest { - t.Helper() - request := &lpxv1alpha1.LPUPipelineRequest{} - require.NoError(t, kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request)) - return request -} - -func requireLPXRequestNotFound( - t *testing.T, - ctx context.Context, - kubeClient client.Reader, - namespace string, - name string, -) { - t.Helper() - request := &lpxv1alpha1.LPUPipelineRequest{} - err := kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request) - require.True(t, apierrors.IsNotFound(err), "expected LPX request %s/%s to be absent, got %v", namespace, name, err) -} diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index aac2cc609f7d..f26dd6d8b54e 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "net/url" + "slices" "sort" "time" @@ -20,12 +21,12 @@ import ( modelpb "github.com/ai-dynamo/modelexpress/modelexpress_client/go/gen/modelexpress/model" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) const ( modelDownloadRequeueAfter = 5 * time.Second - modelDownloadPendingReason string = "model_download_in_progress" modelDownloadPendingMessage string = "Waiting for model downloads to complete" modelDownloadRefreshInterval = 24 * time.Hour @@ -53,68 +54,93 @@ func newLPXModelRegistry(config *configv1alpha1.OperatorConfiguration) (lpx.Mode return registry, nil } -// reconcileModelDownloads checks the nonnil DGD's remote builds and updates the -// owned, nonnil child's download status without mutating the DGD. forceCheck -// bypasses the refresh interval until an immutable LPX selection exists. +// reconcileModelDownloads gates startup on downloaded builds. A successful +// observation remains usable during periodic refreshes of a Ready deployment. +// deployment and dgd are non-nil. func (r *graphReconciler) reconcileModelDownloads( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, - forceCheck bool, -) (bool, error) { - current := deployment.Status.ObservedGeneration == deployment.Generation - previous := deployment.Status.ModelDownload - if !forceCheck && current && previous != nil && previous.LastCheckedAt != nil && - time.Since(previous.LastCheckedAt.Time) < modelDownloadRefreshInterval { - return true, nil +) (ctrl.Result, error) { + var ( + lastCheckedAt *metav1.Time + existingBuilds []string + recheck bool + ) + + if modelDownload := deployment.Status.ModelDownload; modelDownload != nil { + ready := meta.IsStatusConditionTrue(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) + lastCheckedAt = modelDownload.LastCheckedAt + recheck = ready && deployment.Status.ObservedGeneration == deployment.Generation && lastCheckedAt != nil + + if lastCheckedAt != nil && time.Since(lastCheckedAt.Time) < modelDownloadRefreshInterval { + existingBuilds = modelDownload.Builds + } } - running := !forceCheck && current && meta.IsStatusConditionTrue(deployment.Status.Conditions, "Ready") - checkCtx, cancel := context.WithTimeout(ctx, modelDownloadCheckTimeout) - defer cancel() - downloaded, ready, err := ensureModelsDownloaded(checkCtx, dgd, r.modelRegistry) - checkSucceeded := ready && err == nil - // Retain the running child's last observation when refresh is incomplete. - if !running || checkSucceeded { + downloaded, ready, err := ensureModelsDownloaded(ctx, dgd, r.modelRegistry, existingBuilds) + if err != nil || !ready { + if recheck { + if err != nil { + log.FromContext(ctx).Error(err, "Unable to refresh model downloads") + } + return ctrl.Result{}, nil + } + deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: downloaded} - } else if previous == nil { - deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{} - } - if checkSucceeded { - checkedAt := metav1.Now() - deployment.Status.ModelDownload.LastCheckedAt = &checkedAt - } - if !running { - return ready, err + if err != nil { + return ctrl.Result{}, err + } + + setReadyCondition(deployment, v1beta1.DGDStatePending, modelDownloadPendingMessage) + return ctrl.Result{RequeueAfter: modelDownloadRequeueAfter}, nil } - if err != nil { - log.FromContext(ctx).Error(err, "failed to refresh model downloads", "dgd", dgd.Name) + + if len(existingBuilds) == 0 { + lastCheckedAt = new(metav1.Now()) } - return true, nil + + deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: downloaded, LastCheckedAt: lastCheckedAt} + return ctrl.Result{}, nil } func ensureModelsDownloaded( ctx context.Context, dgd *v1beta1.DynamoGraphDeployment, registry lpx.ModelRegistry, + existingBuilds []string, ) ([]string, bool, error) { builds, err := collectBuilds(dgd, registry) if err != nil { return nil, false, err } - downloaded := make([]string, 0, len(builds)) - var downloadErrors []error - for i, buildURL := range builds { + if len(builds) == 0 { + return nil, true, nil + } + + var ( + downloaded = make([]string, 0, len(builds)) + downloadErrors []error + ) + + ctx, cancel := context.WithTimeout(ctx, modelDownloadCheckTimeout) + defer cancel() + + perBuildTimeout := modelDownloadCheckTimeout / time.Duration(len(builds)) + + for _, buildURL := range builds { build := buildURL.String() - buildCtx := ctx - cancel := func() {} - if deadline, ok := ctx.Deadline(); ok && i < len(builds)-1 { - perBuildTimeout := time.Until(deadline) / time.Duration(len(builds)-i) - buildCtx, cancel = context.WithTimeout(ctx, perBuildTimeout) + + if slices.Contains(existingBuilds, build) { + downloaded = append(downloaded, build) + continue } - buildDownloaded, err := registry.EnsureDownloaded(buildCtx, buildURL) + + ctx, cancel := context.WithTimeout(ctx, perBuildTimeout) + buildDownloaded, err := registry.EnsureDownloaded(ctx, buildURL) cancel() + if err != nil { downloadErrors = append(downloadErrors, fmt.Errorf("ensure model %q is downloaded: %w", build, err)) continue diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index 16f661fddc33..fc2743570ea6 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -14,57 +14,35 @@ import ( "time" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" ) const ( modelDownloadTestBuildID = "gs://bucket/model/build" modelDownloadTestSecondBuildID = "gs://bucket/model/second-build" modelDownloadTestLocalBuildID = "file:///models/local-build" - modelDownloadTestDGDName = "test-dgd" - modelDownloadTestNamespace = "default" ) func TestEnsureModelsDownloaded(t *testing.T) { - t.Log("Use the real registry to check missing Model Express configuration") - withoutModelExpress, err := lpx.NewModelRegistry("", nil) - require.NoError(t, err) - - t.Log("Own a mixed graph with reversed build order and a previously cached source") + t.Log("Own a mixed graph with reversed build order") pending := newModelDownloadDGD(modelDownloadTestSecondBuildID, modelDownloadTestBuildID) pending.Spec.Components = append([]v1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "worker", ComponentType: v1beta1.ComponentTypeWorker, }}, pending.Spec.Components...) - cached := newModelDownloadDGD(modelDownloadTestBuildID, modelDownloadTestSecondBuildID) - cached.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{modelDownloadTestBuildID}}, - } tests := []struct { name string dgd *v1beta1.DynamoGraphDeployment registry lpx.ModelRegistry + existing []string wantReady bool wantErr string wantCalls []string wantBuilds []string }{ - { - name: "returns Model Express client error for remote builds when Model Express is not configured", - dgd: newModelDownloadDGD(modelDownloadTestBuildID), - registry: withoutModelExpress, - wantErr: "Model Express client is required for GCS model downloads", - }, { name: "local build does not call Model Express", dgd: newModelDownloadDGD(modelDownloadTestLocalBuildID), @@ -80,14 +58,15 @@ func TestEnsureModelsDownloaded(t *testing.T) { wantBuilds: []string{modelDownloadTestBuildID}, }, { - name: "checks every remote build regardless of cached status", - dgd: cached, + name: "checks only uncached builds and drops unselected cached builds", + dgd: newModelDownloadDGD(modelDownloadTestBuildID, modelDownloadTestSecondBuildID), + existing: []string{modelDownloadTestBuildID, "gs://bucket/removed/build"}, registry: newModelDownloadRegistry(t, map[string]bool{ modelDownloadTestBuildID: true, modelDownloadTestSecondBuildID: true, }, nil), wantReady: true, - wantCalls: []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, + wantCalls: []string{modelDownloadTestSecondBuildID}, wantBuilds: []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, }, { @@ -110,8 +89,8 @@ func TestEnsureModelsDownloaded(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Log("Check every selected remote build without acquiring its snapshot") - builds, ready, err := ensureModelsDownloaded(t.Context(), tt.dgd, tt.registry) + t.Log("Check uncached selected builds without acquiring their compiler snapshots") + builds, ready, err := ensureModelsDownloaded(t.Context(), tt.dgd, tt.registry, tt.existing) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) } else { @@ -136,22 +115,22 @@ func TestEnsureModelsDownloadedSharesDeadlineAcrossBuilds(t *testing.T) { ) ctx, cancel := context.WithTimeout(t.Context(), time.Hour) defer cancel() + started := time.Now() t.Log("Continue checking later builds and retain their completed progress") - builds, ready, err := ensureModelsDownloaded(ctx, dgd, registry) + builds, result, err := ensureModelsDownloaded(ctx, dgd, registry, nil) require.ErrorIs(t, err, context.DeadlineExceeded) require.ErrorContains(t, err, "download failed") - require.False(t, ready) + require.Zero(t, result) require.Equal(t, []string{modelDownloadTestBuildID, modelDownloadTestSecondBuildID}, registry.calls) require.Equal(t, []string{modelDownloadTestSecondBuildID}, builds) require.Zero(t, registry.acquireBuildSnapshotCalls) - t.Log("Reserve time for the second build without extending the parent's budget") - deadline, _ := ctx.Deadline() + t.Log("Each download receives a bounded share of the total check budget") require.Len(t, registry.deadlines, 2) - require.False(t, registry.deadlines[0].IsZero()) - require.True(t, registry.deadlines[0].Before(registry.deadlines[1])) - require.Equal(t, deadline, registry.deadlines[1]) + for _, deadline := range registry.deadlines { + require.WithinDuration(t, started.Add(modelDownloadCheckTimeout/2), deadline, time.Second) + } } func TestRunningLPXModelDownloadRefresh(t *testing.T) { @@ -190,7 +169,7 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Log("Refresh a running workload while preserving its downloaded builds") dgd := newModelDownloadDGD(modelDownloadTestBuildID) - child := newLPXTestDeployment(t, dgd) + child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}} child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: &metav1.Time{Time: tt.checkedAt}, } @@ -203,9 +182,9 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { lpx := &graphReconciler{modelRegistry: registry} for range 2 { - ready, err := lpx.reconcileModelDownloads(t.Context(), child, dgd, false) + result, err := lpx.reconcileModelDownloads(t.Context(), child, dgd) require.NoError(t, err) - require.True(t, ready) + require.Zero(t, result) } if len(registry.calls) != tt.wantCalls { t.Fatalf("ModelExpress calls = %d, want %d", len(registry.calls), tt.wantCalls) @@ -225,93 +204,73 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { } } -func TestInitialLPXModelDownloadChecksHaveDeadline(t *testing.T) { - for _, forceCheck := range []bool{false, true} { - t.Run(fmt.Sprintf("forceCheck=%t", forceCheck), func(t *testing.T) { +func TestEnsureModelsDownloadedCheckTimeout(t *testing.T) { + for _, parentTimeout := range []time.Duration{time.Hour, time.Second} { + t.Run(parentTimeout.String(), func(t *testing.T) { + t.Log("Bound the download RPC by both the check timeout and the caller's deadline") dgd := newModelDownloadDGD(modelDownloadTestBuildID) - child := newLPXTestDeployment(t, dgd) registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestBuildID: true}, nil) - lifecycle := &graphReconciler{modelRegistry: registry} started := time.Now() + ctx, cancel := context.WithTimeout(t.Context(), parentTimeout) + defer cancel() - ready, err := lifecycle.reconcileModelDownloads(t.Context(), child, dgd, forceCheck) - + _, ready, err := ensureModelsDownloaded(ctx, dgd, registry, nil) require.NoError(t, err) require.True(t, ready) require.Len(t, registry.deadlines, 1) - require.False(t, registry.deadlines[0].IsZero()) - require.WithinDuration(t, started.Add(modelDownloadCheckTimeout), registry.deadlines[0], time.Second) + require.WithinDuration(t, started.Add(min(parentTimeout, modelDownloadCheckTimeout)), registry.deadlines[0], 100*time.Millisecond) }) } } -func TestModelDownloadSpecChangeRemainsFailClosed(t *testing.T) { - t.Log("Keep the source's download status independent of the unobserved child") - source := newModelDownloadDGD(modelDownloadTestBuildID) - source.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{modelDownloadTestBuildID}}, - } - child := newLPXTestDeployment(t, source) - child.Status.ObservedGeneration = child.Generation - 1 - child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}} - child.Status.ModelDownload = source.Status.LPX.ModelDownload.DeepCopy() - child.Status.ModelDownload.LastCheckedAt = ptr.To(metav1.Now()) - registry := newModelDownloadRegistry(t, nil, map[string]error{modelDownloadTestBuildID: fmt.Errorf("ModelExpress unavailable")}) - lifecycle := &graphReconciler{modelRegistry: registry} - t.Log("Do not reuse a fresh download check for an unobserved LPX generation") - ready, err := lifecycle.reconcileModelDownloads(t.Context(), child.DeepCopy(), source, false) - require.ErrorContains(t, err, "ModelExpress unavailable") - require.False(t, ready) - t.Log("The child controller persists failed download progress without publishing a PCS") - r := newLPXTestReconciler(t, registry, child, source) - _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.ErrorContains(t, err, "ModelExpress unavailable") - updated := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), updated)) - require.NotNil(t, updated.Status.ModelDownload) - require.Empty(t, updated.Status.ModelDownload.Builds) - require.False(t, meta.IsStatusConditionTrue(updated.Status.Conditions, "Ready")) - require.Equal(t, []string{modelDownloadTestBuildID}, source.Status.LPX.ModelDownload.Builds) -} - -func TestLPXDisabledRevisionPreservesObservationFence(t *testing.T) { - t.Log("A new revision starts with old download observations and no scheduling deadline") - source := newModelDownloadDGD(modelDownloadTestSecondBuildID) - source.Generation++ - child := newLPXTestDeployment(t, source) - child.Status.ObservedGeneration = child.Generation - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ - Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: ptr.To(metav1.Now()), - } - child.Generation++ - previousGeneration := child.Status.ObservedGeneration - require.Nil(t, lpxRequestDeadlineSeconds(source)) - registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestSecondBuildID: true}, nil) - r := newLPXTestReconciler(t, registry, child, source) - r.runtimeConfig.Gate = features.Gates{LPX: true} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) +func TestModelDownloadCacheAcrossRevisions(t *testing.T) { + for _, tc := range []struct { + name string + build string + cacheAge time.Duration + observed bool + ready bool + wantCached bool + }{ + {name: "unchanged build survives an input edit", build: modelDownloadTestBuildID, wantCached: true}, + {name: "new build is checked after an input edit", build: modelDownloadTestSecondBuildID}, + {name: "expired cache is checked after an input edit", build: modelDownloadTestBuildID, cacheAge: modelDownloadRefreshInterval}, + {name: "observed but not Ready still checks new builds", build: modelDownloadTestSecondBuildID, observed: true}, + {name: "old Ready receipt cannot hide a new build failure", build: modelDownloadTestSecondBuildID, ready: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Retain a previous build's successful download while changing the desired input") + dgd := newModelDownloadDGD(tc.build) + child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}} + if tc.observed { + child.Status.ObservedGeneration = child.Generation + } + if tc.ready { + setReadyCondition(child, v1beta1.DGDStateSuccessful, "Previous input is ready") + } + checkedAt := metav1.NewTime(time.Now().Add(-tc.cacheAge)) + child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ + Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: &checkedAt, + } + registry := newModelDownloadRegistry(t, nil, map[string]error{tc.build: fmt.Errorf("ModelExpress unavailable")}) + r := &graphReconciler{modelRegistry: registry} - t.Log("Disabled reconciliation reports the new failure without marking the new revision as observed") - for range 2 { - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) - require.Equal(t, previousGeneration, child.Status.ObservedGeneration) - require.Nil(t, child.Status.ModelDownload) - require.Empty(t, registry.calls, "the early exit has not checked either build") - failed := meta.FindStatusCondition(child.Status.Conditions, "Failed") - require.Equal(t, child.Generation, failed.ObservedGeneration) - require.Equal(t, "LPXUnavailable", failed.Reason) + t.Log("Reuse only fresh matching builds; new or expired builds must be checked successfully") + result, err := r.reconcileModelDownloads(t.Context(), child, dgd) + require.Zero(t, result) + if tc.wantCached { + require.NoError(t, err) + require.Empty(t, registry.calls) + require.Equal(t, &checkedAt, child.Status.ModelDownload.LastCheckedAt) + require.Equal(t, []string{tc.build}, child.Status.ModelDownload.Builds) + } else { + require.ErrorContains(t, err, "ModelExpress unavailable") + require.Equal(t, []string{tc.build}, registry.calls) + require.Empty(t, child.Status.ModelDownload.Builds) + require.Nil(t, child.Status.ModelDownload.LastCheckedAt) + } + }) } - - t.Log("Resumed download checking cannot reuse the previous revision's cached success") - r.runtimeConfig.Gate.Grove = true - ready, err := r.reconcileModelDownloads(t.Context(), child, source, false) - require.NoError(t, err) - require.True(t, ready) - require.Equal(t, []string{modelDownloadTestSecondBuildID}, registry.calls) - require.Equal(t, []string{modelDownloadTestSecondBuildID}, child.Status.ModelDownload.Builds) - require.NotNil(t, child.Status.ModelDownload.LastCheckedAt) } type fakeModelDownloadRegistry struct { @@ -344,35 +303,14 @@ func newModelDownloadRegistry(t *testing.T, ready map[string]bool, errs map[stri return &fakeModelDownloadRegistry{ModelRegistry: registry, ready: ready, err: errs} } +// newModelDownloadDGD supplies only the component-to-build mapping consumed by download checks. func newModelDownloadDGD(buildIDs ...string) *v1beta1.DynamoGraphDeployment { - dgd := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: modelDownloadTestDGDName, - Namespace: modelDownloadTestNamespace, - Generation: 1, - UID: types.UID("test-dgd-uid"), - }, - } - // Each build belongs to a distinct public component; only the target serves. - for i, buildID := range buildIDs { - component := v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "lpx-worker", ComponentType: v1beta1.ComponentTypeLPX, Replicas: ptr.To(int32(1)), + dgd := &v1beta1.DynamoGraphDeployment{} + for index, buildID := range buildIDs { + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: fmt.Sprintf("model-%d", index), ComponentType: v1beta1.ComponentTypeLPX, LPX: &v1beta1.LPXConfig{BuildID: buildID}, - Roles: []v1beta1.ComponentRoleSpec{{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime"}}, - }}}}, - } - if i == len(buildIDs)-1 { - component.Roles = append(component.Roles, v1beta1.ComponentRoleSpec{ - Name: v1beta1.ComponentRoleLPXConductor, - PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime", Command: []string{"/bin/nova"}}}, - }}, - }) - } else { - component.ComponentName = "draft" - } - dgd.Spec.Components = append(dgd.Spec.Components, component) + }) } return dgd } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go new file mode 100644 index 000000000000..f44fd79b92d1 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "cmp" + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "hash" + "slices" + "strconv" + "strings" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + pipelineRequestModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" +) + +// getPipelineRequests returns only the non-nil PCS's requests, in stable order. +// The UID index establishes ownership; callers do not recheck it. The supported +// foreground replacement path needs no discovery of LPRs after their PCS is gone. +func (r *graphReconciler) getPipelineRequests(ctx context.Context, pcs *grovev1alpha1.PodCliqueSet) ([]lpxv1alpha1.LPUPipelineRequest, error) { + requests := &lpxv1alpha1.LPUPipelineRequestList{} + if err := r.List(ctx, requests, client.InNamespace(pcs.Namespace), client.MatchingFields{pipelineRequestPCSOwnerUIDIndex: string(pcs.UID)}); err != nil { + return nil, err + } + slices.SortFunc(requests.Items, func(a, b lpxv1alpha1.LPUPipelineRequest) int { return cmp.Compare(a.Name, b.Name) }) + return requests.Items, nil +} + +// resolvePipelineRequests combines desired intent with matching observed LPRs and +// collects missing requests in replica/model order in the same pass. An immutable +// mismatch returns nil collections and true: the caller must replace the PCS. +// Pointer inputs are non-nil; deployment has a validated DGD controller owner. +// currentRequests contains owned observations and may be empty. +// Inputs are not mutated; owner references are added only on publication. +func resolvePipelineRequests( + deployment *v1alpha1.LPXGraphDeployment, + currentRequests []lpxv1alpha1.LPUPipelineRequest, + workload *lpx.SelectedWorkload, + plan *lpx.MaterializationPlan, +) (map[string]*lpxv1alpha1.LPUPipelineRequest, []*lpxv1alpha1.LPUPipelineRequest, bool) { + // The reconcile boundary already validated the DGD owner; rendering uses that identity. + dgdOwner := metav1.GetControllerOf(deployment) + + // Index the observed requests once, preserving their scheduler-owned status. + currentRequestsByName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(currentRequests)) + for index := range currentRequests { + request := ¤tRequests[index] + currentRequestsByName[request.Name] = request + } + + // Model projections are already ordered (default, or draft0..draft7 then target). + projections := workload.ModelProjections() + requests := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(projections)*int(plan.Replicas)) + var missing []*lpxv1alpha1.LPUPipelineRequest + + for replica := range plan.Replicas { + replicaPlan := plan.ForReplica(replica) + + for index, projection := range projections { + digest := pipelineRequestIdentityDigest(deployment.Namespace, deployment.Name, deployment.UID, projection.Model(), replica) + + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: pipelineRequestName(deployment.Name, digest), Namespace: deployment.Namespace, + Labels: map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgdOwner.Name, + deploymentUIDLabel: string(deployment.UID), + }, + Annotations: map[string]string{ + lpx.DGDUIDAnnotation: string(dgdOwner.UID), + lpx.DeploymentNameAnnotation: deployment.Name, + deploymentUIDAnnotation: string(deployment.UID), + pipelineRequestModelAnnotation: projection.Model(), + lpx.WorkloadDigestAnnotation: projection.Digest().String(), + lpxv1alpha1.CompilerSnapshotDigestAnnotation: projection.CompilerSnapshotDigest(), + }, + }, + Spec: projection.RequestSpec(replicaPlan, replicaPlan.Agents[index].CliqueName), + } + + if currentRequest, exists := currentRequestsByName[request.Name]; exists { + if !pipelineRequestMatches(currentRequest, request) { + return nil, nil, true + } + + requests[request.Name] = currentRequest + } else { + requests[request.Name] = request + missing = append(missing, request) + } + } + } + + return requests, missing, false +} + +// pipelineRequestIdentityDigest identifies one model at one engine ordinal. +// InputRevision and generation are deliberately absent: there is no publication batch. +func pipelineRequestIdentityDigest( + namespace string, + name string, + uid types.UID, + model string, + replicaIndex int32, +) string { + h := sha256.New() + writeIdentityHashField(h, "namespace", namespace) + writeIdentityHashField(h, "name", name) + writeIdentityHashField(h, "uid", string(uid)) + writeIdentityHashField(h, "model", model) + if replicaIndex > 0 { + writeIdentityHashField(h, "group-replica", strconv.FormatInt(int64(replicaIndex), 10)) + } + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +func writeIdentityHashField(h hash.Hash, tag, value string) { + var size [8]byte + for _, field := range []string{tag, value} { + binary.BigEndian.PutUint64(size[:], uint64(len(field))) + _, _ = h.Write(size[:]) + _, _ = h.Write([]byte(field)) + } +} + +func pipelineRequestName(deploymentName, digest string) string { + prefix := strings.ReplaceAll(deploymentName, ".", "-") + if len(prefix) > 23 { + prefix = strings.TrimRight(prefix[:23], "-") + } + hexDigest := strings.TrimPrefix(digest, "sha256:") + return fmt.Sprintf("lpx-%s-%s", prefix, hexDigest[:32]) +} + +// pipelineRequestMatches compares immutable intent after ownership was established +// by the LPR index. Both requests are non-nil. +func pipelineRequestMatches(observed, desired *lpxv1alpha1.LPUPipelineRequest) bool { + for key, value := range desired.Annotations { + if observed.Annotations[key] != value { + return false + } + } + return apiequality.Semantic.DeepEqual(observed.Spec, desired.Spec) +} + +// reconcilePipelineRequests publishes the non-empty list of missing requests and +// waits for their watch events. deployment and pcs are non-nil; requests are in +// ordinal/model order. Each Create is independent: partial publication is safe to retry. +func (r *graphReconciler) reconcilePipelineRequests( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + pcs *grovev1alpha1.PodCliqueSet, + requests []*lpxv1alpha1.LPUPipelineRequest, +) error { + // Publish independently without adopting requests absent from the cache observation. + for _, request := range requests { + if err := controllerutil.SetControllerReference(pcs, request, r.Scheme()); err != nil { + return fmt.Errorf("create LPX request %q: %w", request.Name, err) + } + + // AlreadyExists is an observation delay, not permission to adopt an unseen request. + if err := r.Create(ctx, request); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create LPX request %q: %w", request.Name, err) + } + } + + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for published LPX requests to appear in the cache") + return nil +} + +// pipelineRequestsPendingDeletion selects removed requests and current terminating +// names that must disappear before publication. All requests belong to the observed PCS. +func pipelineRequestsPendingDeletion( + requests []lpxv1alpha1.LPUPipelineRequest, + desiredRequests map[string]*lpxv1alpha1.LPUPipelineRequest, +) []*lpxv1alpha1.LPUPipelineRequest { + var pending []*lpxv1alpha1.LPUPipelineRequest + for index := range requests { + request := &requests[index] + if _, desired := desiredRequests[request.Name]; !desired || !request.DeletionTimestamp.IsZero() { + pending = append(pending, request) + } + } + return pending +} + +// deletePipelineRequests accepts already-validated LPRs. Terminating requests +// remain pending; Grove and garbage collection own Pod cleanup. +func (r *graphReconciler) deletePipelineRequests(ctx context.Context, requests []*lpxv1alpha1.LPUPipelineRequest) error { + for _, request := range requests { + if !request.DeletionTimestamp.IsZero() { + continue + } + + if err := client.IgnoreNotFound(r.Delete(ctx, request, client.Preconditions{UID: &request.UID, ResourceVersion: &request.ResourceVersion})); err != nil { + return fmt.Errorf("delete LPUPipelineRequest %q: %w", request.Name, err) + } + } + + return nil +} diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go new file mode 100644 index 000000000000..64e4bc80b579 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -0,0 +1,257 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "fmt" + "math" + "time" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" +) + +// Keep transient errors from delaying an active scheduling deadline indefinitely. +const pipelineRequestDeadlineRetryInterval = 5 * time.Second + +// pipelineRequestDeadlineSeconds reads the optional scheduling timeout from a non-nil DGD. +func pipelineRequestDeadlineSeconds(dgd *v1beta1.DynamoGraphDeployment) *int64 { + if dgd.Spec.Scheduling == nil { + return nil + } + return dgd.Spec.Scheduling.AttemptDeadlineSeconds +} + +// pipelineRequestDeadlines examines already-selected, owned requests without I/O. +// Missing schedulingStartedAt means the scheduler has not started a cycle yet. +// A surviving LPR can return to Pending; its scheduler-supplied start resets the clock. +// Creation time and managed fields cannot identify that new scheduling cycle. +func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineRequest, seconds *int64) (expired []*lpxv1alpha1.LPUPipelineRequest, next time.Time) { + if seconds == nil { + return nil, time.Time{} + } + now := time.Now() + for _, request := range requests { + if !request.DeletionTimestamp.IsZero() || isPipelineRequestDeadlineExempt(request) { + continue + } + started, known := pipelineRequestSchedulingStartedAt(request) + if !known { + continue + } + deadline := started.Add(time.Duration(*seconds) * time.Second) + if !now.Before(deadline) { + expired = append(expired, request) + } else if next.IsZero() || deadline.Before(next) { + next = deadline + } + } + return expired, next +} + +// pipelineRequestSchedulingStartedAt requires a nonnil request and reports whether the +// scheduler published a usable start for its current scheduling cycle. +func pipelineRequestSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) (time.Time, bool) { + if request.Status == nil || request.Status.SchedulingStartedAt == nil || request.Status.SchedulingStartedAt.IsZero() { + return time.Time{}, false + } + return request.Status.SchedulingStartedAt.Time, true +} + +// isPipelineRequestDeadlineExempt recognizes current receipts outside the scheduling timer. +// This is not a readiness test: NoFit is still pending, and Bound can later return to Pending. +// request is non-nil; stale receipts cannot exempt its current generation. +func isPipelineRequestDeadlineExempt(request *lpxv1alpha1.LPUPipelineRequest) bool { + if request.Status == nil || request.Status.ObservedGeneration == nil || + *request.Status.ObservedGeneration != request.Generation { + return false + } + switch request.Status.Phase { + case lpxv1alpha1.RequestPhaseBound, lpxv1alpha1.RequestPhaseNoFit, lpxv1alpha1.RequestPhaseUnsupported: + return true + case lpxv1alpha1.RequestPhaseDegraded, lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: + committed := request.Status.Committed + return committed != nil && committed.Execution.AcceptedGeneration != nil && + *committed.Execution.AcceptedGeneration == request.Generation + default: + return false + } +} + +// requeueForPipelineRequestDeadline preserves deadline wakes and bounds error retries. +// Without a deadline, errors discard unrelated wakeups and use controller-runtime backoff. +// The caller retains the error until status persistence succeeds. +func requeueForPipelineRequestDeadline( + deadlineAt time.Time, + result ctrl.Result, + reconcileErr error, +) ctrl.Result { + if deadlineAt.IsZero() { + if reconcileErr != nil { + return ctrl.Result{} + } + return result + } + delay := time.Until(deadlineAt) + if reconcileErr != nil { + if delay <= 0 || delay > pipelineRequestDeadlineRetryInterval { + delay = pipelineRequestDeadlineRetryInterval + } + return ctrl.Result{RequeueAfter: delay} + } + if delay <= 0 { + delay = time.Nanosecond + } + if result.RequeueAfter <= 0 || delay < result.RequeueAfter { + result.RequeueAfter = delay + } + return result +} + +// reconcilePipelineRequestDeadline records expiry before cleanup and blocks republication +// for the failed generation. Call only for expired requests or a current SchedulingFailed +// condition. deployment and requests are validated observations. A nil pcsg still +// permits recording failure, but cleanup must wait for an observed scaling group. +func (r *graphReconciler) reconcilePipelineRequestDeadline( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + pcsg *grovev1alpha1.PodCliqueScalingGroup, + requests []lpxv1alpha1.LPUPipelineRequest, + expired []*lpxv1alpha1.LPUPipelineRequest, + manageReplicas bool, +) (ctrl.Result, error) { + // Persist a failure covering every expired scheduling cycle before deleting requests. + if len(expired) > 0 { + if !schedulingFailureCoversPipelineRequests(deployment, expired) { + setSchedulingFailedCondition(deployment, true) + // Status-only LPXGD updates are filtered, so explicitly observe the persisted failure. + return ctrl.Result{RequeueAfter: time.Nanosecond}, nil + } + setSchedulingFailedCondition(deployment, false) + return r.reconcileExpiredPipelineRequests(ctx, pcsg, requests, expired, manageReplicas) + } + + // A failed generation must not recreate requests removed by its deadline. + setSchedulingFailedCondition(deployment, false) + return ctrl.Result{}, nil +} + +// reconcileExpiredPipelineRequests removes only complete expired trailing replicas. +// A nil pcsg defers cleanup: the live replica count is unknown, not zero. Non-nil +// pcsg and all requests are already validated against their PCS. No PodCliques +// are deleted for interior failures. +// manageReplicas is false for omitted DGD replicas: external scaling retains capacity ownership. +func (r *graphReconciler) reconcileExpiredPipelineRequests( + ctx context.Context, + pcsg *grovev1alpha1.PodCliqueScalingGroup, + requests []lpxv1alpha1.LPUPipelineRequest, + expired []*lpxv1alpha1.LPUPipelineRequest, + manageReplicas bool, +) (ctrl.Result, error) { + // Observe capacity before deciding which ordinals are safe to remove; the PCSG watch resumes cleanup. + if pcsg == nil { + return ctrl.Result{}, nil + } + replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, pcsg.Spec.Replicas) + if err != nil || len(removed) == 0 { + return ctrl.Result{}, err + } + + // Persist the lower PCSG count before asynchronous LPR/pod cleanup starts. + if manageReplicas { + if err := r.scaleDownPodCliqueScalingGroup(ctx, pcsg, &replicas); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, r.deletePipelineRequests(ctx, removed) +} + +// expiredPipelineRequestSuffix returns complete engine replicas only when every +// expired replica belongs to one contiguous trailing suffix. It orders the +// expired requests last so transient sibling cleanup leaves durable expiry +// evidence for the next reconciliation. An interior failure blocks all +// scale-down so a healthy higher ordinal is never removed. +// For four replicas, expiry at {2,3} removes that suffix; {0,3} removes nothing. +func expiredPipelineRequestSuffix( + requests []lpxv1alpha1.LPUPipelineRequest, + expired []*lpxv1alpha1.LPUPipelineRequest, + replicas int32, +) (int32, []*lpxv1alpha1.LPUPipelineRequest, error) { + if len(expired) == 0 { + return replicas, nil, nil + } + groupName, _, err := pipelineRequestScalingGroupTarget(expired[0]) + if err != nil { + return 0, nil, err + } + failedReplicas := make(map[int64]struct{}, len(expired)) + for _, request := range expired { + name, replica, err := pipelineRequestScalingGroupTarget(request) + if err != nil { + return 0, nil, err + } + if name != groupName { + return 0, nil, fmt.Errorf("expired LPX requests target different scaling groups") + } + failedReplicas[replica] = struct{}{} + } + + // Find the maximal expired suffix, not a cutoff at the minimum expired ordinal. + targetReplicas := int64(replicas) + for targetReplicas > 0 { + if _, failed := failedReplicas[targetReplicas-1]; !failed { + break + } + targetReplicas-- + } + + // Any hole blocks all cleanup, even a separate expired trailing suffix. + for replica := range failedReplicas { + if replica < targetReplicas { + return replicas, nil, nil + } + } + + expiredNames := make(map[string]struct{}, len(expired)) + for _, request := range expired { + expiredNames[request.Name] = struct{}{} + } + siblings := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) + failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) + for index := range requests { + request := &requests[index] + name, replica, err := pipelineRequestScalingGroupTarget(request) + if err != nil { + return 0, nil, err + } + if name != groupName { + return 0, nil, fmt.Errorf("LPX requests target different scaling groups") + } + + // Include LPRs beyond the live count to finish cleanup after an earlier scale write. + if replica < targetReplicas { + continue + } + if _, expired := expiredNames[request.Name]; expired { + failed = append(failed, request) + } else { + siblings = append(siblings, request) + } + } + return int32(targetReplicas), append(siblings, failed...), nil +} + +func pipelineRequestScalingGroupTarget(request *lpxv1alpha1.LPUPipelineRequest) (string, int64, error) { + target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef + if target == nil || target.Name == "" || target.ReplicaIndex < 0 || target.ReplicaIndex >= math.MaxInt32 { + return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) + } + return target.Name, target.ReplicaIndex, nil +} diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go new file mode 100644 index 000000000000..1d4d95f68dc2 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "fmt" + "slices" + "testing" + "time" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestPipelineRequestDeadlines(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + oldStart := ptr.To(metav1.NewTime(now.Add(-time.Hour))) + newStart := ptr.To(metav1.NewTime(now.Add(time.Minute))) + laterStart := ptr.To(metav1.NewTime(now.Add(2 * time.Minute))) + for _, tc := range []struct { + name string + starts []*metav1.Time + seconds *int64 + noStatus bool + deleting bool + lastPlanRevision int64 + wantExpired []string + wantWake time.Time + }{ + {name: "disabled deadline", starts: []*metav1.Time{oldStart}}, + {name: "no requests", seconds: ptr.To(int64(30))}, + {name: "no scheduler receipt", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), noStatus: true}, + {name: "no scheduling start", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30))}, + {name: "zero scheduling start", starts: []*metav1.Time{{}}, seconds: ptr.To(int64(30))}, + {name: "expired cycle", starts: []*metav1.Time{oldStart}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}}, + {name: "deleting request", starts: []*metav1.Time{oldStart}, seconds: ptr.To(int64(30)), deleting: true}, + {name: "independent starts", starts: []*metav1.Time{laterStart, newStart}, seconds: ptr.To(int64(30)), wantWake: newStart.Add(30 * time.Second)}, + {name: "expired and active cycles", starts: []*metav1.Time{oldStart, newStart}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}, wantWake: newStart.Add(30 * time.Second)}, + {name: "surviving request starts again", starts: []*metav1.Time{newStart}, seconds: ptr.To(int64(30)), lastPlanRevision: 1, wantWake: newStart.Add(30 * time.Second)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe scheduling cycles independently of object creation time and prior plans") + requests := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(tc.starts)) + for index, start := range tc.starts { + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("request-%d", index), Generation: 1, CreationTimestamp: *oldStart}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: lpxv1alpha1.RequestPhasePending, ObservedGeneration: ptr.To(int64(1)), + SchedulingStartedAt: start, LastPlanRevision: tc.lastPlanRevision, + }, + } + if tc.noStatus { + request.Status = nil + } + if tc.deleting { + request.DeletionTimestamp = ptr.To(metav1.Now()) + } + requests[request.Name] = request + } + + t.Log("Return only expired requests and the earliest still-active deadline") + expired, wake := pipelineRequestDeadlines(requests, tc.seconds) + var names []string + for _, request := range expired { + names = append(names, request.Name) + } + require.ElementsMatch(t, tc.wantExpired, names) + require.True(t, tc.wantWake.Equal(wake), "wake = %s, want %s", wake, tc.wantWake) + }) + } +} + +func TestIsPipelineRequestDeadlineExempt(t *testing.T) { + for _, tc := range []struct { + name string + phase lpxv1alpha1.RequestPhase + observed *int64 + accepted *int64 + noStatus bool + want bool + }{ + {name: "no receipt", noStatus: true}, + {name: "unobserved Bound", phase: lpxv1alpha1.RequestPhaseBound}, + {name: "stale Bound", phase: lpxv1alpha1.RequestPhaseBound, observed: ptr.To(int64(1))}, + {name: "current Bound", phase: lpxv1alpha1.RequestPhaseBound, observed: ptr.To(int64(2)), want: true}, + {name: "NoFit", phase: lpxv1alpha1.RequestPhaseNoFit, observed: ptr.To(int64(2)), want: true}, + {name: "Unsupported", phase: lpxv1alpha1.RequestPhaseUnsupported, observed: ptr.To(int64(2)), want: true}, + {name: "Pending", phase: lpxv1alpha1.RequestPhasePending, observed: ptr.To(int64(2)), accepted: ptr.To(int64(2))}, + {name: "Planned", phase: lpxv1alpha1.RequestPhasePlanned, observed: ptr.To(int64(2))}, + {name: "Reserving", phase: lpxv1alpha1.RequestPhaseReserving, observed: ptr.To(int64(2))}, + {name: "Binding", phase: lpxv1alpha1.RequestPhaseBinding, observed: ptr.To(int64(2))}, + {name: "Degraded without execution", phase: lpxv1alpha1.RequestPhaseDegraded, observed: ptr.To(int64(2))}, + {name: "Degraded stale execution", phase: lpxv1alpha1.RequestPhaseDegraded, observed: ptr.To(int64(2)), accepted: ptr.To(int64(1))}, + {name: "Degraded accepted execution", phase: lpxv1alpha1.RequestPhaseDegraded, observed: ptr.To(int64(2)), accepted: ptr.To(int64(2)), want: true}, + {name: "Releasing without execution", phase: lpxv1alpha1.RequestPhaseReleasing, observed: ptr.To(int64(2))}, + {name: "Releasing accepted execution", phase: lpxv1alpha1.RequestPhaseReleasing, observed: ptr.To(int64(2)), accepted: ptr.To(int64(2)), want: true}, + {name: "Released without execution", phase: lpxv1alpha1.RequestPhaseReleased, observed: ptr.To(int64(2))}, + {name: "Released accepted execution", phase: lpxv1alpha1.RequestPhaseReleased, observed: ptr.To(int64(2)), accepted: ptr.To(int64(2)), want: true}, + {name: "unknown phase", phase: "future", observed: ptr.To(int64(2))}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Only current scheduler receipts and accepted executions can exempt a cycle") + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{Phase: tc.phase, ObservedGeneration: tc.observed}, + } + if tc.accepted != nil { + request.Status.Committed = &lpxv1alpha1.Committed{Execution: lpxv1alpha1.CommittedExecution{AcceptedGeneration: tc.accepted}} + } + if tc.noStatus { + request.Status = nil + } + require.Equal(t, tc.want, isPipelineRequestDeadlineExempt(request)) + }) + } +} + +func TestRequeueForPipelineRequestDeadline(t *testing.T) { + transient := errors.New("temporary failure") + for _, tc := range []struct { + name string + until time.Duration + noDeadline bool + requeue time.Duration + err error + wantDelay time.Duration + }{ + {name: "no deadline uses error backoff", noDeadline: true, err: transient}, + {name: "no deadline discards unrelated retry on error", noDeadline: true, err: transient, requeue: time.Minute}, + {name: "no deadline preserves result", noDeadline: true, requeue: time.Minute, wantDelay: time.Minute}, + {name: "active deadline", until: time.Minute, wantDelay: time.Minute}, + {name: "earlier existing retry", until: time.Minute, requeue: time.Second, wantDelay: time.Second}, + {name: "earlier deadline", until: time.Second, requeue: time.Minute, wantDelay: time.Second}, + {name: "elapsed deadline", until: -time.Minute, wantDelay: time.Nanosecond}, + {name: "error bounded by retry interval", until: time.Minute, err: transient, wantDelay: pipelineRequestDeadlineRetryInterval}, + {name: "error bounded by deadline", until: time.Second, err: transient, wantDelay: time.Second}, + {name: "error after deadline", until: -time.Minute, err: transient, wantDelay: pipelineRequestDeadlineRetryInterval}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Preserve an earlier wakeup and keep active-deadline errors on a bounded retry") + var deadline time.Time + if !tc.noDeadline { + deadline = time.Now().Add(tc.until) + } + result := requeueForPipelineRequestDeadline(deadline, ctrl.Result{RequeueAfter: tc.requeue}, tc.err) + require.LessOrEqual(t, result.RequeueAfter, tc.wantDelay) + if tc.wantDelay <= time.Nanosecond || tc.wantDelay == tc.requeue || tc.wantDelay == pipelineRequestDeadlineRetryInterval { + require.Equal(t, tc.wantDelay, result.RequeueAfter) + } else { + require.InDelta(t, tc.wantDelay, result.RequeueAfter, float64(time.Second)) + } + }) + } +} + +func TestReconcileExpiredPipelineRequests(t *testing.T) { + for _, tc := range []struct { + name string + replicas int32 + expired []int + manageReplicas bool + wantReplicas int32 + removed []int + }{ + {name: "healthy lower engine", replicas: 2, expired: []int{1}, manageReplicas: true, wantReplicas: 1, removed: []int{1}}, + {name: "interior hole", replicas: 3, expired: []int{1}, manageReplicas: true, wantReplicas: 3}, + {name: "hole and suffix", replicas: 3, expired: []int{0, 2}, manageReplicas: true, wantReplicas: 3}, + {name: "maximal suffix", replicas: 4, expired: []int{2, 3}, manageReplicas: true, wantReplicas: 2, removed: []int{2, 3}}, + {name: "all expired", replicas: 2, expired: []int{0, 1}, manageReplicas: true, removed: []int{0, 1}}, + {name: "externally managed capacity", replicas: 1, expired: []int{0}, wantReplicas: 1, removed: []int{0}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe independent engine requests with only the selected ordinals expired") + dgd := loadTestDGD(t, lpx.PipelineSingle, "build-v2") + deployment := newLPXTestDeployment(t, dgd) + pcs := newTestPodCliqueSet(deployment) + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "engines", Namespace: deployment.Namespace}, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: tc.replicas}, + } + requests := make([]lpxv1alpha1.LPUPipelineRequest, tc.replicas) + objects := []client.Object{pcs, group} + var expired []*lpxv1alpha1.LPUPipelineRequest + for i := range requests { + request := newTestPipelineRequest(deployment, pcs, fmt.Sprintf("engine-%d", i), time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) + if slices.Contains(tc.expired, i) { + request.Status.Phase = lpxv1alpha1.RequestPhasePending + request.Status.Committed = nil + } + request.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: int64(i)} + requests[i] = *request + objects = append(objects, request) + if slices.Contains(tc.expired, i) { + expired = append(expired, &requests[i]) + } + } + r := newLPXTestReconciler(t, nil, deployment, dgd, objects...) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + for i := range requests { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(&requests[i]), &requests[i])) + } + + t.Log("Apply suffix cleanup without deleting PodCliques or changing healthy requests") + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + _, ok := object.(*lpxv1alpha1.LPUPipelineRequest) + require.True(t, ok, "cleanup must delete only pipeline requests") + live := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(group), live)) + require.Equal(t, tc.wantReplicas, live.Spec.Replicas, "scale must precede deletion") + return delegated.Delete(ctx, object, opts...) + }, + }) + result, err := r.reconcileExpiredPipelineRequests(t.Context(), group, requests, expired, tc.manageReplicas) + require.NoError(t, err) + require.Zero(t, result, "LPR and PCSG watches drive further cleanup") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) + require.Equal(t, tc.wantReplicas, group.Spec.Replicas) + for i := range requests { + got := &lpxv1alpha1.LPUPipelineRequest{} + err := r.Get(t.Context(), client.ObjectKeyFromObject(&requests[i]), got) + if slices.Contains(tc.removed, i) { + require.True(t, apierrors.IsNotFound(err)) + } else { + require.NoError(t, err) + require.Equal(t, requests[i], *got) + } + } + }) + } +} + +func TestExpiredPipelineRequestSuffix(t *testing.T) { + for _, tc := range []struct { + name string + replicas int32 + ordinals []int64 + expired []int + wantReplicas int32 + wantRemoved []string + }{ + {name: "no expiry", replicas: 2, ordinals: []int64{0, 1}, wantReplicas: 2}, + {name: "siblings before expired model", replicas: 1, ordinals: []int64{0, 0}, expired: []int{0}, wantRemoved: []string{"request-1", "request-0"}}, + {name: "scale already lowered", replicas: 0, ordinals: []int64{0, 0}, expired: []int{0}, wantRemoved: []string{"request-1", "request-0"}}, + {name: "finish requests beyond live count", replicas: 1, ordinals: []int64{0, 1, 2}, expired: []int{1}, wantReplicas: 1, wantRemoved: []string{"request-2", "request-1"}}, + {name: "hole still blocks cleanup beyond live count", replicas: 2, ordinals: []int64{0, 1, 2}, expired: []int{0, 2}, wantReplicas: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Derive the complete trailing request set while preserving expiry evidence") + requests := make([]lpxv1alpha1.LPUPipelineRequest, len(tc.ordinals)) + var expired []*lpxv1alpha1.LPUPipelineRequest + for i, ordinal := range tc.ordinals { + requests[i].Name = fmt.Sprintf("request-%d", i) + requests[i].Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: "engines", ReplicaIndex: ordinal} + if slices.Contains(tc.expired, i) { + expired = append(expired, &requests[i]) + } + } + replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, tc.replicas) + require.NoError(t, err) + require.Equal(t, tc.wantReplicas, replicas) + var names []string + for _, request := range removed { + names = append(names, request.Name) + } + require.Equal(t, tc.wantRemoved, names) + }) + } +} + +func newTestPipelineRequest(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, name string, created time.Time, phase lpxv1alpha1.RequestPhase) *lpxv1alpha1.LPUPipelineRequest { + generation := int64(1) + schedulingStartedAt := metav1.NewTime(created.UTC().Truncate(time.Second)) + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: deployment.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", + Generation: generation, CreationTimestamp: metav1.NewTime(created), + Labels: map[string]string{deploymentUIDLabel: string(deployment.UID)}, + Annotations: map[string]string{deploymentUIDAnnotation: string(deployment.UID)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: phase, ObservedGeneration: &generation, + SchedulingStartedAt: &schedulingStartedAt, + }, + } + if phase == lpxv1alpha1.RequestPhaseBound { + request.Status.LastPlanRevision = 1 + request.Status.Committed = &lpxv1alpha1.Committed{ + Execution: lpxv1alpha1.CommittedExecution{AcceptedGeneration: &generation, NodeLocal: &lpxv1alpha1.NodeLocalExecution{}}, + Plan: lpxv1alpha1.CommittedPlan{ + PlannedFromGeneration: generation, Revision: 1, PlanDigest: "sha256:plan", + Placement: lpxv1alpha1.PlanPlacement{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal, NodeLocal: &lpxv1alpha1.NodeLocalPlacement{}}, + }, + } + } + return request +} + +func newTestPodCliqueSet(deployment *v1alpha1.LPXGraphDeployment) *grovev1alpha1.PodCliqueSet { + return &grovev1alpha1.PodCliqueSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.PCSNameForLPX(deployment), Namespace: deployment.Namespace, UID: "pcs-uid", ResourceVersion: "1", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)}, + }, + Spec: grovev1alpha1.PodCliqueSetSpec{Replicas: 1}, + } +} + +func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { + t.Log("Publish multiple model requests for one engine and expire one of them") + ctx := t.Context() + child, dgd, registry := newLPXSpecDecodeTestDGD(t) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + requests, err := r.getPipelineRequests(ctx, pcs) + require.NoError(t, err) + group, err := getPodCliqueScalingGroup(ctx, r.Client, pcs) + require.NoError(t, err) + require.Greater(t, len(requests), 1) + expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) + var sibling *lpxv1alpha1.LPUPipelineRequest + for index := range requests { + if requests[index].Name != expired.Name { + sibling = &requests[index] + break + } + } + require.NotNil(t, sibling) + + t.Log("Fail the first sibling deletion and retain the expired request for a retry") + deleteErr := errors.New("transient sibling deletion failure") + base := r.Client + r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok && request.Name == sibling.Name { + return deleteErr + } + return delegated.Delete(ctx, object, opts...) + }, + }) + _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + require.ErrorIs(t, err, deleteErr) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("Retry from the retained expiry proof and remove the complete engine request set") + r.Client = base + requests, err = r.getPipelineRequests(ctx, pcs) + require.NoError(t, err) + expired = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, expired.Name) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + require.NoError(t, err) + for index := range desired.requests { + requirePipelineRequestNotFound(t, ctx, r.Client, child.Namespace, desired.requests[index].Name) + } +} + +func TestExpiredPipelineRequestsWaitForScalingGroup(t *testing.T) { + t.Log("Keep an expired request while its live PCS has not materialized a scaling group") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) + require.NoError(t, r.Delete(ctx, group)) + requests, err := r.getPipelineRequests(ctx, pcs) + require.NoError(t, err) + _, err = r.reconcileExpiredPipelineRequests(ctx, nil, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) + + t.Log("Once Grove creates the group, issue scale-down before deleting the LPR") + group.ResourceVersion = "" + group.UID = "replacement-group" + group.DeletionTimestamp = nil + group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + require.NoError(t, r.Create(ctx, group)) + _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) + require.Zero(t, group.Spec.Replicas) + require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) +} diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go new file mode 100644 index 000000000000..554a57562e85 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -0,0 +1,374 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "context" + "slices" + "strings" + "testing" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestGetPipelineRequests(t *testing.T) { + for _, tc := range []struct { + name string + namespace string + uid types.UID + want []string + }{ + {name: "owned requests", namespace: "default", uid: "pcs-uid", want: []string{"first", "last"}}, + {name: "different PCS with the same name", namespace: "default", uid: "other-uid", want: []string{"other-owner"}}, + {name: "different namespace", namespace: "other", uid: "pcs-uid", want: []string{"other-namespace"}}, + {name: "no owned requests", namespace: "default", uid: "missing-uid"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Seed only pipeline requests, with separate owners and namespaces") + var objects []client.Object + for _, request := range []struct { + name, namespace string + uid types.UID + }{ + {"last", "default", "pcs-uid"}, + {"first", "default", "pcs-uid"}, + {"other-owner", "default", "other-uid"}, + {"other-namespace", "other", "pcs-uid"}, + } { + objects = append(objects, &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{ + Name: request.name, Namespace: request.namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet", + Name: "pcs", UID: request.uid, Controller: ptr.To(true), + }}, + }}) + } + r := &graphReconciler{Client: newLPXTestClient(t, objects...)} + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: "pcs", Namespace: tc.namespace, UID: tc.uid}} + + t.Log("Return only this PCS's requests in stable name order") + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + var names []string + for _, request := range requests { + names = append(names, request.Name) + } + require.Equal(t, tc.want, names) + }) + } +} + +func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { + t.Log("Publish the implicit hybrid runtime without interpreting stale selector annotations") + ctx := t.Context() + deployment, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + dgd.Annotations[consts.KubeAnnotationLPXSchedulerBackend] = "unknown-scheduler" + dgd.Annotations[consts.KubeAnnotationLPXExecutionBackend] = "unknown-execution" + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) + require.NotNil(t, desired) + require.Equal(t, "unknown-scheduler", dgd.Annotations[consts.KubeAnnotationLPXSchedulerBackend]) + require.Equal(t, "unknown-execution", dgd.Annotations[consts.KubeAnnotationLPXExecutionBackend]) + require.Empty(t, desired.plan.ConductorTemplate) + require.Empty(t, desired.plan.ConductorClique) + require.NotEmpty(t, desired.plan.CyborgClique) + + objects := lpxMaterializedObjects(t, reconciler, deployment, dgd, desired) + for _, object := range objects { + require.NotEmpty(t, object.GetName()) + } + group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + require.NotContains(t, group.Spec.CliqueNames, "") + cyborg := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique) + require.NotContains(t, cyborg.Spec.StartsAfter, "") + require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + condition := publishSelectedLPXForTest(t, ctx, reconciler, deployment, desired) + require.NotNil(t, condition) + require.Equal(t, v1alpha1.LPXReadyReasonPending, condition.Reason) + request := getTestPipelineRequest(t, ctx, reconciler.Client, deployment.Namespace, desired.requests[0].Name) + pcs := findLPXTestPodCliqueSet(t, objects) + require.True(t, metav1.IsControlledBy(pcs, deployment)) + require.True(t, metav1.IsControlledBy(request, pcs)) + require.True(t, *metav1.GetControllerOf(request).BlockOwnerDeletion) + require.Empty(t, request.Finalizers) + require.NotNil(t, request.Spec.CyborgPodCliqueRef) + require.Equal(t, desired.plan.CyborgClique, request.Spec.CyborgPodCliqueRef.Name) +} + +func TestPipelineRequestIdentityDigest(t *testing.T) { + t.Log("Every identity field distinguishes requests; the same tuple always has the same digest") + base := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "default", 0) + for _, test := range []struct { + name string + namespace string + deployment string + uid types.UID + model string + replica int32 + same bool + }{ + {"unchanged", "ns", "dgd", "uid-a", "default", 0, true}, + {"namespace", "other-ns", "dgd", "uid-a", "default", 0, false}, + {"deployment", "ns", "other-dgd", "uid-a", "default", 0, false}, + {"UID", "ns", "dgd", "uid-b", "default", 0, false}, + {"model", "ns", "dgd", "uid-a", "other-model", 0, false}, + {"replica", "ns", "dgd", "uid-a", "default", 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + digest := pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, test.model, test.replica) + require.NotEmpty(t, digest) + require.Equal(t, test.same, base == digest) + require.Equal(t, digest, pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, test.model, test.replica)) + }) + } +} + +func TestPipelineRequestName(t *testing.T) { + t.Log("Keep request names readable and within the DNS label limit") + digest := pipelineRequestIdentityDigest("ns", "dgd", "uid", "default", 0) + for _, test := range []struct{ name, prefix string }{ + {"dgd", "dgd"}, + {"a-very-long-but-readable-dynamo-graph-deployment-name", "a-very-long-but-readabl"}, + {"graph.name", "graph-name"}, + } { + t.Run(test.name, func(t *testing.T) { + actual := pipelineRequestName(test.name, digest) + require.LessOrEqual(t, len(actual), 63) + require.Equal(t, "lpx-"+test.prefix+"-"+strings.TrimPrefix(digest, "sha256:")[:32], actual) + }) + } +} + +func TestResolvePipelineRequestsPreservesImmutableIntent(t *testing.T) { + t.Log("Immutable intent compares desired annotations and spec, not scheduler-owned metadata") + deployment, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + desired := resolveLPXTestWorkload(t, registry, t.Context(), deployment, dgd) + for _, test := range []struct { + name string + annotation string + value string + changed bool + }{ + {"scheduler annotation", "scheduler.example/status", "observed", false}, + {"workload changed", lpx.WorkloadDigestAnnotation, "another-build", true}, + {"compiler snapshot changed", lpxv1alpha1.CompilerSnapshotDigestAnnotation, "another-snapshot", true}, + } { + t.Run(test.name, func(t *testing.T) { + current := desired.requests[0].DeepCopy() + current.Annotations[test.annotation] = test.value + require.Equal(t, !test.changed, pipelineRequestMatches(current, &desired.requests[0])) + requests, missing, changed := resolvePipelineRequests(deployment, []lpxv1alpha1.LPUPipelineRequest{*current}, desired.workload, desired.plan) + require.Equal(t, test.changed, changed) + require.Empty(t, missing) + if !changed { + require.Equal(t, current, requests[current.Name]) + } else { + require.Nil(t, requests) + } + }) + } +} + +func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection(t *testing.T) { + ctx := t.Context() + deployment, dgd, registry := newLPXSpecDecodeTestDGD(t) + + reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) + require.Len(t, desired.requests, 3) + require.Len(t, desired.plan.Agents, 3) + objects := lpxMaterializedObjects(t, reconciler, deployment, dgd, desired) + createLPXTestObjects(t, ctx, reconciler.Client, objects...) + + condition := publishSelectedLPXForTest(t, ctx, reconciler, deployment, desired) + require.NotNil(t, condition) + require.Equal(t, v1alpha1.LPXReadyReasonPending, condition.Reason) + requests, err := reconciler.getPipelineRequests(ctx, findLPXTestPodCliqueSet(t, objects)) + require.NoError(t, err) + require.Len(t, requests, 3) + + requestByModel := make(map[string]lpxv1alpha1.LPUPipelineRequest, len(requests)) + for _, request := range requests { + model := request.Annotations[pipelineRequestModelAnnotation] + requestByModel[model] = request + require.Equal(t, string(deployment.UID), request.Labels[deploymentUIDLabel]) + require.Equal(t, model, request.Spec.NodeLocal.Model) + require.Equal(t, desired.plan.LPXScalingGroup, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name) + } + for _, projection := range desired.requests { + request, found := requestByModel[projection.Annotations[pipelineRequestModelAnnotation]] + require.True(t, found) + require.Equal(t, projection.Annotations[lpx.WorkloadDigestAnnotation], request.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, projection.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation], request.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) + } + + for index, expected := range desired.plan.Agents { + projection := &desired.requests[index] + clique := getResource[*grovev1alpha1.PodClique](t, objects, expected.CliqueName) + require.Equal(t, int32(expected.Replicas), clique.Spec.Replicas) + require.Equal(t, ptr.To(int32(expected.Replicas)), clique.Spec.MinAvailable) + require.Equal(t, projection.Annotations[lpx.WorkloadDigestAnnotation], clique.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, projection.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation], clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) + require.Equal(t, projection.Annotations[pipelineRequestModelAnnotation], clique.Annotations[lpxv1alpha1.PodModelAnnotation]) + require.NotContains(t, clique.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) + require.NotContains(t, clique.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) + } + pcs := findLPXTestPodCliqueSet(t, objects) + require.Equal(t, desired.workload.Digest().String(), pcs.Annotations[lpx.WorkloadDigestAnnotation]) + +} + +func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { + for _, tc := range []struct { + name string + replicas int32 + published []int + wantMissing []int + intentChanged bool + }{ + {name: "all missing", replicas: 2, wantMissing: []int{0, 1, 2, 3, 4, 5}}, + {name: "partial publication", replicas: 2, published: []int{4, 0, 2}, wantMissing: []int{1, 3, 5}}, + {name: "all published", replicas: 2, published: []int{5, 4, 3, 2, 1, 0}}, + {name: "scaled down", replicas: 1, published: []int{5, 4, 3, 2, 1, 0}}, + {name: "zero replicas", published: []int{5, 4, 3, 2, 1, 0}}, + {name: "mismatch discards earlier missing requests", replicas: 2, published: []int{5}, intentChanged: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Resolve two engine replicas with draft0, draft1 and target models") + deployment, dgd, registry := newLPXSpecDecodeTestDGD(t) + desired := resolveLPXTestWorkload(t, registry, t.Context(), deployment, dgd) + desired.plan.Replicas = 2 + _, rendered, changed := resolvePipelineRequests(deployment, nil, desired.workload, desired.plan) + require.False(t, changed) + require.Len(t, rendered, 6) + for index, request := range rendered { + require.Equal(t, int64(index/3), request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex) + require.Equal(t, []string{"draft0", "draft1", "target"}[index%3], request.Annotations[pipelineRequestModelAnnotation]) + } + + t.Log("Keep observed receipts and collect only absent desired names without sorting") + var observed []lpxv1alpha1.LPUPipelineRequest + for _, index := range tc.published { + request := rendered[index].DeepCopy() + request.UID = types.UID(request.Name) + request.Status = &lpxv1alpha1.LPUPipelineRequestStatus{Phase: lpxv1alpha1.RequestPhaseBound} + if tc.intentChanged { + request.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation] = "another-snapshot" + } + observed = append(observed, *request) + } + desired.plan.Replicas = tc.replicas + requests, missing, changed := resolvePipelineRequests(deployment, observed, desired.workload, desired.plan) + require.Equal(t, tc.intentChanged, changed) + if changed { + require.Nil(t, requests) + require.Nil(t, missing) + return + } + require.Len(t, requests, int(tc.replicas)*3) + require.Len(t, missing, len(tc.wantMissing)) + for index, renderedIndex := range tc.wantMissing { + require.Equal(t, rendered[renderedIndex], missing[index]) + require.Same(t, requests[missing[index].Name], missing[index]) + } + for index := range observed { + request := &observed[index] + if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex < int64(tc.replicas) { + require.Same(t, request, requests[request.Name]) + } + } + }) + } +} + +func TestPipelineRequestsPendingDeletion(t *testing.T) { + for _, tc := range []struct { + name string + desired []string + terminating []string + want []string + }{ + {"all current", []string{"first", "second"}, nil, nil}, + {"removed request", []string{"first"}, nil, []string{"second"}}, + {"terminating removed request", []string{"first"}, []string{"second"}, []string{"second"}}, + {"terminating current name", []string{"first", "second"}, []string{"first"}, []string{"first"}}, + {"both pending", []string{"first"}, []string{"first"}, []string{"first", "second"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Block publication until removed and already-terminating request names disappear") + requests := []lpxv1alpha1.LPUPipelineRequest{ + {ObjectMeta: metav1.ObjectMeta{Name: "first"}}, {ObjectMeta: metav1.ObjectMeta{Name: "second"}}, + } + desired := make(map[string]*lpxv1alpha1.LPUPipelineRequest) + for index := range requests { + request := &requests[index] + if slices.Contains(tc.desired, request.Name) { + desired[request.Name] = request + } + if slices.Contains(tc.terminating, request.Name) { + request.DeletionTimestamp = ptr.To(metav1.Now()) + } + } + + var names []string + for _, request := range pipelineRequestsPendingDeletion(requests, desired) { + names = append(names, request.Name) + } + require.Equal(t, tc.want, names) + }) + } +} + +func getTestPipelineRequest(t *testing.T, ctx context.Context, kubeClient client.Reader, namespace, name string) *lpxv1alpha1.LPUPipelineRequest { + t.Helper() + request := &lpxv1alpha1.LPUPipelineRequest{} + require.NoError(t, kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request)) + return request +} + +// publishSelectedLPXForTest seeds requests through the production publication helper. +// It does not simulate reconciliation, readiness or deletion decisions. +func publishSelectedLPXForTest( + t *testing.T, + ctx context.Context, + reconciler *graphReconciler, + deployment *v1alpha1.LPXGraphDeployment, + desired *lpxTestWorkload, +) *metav1.Condition { + t.Helper() + pcs := observedLPXTestPodCliqueSet(t, ctx, reconciler, deployment, desired) + require.NotNil(t, pcs) + requests := make([]*lpxv1alpha1.LPUPipelineRequest, len(desired.requests)) + for index := range desired.requests { + requests[index] = desired.requests[index].DeepCopy() + } + require.NoError(t, reconciler.reconcilePipelineRequests(ctx, deployment, pcs, requests)) + return meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) +} + +func requirePipelineRequestNotFound( + t *testing.T, + ctx context.Context, + kubeClient client.Reader, + namespace string, + name string, +) { + t.Helper() + request := &lpxv1alpha1.LPUPipelineRequest{} + err := kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, request) + require.True(t, apierrors.IsNotFound(err), "expected LPX request %s/%s to be absent, got %v", namespace, name, err) +} diff --git a/deploy/operator/internal/controller/lpx/preflight.go b/deploy/operator/internal/controller/lpx/preflight.go deleted file mode 100644 index d030f3a331a7..000000000000 --- a/deploy/operator/internal/controller/lpx/preflight.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package lpx - -import ( - "context" - "fmt" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" -) - -// reconcileLPXSafetyPreflight classifies and fences scheduler-owned state -// after the LPX controller validates source identity and the Grove provider. -// pcs is nil when its stable name was not found. -func (r *graphReconciler) reconcileLPXSafetyPreflight( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, - requests []lpxv1alpha1.LPUPipelineRequest, -) (*lpxMaterializing, lpxClassification, error) { - classification, requests, err := r.reconcileLPXKnownIntentFence(ctx, deployment, source, pcs, requests) - if err != nil { - return nil, nil, fmt.Errorf("failed to fence prior LPX intent: %w", err) - } - if classification != nil { - return nil, classification, nil - } - - if len(requests) == 0 { - if err := dynamolpx.ValidateSelectedIntent(source).ToAggregate(); err != nil { - return nil, &lpxRejected{reason: err.Error()}, nil - } - return nil, nil, nil - } - - return r.reconcileSelectedLPXSafetyPreflight(ctx, deployment, source, pcs) -} - -// reconcileSelectedLPXSafetyPreflight resolves and validates one selected LPX -// workload after its caller establishes the required download order. pcs is -// nil when its stable name was not found. -func (r *graphReconciler) reconcileSelectedLPXSafetyPreflight( - ctx context.Context, - deployment *nvidiacomv1alpha1.LPXGraphDeployment, - source *nvidiacomv1beta1.DynamoGraphDeployment, - pcs *grovev1alpha1.PodCliqueSet, -) (*lpxMaterializing, lpxClassification, error) { - // Resolve one immutable workload projection for selection and rendering. - selectedLPX, classification, prepareErr := r.prepareLPXMaterializing(ctx, deployment, source, pcs) - if prepareErr != nil { - return nil, nil, fmt.Errorf("failed to resolve the LPX workload: %w", prepareErr) - } - if classification != nil { - return nil, classification, nil - } - - // Grove accepts the desired spec before individual engine requests are reconciled. - return selectedLPX, nil, nil -} diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index b0093d3bd00e..688f60d528fe 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -9,45 +9,52 @@ import ( "fmt" "maps" - configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -// renderPodCliqueSet renders the exact workload selected by controller preflight. -// The source, child, workload and plan must be non-nil and validated by that -// preflight; the plan must use the child's PodCliqueSet identity. The renderer +const ( + deploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation + deploymentUIDLabel = deploymentUIDAnnotation +) + +// renderPodCliqueSet renders the admitted DGD using the resolved build and plan. +// The DGD, child, workload and plan must be non-nil; the plan must use the +// child's PodCliqueSet identity. The renderer // does not mutate its inputs. secretsRetriever may be nil when no secrets are used. -func renderPodCliqueSet( +func (r *graphReconciler) renderPodCliqueSet( ctx context.Context, - source *v1beta1.DynamoGraphDeployment, - operatorConfig *configv1alpha1.OperatorConfiguration, - runtimeConfig *controller_common.RuntimeConfig, - secretsRetriever dynamo.SecretsRetriever, - workload *dynamolpx.SelectedWorkload, - plan *dynamolpx.MaterializationPlan, deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + workload *lpx.SelectedWorkload, + plan *lpx.MaterializationPlan, ) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { // Apply shared Dynamo defaults before materializing the independently owned LPX roles. - pcs, input, err := dynamo.RenderLPXBasePodCliqueSet(ctx, source, operatorConfig, runtimeConfig, - secretsRetriever, workload, plan) + pcs, input, err := dynamo.RenderLPXBasePodCliqueSet( + ctx, + dgd, + r.config, + r.runtimeConfig, + r.dockerSecretRetriever, + workload, + plan, + ) if err != nil { return nil, nil, err } - resources, err := dynamolpx.RenderSelectedNodeLocal(pcs, workload, plan, *input) + resources, err := lpx.RenderSelectedNodeLocal(pcs, workload, plan, *input) if err != nil { return nil, nil, err } // Stamp stable ownership and identify the engine's serving endpoint. - stampLPXIdentity(deployment, pcs, resources) + stampDeploymentIdentity(deployment, pcs, resources) serving := plan.ConductorTemplate if serving == "" { serving = plan.CyborgTemplate @@ -57,16 +64,16 @@ func renderPodCliqueSet( restartToken := deployment.Annotations[dynamo.LPXRestartAnnotation] for _, clique := range pcs.Spec.Template.Cliques { if restartToken != "" { - clique.Annotations[commonconsts.RestartAnnotation] = restartToken + clique.Annotations[consts.RestartAnnotation] = restartToken } delete(clique.Labels, dynamo.LPXServingLabel) if clique.Name == serving { - clique.Labels[dynamo.LPXServingLabel] = commonconsts.KubeLabelValueTrue + clique.Labels[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue } else { // Auxiliary roles must be private in both Service and Pod discovery modes. - delete(clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) - delete(clique.Labels, commonconsts.KubeLabelDynamoDiscoveryBackend) - delete(clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) + delete(clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) + delete(clique.Labels, consts.KubeLabelDynamoDiscoveryBackend) + delete(clique.Labels, consts.KubeLabelDynamoBaseModelHash) } } @@ -75,27 +82,27 @@ func renderPodCliqueSet( if err != nil { return nil, nil, fmt.Errorf("serializing selected LPX PodCliqueSet: %w", err) } - if len(serialized) > dynamolpx.MaxRenderedPodCliqueSetBytes { + if len(serialized) > lpx.MaxRenderedPodCliqueSetBytes { return nil, nil, fmt.Errorf("rendered LPX PodCliqueSet is %d bytes; maximum is %d", - len(serialized), dynamolpx.MaxRenderedPodCliqueSetBytes) + len(serialized), lpx.MaxRenderedPodCliqueSetBytes) } return pcs, resources, nil } -// stampLPXIdentity propagates stable ownership labels and annotations, never source revision. -func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { - // Preflight validates the exact source controller owner before rendering resources. - sourceOwner := metav1.GetControllerOf(deployment) +// stampDeploymentIdentity propagates stable ownership labels and annotations, never DGD revision. +func stampDeploymentIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { + // The initial DGD lookup validated this controller owner; rendering trusts that observation. + dgdOwner := metav1.GetControllerOf(deployment) identity := map[string]string{ - dynamolpx.DeploymentNameAnnotation: deployment.Name, - lpxDeploymentUIDAnnotation: string(deployment.UID), - dynamolpx.DGDUIDAnnotation: string(sourceOwner.UID), + lpx.DeploymentNameAnnotation: deployment.Name, + deploymentUIDAnnotation: string(deployment.UID), + lpx.DGDUIDAnnotation: string(dgdOwner.UID), } stamp := func(annotations *map[string]string) { if *annotations == nil { *annotations = make(map[string]string) } - delete(*annotations, dynamolpx.DGDGenerationAnnotation) + delete(*annotations, lpx.DGDGenerationAnnotation) maps.Copy(*annotations, identity) } stampOwnerLabel := func(object client.Object) { @@ -103,7 +110,7 @@ func stampLPXIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha if labels == nil { labels = make(map[string]string) } - labels[lpxOwnerUIDLabel] = string(deployment.UID) + labels[deploymentUIDLabel] = string(deployment.UID) object.SetLabels(labels) } stamp(&pcs.Annotations) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 1e424e786d6f..0841a6f2ad33 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -4,14 +4,14 @@ package lpx import ( + "context" "encoding/json" "fmt" - "maps" + "net/url" "os" "path/filepath" "slices" "sort" - "strconv" "strings" "testing" "time" @@ -20,8 +20,8 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" @@ -30,8 +30,6 @@ import ( "github.com/stretchr/testify/require" "gotest.tools/v3/golden" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,16 +37,131 @@ import ( "sigs.k8s.io/yaml" ) -func newLPXRenderDeployment(t *testing.T, source *v1beta1.DynamoGraphDeployment) *v1alpha1.LPXGraphDeployment { - t.Helper() - revision, err := dynamo.LPXInputRevision(source, "") - require.NoError(t, err) - return &v1alpha1.LPXGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: "lpx-render-uid", Generation: 1, - Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(source.Generation, 10)}, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, v1beta1.DynamoGraphDeploymentGVK)}, +// Update this golden with: +// go test ./internal/controller/lpx -run TestGenerateGrovePodCliqueSet_FromDGDYaml -count=1 -args -update +func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { + controllerConfig := &configv1alpha1.OperatorConfiguration{ + Infrastructure: configv1alpha1.InfrastructureConfiguration{ + ETCDAddress: "etcd-address", + NATSAddress: "nats-address", + }, + Orchestrators: configv1alpha1.OrchestratorConfiguration{ + Grove: configv1alpha1.GroveConfiguration{ + TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}, + }, }, - Spec: v1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, + LPX: configv1alpha1.LPXConfiguration{Enabled: true}, + } + + registryRoot := t.TempDir() + testdataModelRegistry := newTestDataModelRegistry(t, registryRoot) + kubeClient := fake.NewClientBuilder().Build() + runtimeConfig := &commoncontroller.RuntimeConfig{} + + tests := []string{ + "from_dgd_yaml/lpx_v2_vllm", + "from_dgd_yaml/node-local-v2-lpu-only", + "from_dgd_yaml/node-local-v2-hybrid", + "from_dgd_yaml/node-local-v2-specdecode", + "from_dgd_yaml/node-local-v3-hx-lpu-only", + "from_dgd_yaml/node-local-v3-hx-specdecode", + "from_dgd_yaml/node-local-v3-hx-hybrid", + "from_dgd_yaml/single_v2", + } + + for _, name := range tests { + t.Run(name, func(t *testing.T) { + t.Log("Load the authored input before selection") + b, err := os.ReadFile("../../dynamo/lpx/testdata/" + name + ".input.yaml") + require.NoError(t, err) + + var dynamoDeployment v1beta1.DynamoGraphDeployment + require.NoError(t, yaml.Unmarshal(b, &dynamoDeployment)) + if dynamoDeployment.UID == "" { + dynamoDeployment.UID = "11111111-1111-4111-8111-111111111111" + } + if dynamoDeployment.Generation == 0 { + dynamoDeployment.Generation = 1 + } + selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) + require.NoError(t, err) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment))) + require.NoError(t, err) + t.Log("Render all LPX roles with the separate LPX PCS identity") + r := &graphReconciler{config: controllerConfig, runtimeConfig: runtimeConfig} + got, extraResources, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, &dynamoDeployment), &dynamoDeployment, selected, plan) + require.NoError(t, err) + for _, clique := range got.Spec.Template.Cliques { + require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + component := dynamoDeployment.GetComponentByName(clique.Labels[consts.KubeLabelDynamoComponent]) + require.NotNil(t, component) + require.True(t, component.IsLPX()) + require.Equal(t, component.ComponentName, clique.Labels[lpx.StageLabel]) + } + + t.Log("Render conventional components independently from their ordinary-only input") + ordinary := dynamoDeployment.DeepCopy() + ordinary.Spec.Components = slices.DeleteFunc(ordinary.Spec.Components, func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.IsLPX() + }) + normal, err := dynamo.GenerateGrovePodCliqueSet( + t.Context(), ordinary, controllerConfig, runtimeConfig, + kubeClient, nil, nil, nil, nil, + ) + require.NoError(t, err) + require.NotEqual(t, normal.Name, got.Name) + for _, clique := range normal.Spec.Template.Cliques { + require.NotEqual(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.NotEqual(t, selected.LPXComponentName(), clique.Labels[consts.KubeLabelDynamoComponent]) + } + podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} + if len(normal.Spec.Template.Cliques) > 0 { + podCliqueSets = append(podCliqueSets, normal) + } + + for _, pcs := range podCliqueSets { + sort.Slice(pcs.Spec.Template.Cliques, func(i, j int) bool { + return pcs.Spec.Template.Cliques[i].Name < pcs.Spec.Template.Cliques[j].Name + }) + for _, clique := range pcs.Spec.Template.Cliques { + for i := range clique.Spec.PodSpec.Containers { + slices.SortFunc(clique.Spec.PodSpec.Containers[i].Env, func(a, b corev1.EnvVar) int { + return strings.Compare(a.Name, b.Name) + }) + } + } + extraResources = append(extraResources, pcs) + } + + var out []string + for _, resource := range extraResources { + b, err := yaml.Marshal(resource) + if err != nil { + t.Errorf("Marshal() error = %v", err) + } + out = append(out, string(b)) + } + + t.Log("Replace computed content hashes with stable golden placeholders; mismatched references remain visible") + replacements := []string{registryRoot, "/testdata"} + for _, resource := range extraResources { + config, ok := resource.(*corev1.ConfigMap) + if !ok { + continue + } + hash := lpx.LPUConfigMapHash(config) + role := strings.TrimPrefix(strings.TrimSuffix(config.Name, "-"+hash[:16]), got.Name+"-") + replacements = append(replacements, + hash, "<"+role+"-config-hash>", + hash[:16], "<"+role+"-config-hash-prefix>", + ) + } + goldenPath, err := filepath.Abs("../../dynamo/lpx/testdata/" + name + ".yaml") + require.NoError(t, err) + const header = "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" + + "# SPDX-License-Identifier: Apache-2.0\n\n" + golden.Assert(t, header+strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) + }) } } @@ -56,25 +169,24 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { t.Log("Resolve a real LPU workload including inherited scheduler metadata") payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml") require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - source.Annotations = map[string]string{"kai.scheduler/padding": ""} - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + dgd.Annotations = map[string]string{"kai.scheduler/padding": ""} + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{} - runtimeConfig := &controller_common.RuntimeConfig{} - pcs, _, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, - selected, plan, newLPXRenderDeployment(t, source)) + runtimeConfig := &commoncontroller.RuntimeConfig{} + r := &graphReconciler{config: config, runtimeConfig: runtimeConfig} + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) require.NoError(t, err) serialized, err := json.Marshal(pcs) require.NoError(t, err) t.Log("Accept exactly one MiB including final identity, discovery and scheduler metadata") - source.Annotations["kai.scheduler/padding"] = strings.Repeat("x", lpx.MaxRenderedPodCliqueSetBytes-len(serialized)) - pcs, _, err = renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, - selected, plan, newLPXRenderDeployment(t, source)) + dgd.Annotations["kai.scheduler/padding"] = strings.Repeat("x", lpx.MaxRenderedPodCliqueSetBytes-len(serialized)) + pcs, _, err = r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) require.NoError(t, err) serialized, err = json.Marshal(pcs) require.NoError(t, err) @@ -84,14 +196,80 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { } t.Log("Reject one additional final-metadata byte as a selected-render failure before publication") - source.Annotations["kai.scheduler/padding"] += "x" - pcs, resources, err := renderPodCliqueSet(t.Context(), source, config, runtimeConfig, nil, - selected, plan, newLPXRenderDeployment(t, source)) + dgd.Annotations["kai.scheduler/padding"] += "x" + pcs, resources, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) require.ErrorContains(t, err, "rendered LPX PodCliqueSet is 1048577 bytes; maximum is 1048576") require.Nil(t, pcs) require.Nil(t, resources) } +func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { + t.Log("Load an authored hybrid engine and override its independent leader template") + payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + require.NoError(t, err) + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + leader := lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: "config", MountPath: "/custom-config", ReadOnly: true}, + {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, + {Name: "model-storage", MountPath: "/nfs"}, + } + leader.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, + {Name: "TOTAL_REPLICAS", Value: "9"}, + {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, + } + for _, name := range []string{"CYBORG_BATCH_SIZE", "CYBORG_FPGA_GPI_IO_FPGA_COUNT", "CYBORG_SWA_CACHE_IDS"} { + leader.Spec.Containers[0].Env = append(leader.Spec.Containers[0].Env, corev1.EnvVar{ + Name: name, ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "runtime-settings"}, Key: name, + }}, + }) + } + authoredConfig := corev1.Volume{ + Name: "config", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + } + leader.Spec.Volumes = append(leader.Spec.Volumes, authoredConfig) + + t.Log("Render the complete LPX workload with ordinary pod spec overrides") + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, newTestDataModelRegistry(t, t.TempDir())) + require.NoError(t, err) + plan := mustPlanSelectedLPX(t, dgd, selected) + r := &graphReconciler{config: &configv1alpha1.OperatorConfiguration{}, runtimeConfig: &commoncontroller.RuntimeConfig{}} + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + require.NoError(t, err) + cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { + return clique.Name == plan.CyborgTemplate + }) + require.GreaterOrEqual(t, cliqueIndex, 0) + podSpec := pcs.Spec.Template.Cliques[cliqueIndex].Spec.PodSpec + + t.Log("Keep one config volume alongside the authored InfiniBand volume") + require.Contains(t, podSpec.Volumes, leader.Spec.Volumes[0]) + var configVolumes []corev1.Volume + for _, volume := range podSpec.Volumes { + if volume.Name == authoredConfig.Name { + configVolumes = append(configVolumes, volume) + } + } + require.Equal(t, []corev1.Volume{authoredConfig}, configVolumes) + + t.Log("Preserve authored mounts and runtime environment without interpreting literal or valueFrom settings") + main := podSpec.Containers[0] + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom-config", ReadOnly: true}) + require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}) + require.False(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" })) + for _, variable := range leader.Spec.Containers[0].Env { + require.Contains(t, main.Env, variable) + } + + t.Log("Preserve the authored command and host-file override without adding a launcher") + require.Equal(t, leader.Spec.Containers[0].Command, main.Command) + require.Equal(t, leader.Spec.Containers[0].Args, main.Args) +} + func TestLPXRenderingPreservesInputs(t *testing.T) { const restartToken = "2026-09-08T00:00:00Z" t.Log("Create immutable native builds while keeping each rendering's mutable inputs separate") @@ -114,39 +292,39 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { fixture := test.name payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + fixture + ".input.yaml") require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - source.UID, source.Generation = "source-dgd-uid", 7 - source.Labels = map[string]string{ + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + dgd.UID, dgd.Generation = "source-dgd-uid", 7 + dgd.Labels = map[string]string{ "priorityClassName": "inference", "kai.scheduler/preemptibility": "NonPreemptible", "project": "test", "unrelated": "ignored", - commonconsts.KubeLabelKaiSchedulerQueue: "lpx-queue", + consts.KubeLabelKaiSchedulerQueue: "lpx-queue", } - metav1.SetMetaDataAnnotation(&source.ObjectMeta, "kai.scheduler/topology", "source-topology") - metav1.SetMetaDataAnnotation(&source.ObjectMeta, "unrelated", "ignored") - metav1.SetMetaDataAnnotation(&source.ObjectMeta, commonconsts.KubeAnnotationEnableMetrics, "false") - source.Spec.Labels = map[string]string{"shared-label": "source"} - source.Spec.Annotations = map[string]string{ + metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, "kai.scheduler/topology", "source-topology") + metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, "unrelated", "ignored") + metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, consts.KubeAnnotationEnableMetrics, "false") + dgd.Spec.Labels = map[string]string{"shared-label": "source"} + dgd.Spec.Annotations = map[string]string{ "shared-annotation": "source", "kai.scheduler/topology": "explicit-pcs-topology", "selected-metadata": "from-dgd", lpx.DGDUIDAnnotation: "stale", lpx.DGDGenerationAnnotation: "stale", } - source.Spec.Env = []corev1.EnvVar{{Name: "LPX_DGD_ENV", Value: test.name}} - for _, component := range lpx.Components(source) { + dgd.Spec.Env = []corev1.EnvVar{{Name: "LPX_DGD_ENV", Value: test.name}} + for _, component := range lpx.Components(dgd) { component.GlobalDynamoNamespace = true component.RuntimeVersionOverride = "1.5.0" for _, role := range component.Roles { if role.PodTemplate != nil { metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, "selected-metadata", "from-component") - metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, commonconsts.RestartAnnotation, "2026-09-07T00:00:00Z") + metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, consts.RestartAnnotation, "2026-09-07T00:00:00Z") metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, "shared-label", "from-component") - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, commonconsts.KubeLabelKaiSchedulerQueue, "lpx-queue") - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, commonconsts.KubeLabelDynamoNamespace, "authored-namespace") + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "lpx-queue") + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelDynamoNamespace, "authored-namespace") } } } - component := lpx.ServingComponent(source) + component := lpx.ServingComponent(dgd) component.ModelRef = &v1beta1.ModelReference{Name: "test/model"} component.MinAvailable = ptr.To(int32(1)) hybrid := strings.HasSuffix(fixture, "-hybrid") @@ -164,21 +342,16 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { main.VolumeMounts[configMount].ReadOnly = true } - t.Log("Preserve role metadata and replica units through alpha/beta conversion before freezing the child") - alpha := &v1alpha1.DynamoGraphDeployment{} - require.NoError(t, alpha.ConvertFrom(source)) - source = &v1beta1.DynamoGraphDeployment{} - require.NoError(t, alpha.ConvertTo(source)) - component = lpx.ServingComponent(source) - child := newLPXRenderDeployment(t, source) + t.Log("Freeze the authored role metadata and replica units before rendering") + child := newLPXRenderDeployment(t, dgd) child.Annotations[dynamo.LPXRestartAnnotation] = restartToken - child.Spec.InputRevision, err = dynamo.LPXInputRevision(source, restartToken) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, restartToken) require.NoError(t, err) - source.Generation++ - before, childBefore := source.DeepCopy(), child.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + dgd.Generation++ + before, childBefore := dgd.DeepCopy(), child.DeepCopy() + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{ Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, @@ -209,37 +382,36 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { } t.Log("Render twice and require identical resources with no input or shared-metadata mutation") - first, firstResources, err := renderPodCliqueSet(t.Context(), source, config, - &controller_common.RuntimeConfig{}, nil, selected, plan, child) + r := &graphReconciler{config: config, runtimeConfig: &commoncontroller.RuntimeConfig{}} + first, firstResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, selected, plan) require.NoError(t, err) - second, secondResources, err := renderPodCliqueSet(t.Context(), source, config, - &controller_common.RuntimeConfig{}, nil, selected, plan, child) + second, secondResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, selected, plan) require.NoError(t, err) require.Equal(t, first, second) require.Equal(t, firstResources, secondResources) require.Equal(t, "inference", first.Labels["priorityClassName"]) require.Equal(t, "NonPreemptible", first.Labels["kai.scheduler/preemptibility"]) - require.Equal(t, "lpx-queue", first.Labels[commonconsts.KubeLabelKaiSchedulerQueue]) + require.Equal(t, "lpx-queue", first.Labels[consts.KubeLabelKaiSchedulerQueue]) require.Equal(t, "test", first.Labels["project"]) require.Equal(t, "explicit-pcs-topology", first.Annotations["kai.scheduler/topology"]) - require.NotContains(t, first.Annotations, commonconsts.RestartAnnotation) + require.NotContains(t, first.Annotations, consts.RestartAnnotation) require.NotContains(t, first.Labels, "unrelated") require.NotContains(t, first.Annotations, "unrelated") require.Equal(t, selected.Digest().String(), first.Annotations[lpx.WorkloadDigestAnnotation]) t.Log("Stamp frozen source identity on every final resource despite a newer ordinary DGD generation") - require.NotEqual(t, source.UID, child.UID) + require.NotEqual(t, dgd.UID, child.UID) annotationMaps := []map[string]string{first.Annotations} for _, clique := range first.Spec.Template.Cliques { - require.Equal(t, restartToken, clique.Annotations[commonconsts.RestartAnnotation]) + require.Equal(t, restartToken, clique.Annotations[consts.RestartAnnotation]) annotationMaps = append(annotationMaps, clique.Annotations) } for _, group := range first.Spec.Template.PodCliqueScalingGroupConfigs { - require.NotContains(t, group.Annotations, commonconsts.RestartAnnotation) + require.NotContains(t, group.Annotations, consts.RestartAnnotation) annotationMaps = append(annotationMaps, group.Annotations) } for _, resource := range firstResources { - require.NotContains(t, resource.GetAnnotations(), commonconsts.RestartAnnotation) + require.NotContains(t, resource.GetAnnotations(), consts.RestartAnnotation) annotationMaps = append(annotationMaps, resource.GetAnnotations()) } for _, annotations := range annotationMaps { @@ -251,7 +423,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { } t.Log("Component and role order preserve the exact child, names and resources") - equivalent := source.DeepCopy() + equivalent := dgd.DeepCopy() slices.Reverse(equivalent.Spec.Components) for index := range equivalent.Spec.Components { slices.Reverse(equivalent.Spec.Components[index].Roles) @@ -266,8 +438,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.NoError(t, err) require.Equal(t, selected.Digest(), equivalentSelected.Digest()) require.Equal(t, plan, equivalentPlan) - equivalentPCS, equivalentResources, err := renderPodCliqueSet(t.Context(), equivalent, config, - &controller_common.RuntimeConfig{}, nil, equivalentSelected, equivalentPlan, child) + equivalentPCS, equivalentResources, err := r.renderPodCliqueSet(t.Context(), child, equivalent, equivalentSelected, equivalentPlan) require.NoError(t, err) require.Equal(t, first, equivalentPCS) require.Equal(t, firstResources, equivalentResources) @@ -279,7 +450,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { replicaCounts = []int32{1} // Shared-draft topology supports one target engine. } for _, replicas := range replicaCounts { - scaled := source.DeepCopy() + scaled := dgd.DeepCopy() metav1.SetMetaDataAnnotation(&scaled.ObjectMeta, "unrelated", "changed") lpx.ServingComponent(scaled).Replicas = ptr.To(replicas) scaledChild := child.DeepCopy() @@ -290,40 +461,39 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.NoError(t, err) scaledPlan, err := workload.PlanNodeLocalMaterialization(plan.PodCliqueSetName) require.NoError(t, err) - pcs, resources, err := renderPodCliqueSet(t.Context(), scaled, config, - &controller_common.RuntimeConfig{}, nil, workload, scaledPlan, scaledChild) + pcs, resources, err := r.renderPodCliqueSet(t.Context(), scaledChild, scaled, workload, scaledPlan) require.NoError(t, err) require.Equal(t, firstResources, resources) require.Equal(t, first.Spec.Template.Cliques, pcs.Spec.Template.Cliques) - require.Equal(t, replicas, *pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas) + require.Equal(t, first.Spec.Template.PodCliqueScalingGroupConfigs, pcs.Spec.Template.PodCliqueScalingGroupConfigs) } t.Log("Connect every model projection to its rendered Agent while preserving shared runtime metadata and replica units") require.Len(t, first.Spec.Template.PodCliqueScalingGroupConfigs, 1) group := first.Spec.Template.PodCliqueScalingGroupConfigs[0] - require.Equal(t, component.Replicas, group.Replicas) + require.Equal(t, ptr.To(ptr.Deref(component.MinAvailable, 1)), group.Replicas) require.Equal(t, component.MinAvailable, group.MinAvailable) var conductor, gpuClique *grovev1alpha1.PodCliqueTemplateSpec serving := 0 for index, clique := range first.Spec.Template.Cliques { require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName, "role %s must use the LPX backend", clique.Name) - require.Equal(t, "lpx-queue", clique.Labels[commonconsts.KubeLabelKaiSchedulerQueue]) - if clique.Labels[dynamo.LPXServingLabel] == commonconsts.KubeLabelValueTrue { + require.Equal(t, "lpx-queue", clique.Labels[consts.KubeLabelKaiSchedulerQueue]) + if clique.Labels[dynamo.LPXServingLabel] == consts.KubeLabelValueTrue { serving++ require.NotEqual(t, lpxv1alpha1.PodRoleAgent, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) - require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) + require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[consts.KubeLabelDynamoBaseModelHash]) } else { - require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) - require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) + require.NotContains(t, clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) + require.NotContains(t, clique.Labels, consts.KubeLabelDynamoBaseModelHash) } require.Equal(t, ptr.To(clique.Spec.Replicas), clique.Spec.MinAvailable) require.NotEmpty(t, clique.Spec.PodSpec.Containers) require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_DGD_ENV", Value: test.name}) - require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: commonconsts.DynamoNamespaceEnvVar, Value: commonconsts.GlobalDynamoNamespace}) - require.Equal(t, commonconsts.GlobalDynamoNamespace, clique.Labels[commonconsts.KubeLabelDynamoNamespace]) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: consts.DynamoNamespaceEnvVar, Value: consts.GlobalDynamoNamespace}) + require.Equal(t, consts.GlobalDynamoNamespace, clique.Labels[consts.KubeLabelDynamoNamespace]) require.Equal(t, "from-component", clique.Labels["shared-label"]) - require.Equal(t, "false", clique.Annotations[commonconsts.KubeAnnotationEnableMetrics]) - require.Equal(t, "kubernetes", clique.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend]) + require.Equal(t, "false", clique.Annotations[consts.KubeAnnotationEnableMetrics]) + require.Equal(t, "kubernetes", clique.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend]) if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != lpxv1alpha1.PodRoleCyborgWorker || test.wantFamily == lpxv1alpha1.TargetFamilyXt8888 { t.Log("Bind the generated immutable ConfigMap alongside authored volumes and mounts") configIndex := slices.IndexFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "config" }) @@ -379,7 +549,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { first.Annotations["shared-annotation"] = "rendered" require.Equal(t, "source", second.Labels["shared-label"]) require.Equal(t, "source", second.Annotations["shared-annotation"]) - require.Equal(t, before, source) + require.Equal(t, before, dgd) require.Equal(t, childBefore, child) }) } @@ -390,15 +560,15 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { registry := newTestDataModelRegistry(t, t.TempDir()) payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml") require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - for _, component := range lpx.Components(source) { + dgd := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, yaml.Unmarshal(payload, dgd)) + for _, component := range lpx.Components(dgd) { agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate agent.Spec.Containers[0].Image = component.ComponentName + "-runtime" agent.Spec.Containers[0].Command = []string{"/bin/quasar-entrypoint"} agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} } - target := lpx.ServingComponent(source) + target := lpx.ServingComponent(dgd) template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate template.Spec.Containers[0].Image = "independent-conductor-runtime" template.Spec.Containers[0].Command = []string{"/bin/nova"} @@ -410,16 +580,15 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"conductor"}}}, }}}, }} - before := source.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, registry) + before := dgd.DeepCopy() + selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, source))) + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) t.Log("Render one shared conductor without changing either component's Agent template") - pcs, _, err := renderPodCliqueSet(t.Context(), source, - &configv1alpha1.OperatorConfiguration{}, - &controller_common.RuntimeConfig{}, nil, selected, plan, newLPXRenderDeployment(t, source)) + r := &graphReconciler{config: &configv1alpha1.OperatorConfiguration{}, runtimeConfig: &commoncontroller.RuntimeConfig{}} + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) require.NoError(t, err) t.Log("Keep Nova separate from Quasar and preserve each role's metadata and storage") @@ -446,7 +615,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { require.Contains(t, container.Env, corev1.EnvVar{Name: "CONDUCTOR_ONLY", Value: "kept"}) require.NotContains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) } else { - component := clique.Labels[commonconsts.KubeLabelDynamoComponent] + component := clique.Labels[consts.KubeLabelDynamoComponent] require.Equal(t, component+"-runtime", container.Image) require.Equal(t, []string{"/bin/quasar-entrypoint"}, container.Command) require.Contains(t, container.Env, corev1.EnvVar{Name: "AGENT_ONLY", Value: "kept"}) @@ -454,7 +623,85 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { } } require.Equal(t, 1, conductors) - require.Equal(t, before, source) + require.Equal(t, before, dgd) +} + +func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { + controllerConfig := &configv1alpha1.OperatorConfiguration{ + Orchestrators: configv1alpha1.OrchestratorConfiguration{ + Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, + }, + } + const buildID = "single-v2-manifest-defaults/build_manifest_defaults" + registryRoot := t.TempDir() + writeTestGraphBuild(t, registryRoot, buildID, testGbuildManifestCapnp(t, buildID)) + registry, err := lpx.NewModelRegistry(registryRoot, nil) + require.NoError(t, err) + + t.Log("Render runtime overrides supplied directly by the conductor template") + var partitionData map[string]string + for _, env := range []corev1.EnvVar{ + {Name: "NOVA_BATCH_SIZE", Value: "1"}, + {Name: "NOVA_SEQUENCE_LENGTH", Value: "65536"}, + {Name: "A_CUSTOM_MODEL_PATH", Value: "$(LPX_MODEL_PATH)"}, + } { + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml") + require.NoError(t, err) + var deployment v1beta1.DynamoGraphDeployment + require.NoError(t, yaml.Unmarshal(payload, &deployment)) + component := &deployment.Spec.Components[0] + component.LPX = &v1beta1.LPXConfig{BuildID: buildID} + template := component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + template.Spec.Containers[0].Env = []corev1.EnvVar{env} + + selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) + require.NoError(t, err) + child := newLPXRenderDeployment(t, &deployment) + plan := mustPlanSelectedLPX(t, &deployment, selected) + r := &graphReconciler{config: controllerConfig, runtimeConfig: &commoncontroller.RuntimeConfig{}} + pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, &deployment, selected, plan) + require.NoError(t, err) + + t.Log("Keep authored environment values without changing shared partition files") + var hash string + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Name == plan.ConductorTemplate { + require.Equal(t, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}, clique.Spec.PodSpec.Containers[0].Env[0]) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, env) + hash = clique.Annotations[consts.AnnotationExtraResourcesHash] + } + } + require.NotEmpty(t, hash) + configMap := getResource[*corev1.ConfigMap](t, resources, lpx.LPUConfigMapName(dynamo.PCSNameForLPX(child), hash)) + require.NotContains(t, configMap.Data, "model_config.toml") + require.NotContains(t, configMap.Data, "datacenter.toml") + if partitionData == nil { + partitionData = configMap.Data + } else { + require.Equal(t, partitionData, configMap.Data) + } + } +} + +func getResource[T any](t *testing.T, resources []client.Object, name string) T { + t.Helper() + i := slices.IndexFunc(resources, func(resource client.Object) bool { return resource.GetName() == name }) + require.GreaterOrEqual(t, i, 0) + return resources[i].(T) +} + +func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { + t.Helper() + plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) + require.NoError(t, err) + return plan +} + +func newLPXRenderDeployment(t *testing.T, dgd *v1beta1.DynamoGraphDeployment) *v1alpha1.LPXGraphDeployment { + t.Helper() + deployment := newLPXTestDeployment(t, dgd) + deployment.UID = "lpx-render-uid" + return deployment } func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegistry { @@ -477,18 +724,6 @@ func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegist compilationMode: manifestcapnpv2.CompilationMode_lpx, nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, }, - "node-local-v2-cpu-embeddings": { - topology: "URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA", - partitionCount: 3, - numChips: 16, - devicesPerNode: 8, - selectedPropSyncChain: []uint32{0, 1, 2}, - compilationMode: manifestcapnpv2.CompilationMode_lpx, - nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, - standaloneTokenEmbeddings: true, - supportsCPUEmbeddings: true, - runtimeTokenEmbeddingsPath: "runtime/text_embeddings.npz", - }, "llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c": { topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", partitionCount: 17, @@ -526,28 +761,6 @@ func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegist return registry } -type testV2GraphManifestFixture struct { - topology string - partitionCount int - numChips uint32 - devicesPerNode uint32 - selectedPropSyncChain []uint32 - compilationMode manifestcapnpv2.CompilationMode - nonLPUDeviceTypes []manifestcapnpv2.DeviceType - standaloneTokenEmbeddings bool - supportsCPUEmbeddings bool - runtimeTokenEmbeddingsPath string -} - -func writeTestGraphBuild(t *testing.T, registryRoot, buildID string, manifest []byte) { - t.Helper() - - buildDir := filepath.Join(registryRoot, buildID) - require.NoError(t, os.MkdirAll(buildDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), manifest, 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.json"), []byte(`{}`), 0o600)) -} - func newTestGraphManifest(t *testing.T) (*capnp.Message, manifestcapnpv2.Manifest) { t.Helper() @@ -567,16 +780,6 @@ func newTestGraphManifest(t *testing.T) (*capnp.Message, manifestcapnpv2.Manifes return message, manifest } -func populateTestGraphBuild(t *testing.T, manifest manifestcapnpv2.Manifest, wireBuildID string) { - t.Helper() - - t.Log("Populate the compiler-owned build identity used only as opaque provenance") - build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) - require.NoError(t, err) - require.NoError(t, build.SetText(11, wireBuildID)) - require.NoError(t, manifest.SetReserved3(build.ToPtr())) -} - func newTestGraphProgram( t *testing.T, manifest manifestcapnpv2.Manifest, @@ -606,6 +809,83 @@ func newTestGraphProgram( return deployment, program } +func populateTestGraphBuild(t *testing.T, manifest manifestcapnpv2.Manifest, wireBuildID string) { + t.Helper() + + t.Log("Populate the compiler-owned build identity used only as opaque provenance") + build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) + require.NoError(t, err) + require.NoError(t, build.SetText(11, wireBuildID)) + require.NoError(t, manifest.SetReserved3(build.ToPtr())) +} + +func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { + t.Helper() + + const topology = "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106" + + msg, seg := capnp.NewSingleSegmentMessage(nil) + manifest, err := manifestcapnpv2.NewRootManifest(seg) + require.NoError(t, err) + manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) + populateTestGraphBuild(t, manifest, filepath.Base(registryDir)) + + model, err := manifest.NewModel() + require.NoError(t, err) + arch, err := model.NewArch() + require.NoError(t, err) + arch.SetNLayers(24) + arch.SetVocabSize(201088) + swa, err := arch.NewSwa() + require.NoError(t, err) + swa.SetMaxSeqLen(128) + swa.SetPadding(0) + swaNumUsers, err := swa.NewNumUsers() + require.NoError(t, err) + swaNumUsers.SetValue(8) + tokenizer, err := model.NewTokenizer() + require.NoError(t, err) + require.NoError(t, tokenizer.SetPath("tokenizer")) + stopTokens, err := tokenizer.NewStopTokens(3) + require.NoError(t, err) + stopTokens.Set(0, 200002) + stopTokens.Set(1, 199999) + stopTokens.Set(2, 200012) + + deployment, program := newTestGraphProgram(t, manifest, manifestcapnpv2.CompilationMode_lpuOnly, 1, 131072) + _, err = deployment.NewSelectedPropSyncChains(0) + require.NoError(t, err) + dkvc, err := program.NewDkvc() + require.NoError(t, err) + dkvc.SetNumBlocksPerKvCache(256) + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + program.SetBatchFolding(false) + program.SetSupportsCpuEmbeddings(true) + program.SetSwaChunked(false) + program.SetNumSwaDkvcBlocks(1) + + artifacts, err := manifest.NewArtifacts() + require.NoError(t, err) + partitions, err := artifacts.NewPartitions(1) + require.NoError(t, err) + partition := partitions.At(0) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(0) + lpuDetail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, lpuDetail.SetPath("assemble/part-0")) + require.NoError(t, lpuDetail.SetTopology(topology)) + lpuDetail.SetNumChips(8) + lpuDetail.SetDevicesPerNode(8) + + data, err := msg.Marshal() + require.NoError(t, err) + return data +} + func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphManifestFixture) []byte { t.Helper() @@ -624,16 +904,9 @@ func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphM } program.SetNumKvCaches(1) program.SetNumBatchSplitDivisions(1) - program.SetSupportsCpuEmbeddings(fixture.supportsCPUEmbeddings) - program.SetStandaloneTokenEmbeddings(fixture.standaloneTokenEmbeddings) artifacts, err := manifest.NewArtifacts() require.NoError(t, err) - if fixture.runtimeTokenEmbeddingsPath != "" { - runtimeAssets, err := artifacts.NewRuntimeAssets() - require.NoError(t, err) - require.NoError(t, runtimeAssets.SetTokenEmbeddingsPath(fixture.runtimeTokenEmbeddingsPath)) - } partitions, err := artifacts.NewPartitions(int32(fixture.partitionCount + len(fixture.nonLPUDeviceTypes))) require.NoError(t, err) for index := range fixture.partitionCount { @@ -671,16 +944,14 @@ func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphM return data } -func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { - t.Helper() - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) - require.NoError(t, err) - return plan -} - -type testV3GraphManifestFixture struct { - compilationMode manifestcapnpv2.CompilationMode - nonLPUDeviceTypes []manifestcapnpv2.DeviceType +type testV2GraphManifestFixture struct { + topology string + partitionCount int + numChips uint32 + devicesPerNode uint32 + selectedPropSyncChain []uint32 + compilationMode manifestcapnpv2.CompilationMode + nonLPUDeviceTypes []manifestcapnpv2.DeviceType } func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphManifestFixture) []byte { @@ -729,878 +1000,109 @@ func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphM return data } -func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { - controllerConfig := &configv1alpha1.OperatorConfiguration{ - Orchestrators: configv1alpha1.OrchestratorConfiguration{ - Grove: configv1alpha1.GroveConfiguration{TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}}, - }, - } - const buildID = "single-v2-manifest-defaults/build_manifest_defaults" - registryRoot := t.TempDir() - writeTestGraphBuild(t, registryRoot, buildID, testGbuildManifestCapnp(t, buildID)) - registry, err := lpx.NewModelRegistry(registryRoot, nil) - require.NoError(t, err) - - t.Log("Render runtime overrides supplied directly by the conductor template") - var partitionData map[string]string - for _, env := range []corev1.EnvVar{ - {Name: "NOVA_BATCH_SIZE", Value: "1"}, - {Name: "NOVA_SEQUENCE_LENGTH", Value: "65536"}, - {Name: "A_CUSTOM_MODEL_PATH", Value: "$(LPX_MODEL_PATH)"}, - } { - payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml") - require.NoError(t, err) - var deployment v1beta1.DynamoGraphDeployment - require.NoError(t, yaml.Unmarshal(payload, &deployment)) - component := &deployment.Spec.Components[0] - component.LPX = &v1beta1.LPXConfig{BuildID: buildID} - template := component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate - template.Spec.Containers[0].Env = []corev1.EnvVar{env} - - selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) - require.NoError(t, err) - child := newLPXRenderDeployment(t, &deployment) - plan := mustPlanSelectedLPX(t, &deployment, selected) - pcs, resources, err := renderPodCliqueSet(t.Context(), &deployment, controllerConfig, - &controller_common.RuntimeConfig{}, nil, selected, plan, child) - require.NoError(t, err) - - t.Log("Keep authored environment values without changing shared partition files") - var hash string - for _, clique := range pcs.Spec.Template.Cliques { - if clique.Name == plan.ConductorTemplate { - require.Equal(t, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}, clique.Spec.PodSpec.Containers[0].Env[0]) - require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, env) - hash = clique.Annotations[commonconsts.AnnotationExtraResourcesHash] - } - } - require.NotEmpty(t, hash) - configMap := getResource[*corev1.ConfigMap](t, resources, lpx.LPUConfigMapName(dynamo.PCSNameForLPX(child), hash)) - require.NotContains(t, configMap.Data, "model_config.toml") - require.NotContains(t, configMap.Data, "datacenter.toml") - if partitionData == nil { - partitionData = configMap.Data - } else { - require.Equal(t, partitionData, configMap.Data) - } - } +type testV3GraphManifestFixture struct { + compilationMode manifestcapnpv2.CompilationMode + nonLPUDeviceTypes []manifestcapnpv2.DeviceType } -func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { +func writeTestGraphBuild(t *testing.T, registryRoot, buildID string, manifest []byte) { t.Helper() - const topology = "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106" - - msg, seg := capnp.NewSingleSegmentMessage(nil) - manifest, err := manifestcapnpv2.NewRootManifest(seg) - require.NoError(t, err) - manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) - populateTestGraphBuild(t, manifest, filepath.Base(registryDir)) - - model, err := manifest.NewModel() - require.NoError(t, err) - arch, err := model.NewArch() - require.NoError(t, err) - arch.SetNLayers(24) - arch.SetVocabSize(201088) - swa, err := arch.NewSwa() - require.NoError(t, err) - swa.SetMaxSeqLen(128) - swa.SetPadding(0) - swaNumUsers, err := swa.NewNumUsers() - require.NoError(t, err) - swaNumUsers.SetValue(8) - tokenizer, err := model.NewTokenizer() - require.NoError(t, err) - require.NoError(t, tokenizer.SetPath("tokenizer")) - stopTokens, err := tokenizer.NewStopTokens(3) - require.NoError(t, err) - stopTokens.Set(0, 200002) - stopTokens.Set(1, 199999) - stopTokens.Set(2, 200012) - - deployment, program := newTestGraphProgram(t, manifest, manifestcapnpv2.CompilationMode_lpuOnly, 1, 131072) - _, err = deployment.NewSelectedPropSyncChains(0) - require.NoError(t, err) - dkvc, err := program.NewDkvc() - require.NoError(t, err) - dkvc.SetNumBlocksPerKvCache(256) - program.SetNumKvCaches(1) - program.SetNumBatchSplitDivisions(1) - program.SetBatchFolding(false) - program.SetSupportsCpuEmbeddings(true) - program.SetSwaChunked(false) - program.SetNumSwaDkvcBlocks(1) + buildDir := filepath.Join(registryRoot, buildID) + require.NoError(t, os.MkdirAll(buildDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), manifest, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.json"), []byte(`{}`), 0o600)) +} - artifacts, err := manifest.NewArtifacts() - require.NoError(t, err) - partitions, err := artifacts.NewPartitions(1) - require.NoError(t, err) - partition := partitions.At(0) - ref, err := partition.NewPartition() - require.NoError(t, err) - ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) - ref.SetPartitionId(0) - lpuDetail, err := partition.Detail().NewLpu() +func renderLPXTestPodCliqueSet( + t *testing.T, ctx context.Context, reconciler *graphReconciler, + deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, desired *lpxTestWorkload, +) *grovev1alpha1.PodCliqueSet { + t.Helper() + rendered, _, err := reconciler.renderPodCliqueSet(ctx, deployment, dgd, desired.workload, desired.plan) require.NoError(t, err) - require.NoError(t, lpuDetail.SetPath("assemble/part-0")) - require.NoError(t, lpuDetail.SetTopology(topology)) - lpuDetail.SetNumChips(8) - lpuDetail.SetDevicesPerNode(8) + return rendered +} - data, err := msg.Marshal() +func newLPXTestRegistryWithPartitionsAndMode( + t *testing.T, + buildID string, + partitionIDs []int, + compilationMode manifestcapnpv2.CompilationMode, +) lpx.ModelRegistry { + t.Helper() + root := t.TempDir() + writeLPXTestBuild(t, root, buildID, partitionIDs, compilationMode) + registryURL := (&url.URL{Scheme: lpx.BuildSchemeFile, Path: root}).String() + registry, err := lpx.NewModelRegistry(registryURL, nil) require.NoError(t, err) - return data + return registry } -// Update this golden with: -// go test ./internal/controller/lpx -run TestGenerateGrovePodCliqueSet_FromDGDYaml -count=1 -args -update -func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { - controllerConfig := &configv1alpha1.OperatorConfiguration{ - Infrastructure: configv1alpha1.InfrastructureConfiguration{ - ETCDAddress: "etcd-address", - NATSAddress: "nats-address", - }, - Orchestrators: configv1alpha1.OrchestratorConfiguration{ - Grove: configv1alpha1.GroveConfiguration{ - TerminationDelay: metav1.Duration{Duration: 15 * time.Minute}, - }, - }, - LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - } - - registryRoot := t.TempDir() - testdataModelRegistry := newTestDataModelRegistry(t, registryRoot) - kubeClient := fake.NewClientBuilder().Build() - runtimeConfig := &controller_common.RuntimeConfig{} - - tests := []string{ - "from_dgd_yaml/lpx_v2_vllm", - "from_dgd_yaml/node-local-v2-lpu-only", - "from_dgd_yaml/node-local-v2-hybrid", - "from_dgd_yaml/node-local-v2-specdecode", - "from_dgd_yaml/node-local-v3-hx-lpu-only", - "from_dgd_yaml/node-local-v3-hx-specdecode", - "from_dgd_yaml/node-local-v3-hx-hybrid", - "from_dgd_yaml/single_v2", - } - - for _, name := range tests { - t.Run(name, func(t *testing.T) { - t.Log("Load the authored input before selection") - b, err := os.ReadFile("../../dynamo/lpx/testdata/" + name + ".input.yaml") - require.NoError(t, err) - - var dynamoDeployment v1beta1.DynamoGraphDeployment - require.NoError(t, yaml.Unmarshal(b, &dynamoDeployment)) - if dynamoDeployment.UID == "" { - dynamoDeployment.UID = "11111111-1111-4111-8111-111111111111" - } - if dynamoDeployment.Generation == 0 { - dynamoDeployment.Generation = 1 - } - selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) - require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment))) - require.NoError(t, err) - t.Log("Render all LPX roles with the separate LPX PCS identity") - got, extraResources, err := renderPodCliqueSet( - t.Context(), &dynamoDeployment, controllerConfig, runtimeConfig, - nil, selected, - plan, - newLPXRenderDeployment(t, &dynamoDeployment), - ) - require.NoError(t, err) - for _, clique := range got.Spec.Template.Cliques { - require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) - component := dynamoDeployment.GetComponentByName(clique.Labels[commonconsts.KubeLabelDynamoComponent]) - require.NotNil(t, component) - require.True(t, component.IsLPX()) - require.Equal(t, component.ComponentName, clique.Labels[lpx.StageLabel]) - } - - t.Log("Render conventional components independently from their ordinary-only input") - ordinary := dynamoDeployment.DeepCopy() - ordinary.Spec.Components = slices.DeleteFunc(ordinary.Spec.Components, func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { - return component.IsLPX() - }) - normal, err := dynamo.GenerateGrovePodCliqueSet( - t.Context(), ordinary, controllerConfig, runtimeConfig, - kubeClient, nil, nil, nil, nil, - ) - require.NoError(t, err) - require.NotEqual(t, normal.Name, got.Name) - for _, clique := range normal.Spec.Template.Cliques { - require.NotEqual(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) - require.NotEqual(t, selected.LPXComponentName(), clique.Labels[commonconsts.KubeLabelDynamoComponent]) - } - podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} - if len(normal.Spec.Template.Cliques) > 0 { - podCliqueSets = append(podCliqueSets, normal) - } - - for _, pcs := range podCliqueSets { - sort.Slice(pcs.Spec.Template.Cliques, func(i, j int) bool { - return pcs.Spec.Template.Cliques[i].Name < pcs.Spec.Template.Cliques[j].Name - }) - for _, clique := range pcs.Spec.Template.Cliques { - for i := range clique.Spec.PodSpec.Containers { - slices.SortFunc(clique.Spec.PodSpec.Containers[i].Env, func(a, b corev1.EnvVar) int { - return strings.Compare(a.Name, b.Name) - }) - } - } - extraResources = append(extraResources, pcs) - } - - var out []string - for _, resource := range extraResources { - b, err := yaml.Marshal(resource) - if err != nil { - t.Errorf("Marshal() error = %v", err) - } - out = append(out, string(b)) - } +func writeLPXTestBuild( + t *testing.T, + root string, + buildID string, + partitionIDs []int, + compilationMode manifestcapnpv2.CompilationMode, +) { + t.Helper() + require.NotEmpty(t, partitionIDs) + buildDir := filepath.Join(root, buildID) + require.NoError(t, os.Mkdir(buildDir, 0o700)) - t.Log("Normalize temporary paths and their content hashes only at the golden boundary; incorrect hash annotations still fail") - replacements := []string{registryRoot, "/testdata"} - for _, resource := range extraResources { - config, ok := resource.(*corev1.ConfigMap) - if !ok { - continue - } - actualHash := lpx.LPUConfigMapHash(config) - normalized := config.DeepCopy() - for key, value := range normalized.Data { - normalized.Data[key] = strings.ReplaceAll(value, registryRoot, "/testdata") - } - normalizedHash := lpx.LPUConfigMapHash(normalized) - replacements = append(replacements, actualHash, normalizedHash, actualHash[:16], normalizedHash[:16]) - } - goldenPath, err := filepath.Abs("../../dynamo/lpx/testdata/" + name + ".yaml") - require.NoError(t, err) - const header = "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" + - "# SPDX-License-Identifier: Apache-2.0\n\n" - golden.Assert(t, header+strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) - }) - } -} + t.Log("Encode the V2 manifest header and the tokenizer consumed by rendered runtimes") + message, manifest := newTestGraphManifest(t) -func TestGenerateGrovePodCliqueSet_ImplicitV2HybridPreservesAgentRuntime(t *testing.T) { - t.Log("Load the pre-scheduler V2 hybrid DGD shape without scheduler intent and with an authored runtime command") - payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) - require.NoError(t, err) - dgd := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, dgd)) - delete(dgd.Annotations, commonconsts.KubeAnnotationLPXSchedulerBackend) - dgd.Spec.Scheduling = nil - lpxComponent := dgd.GetComponentByName("lpu") - require.NotNil(t, lpxComponent) - require.NotNil(t, lpxComponent.LPX) - lpxComponent.LPX.BuildID = "node-local-v2-cpu-embeddings" - lpxComponent.ModelRef = &v1beta1.ModelReference{Name: "test/model"} - - t.Log("Author the Cyborg batch while keeping its sidecar separate") - cyborgRole := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXConductor) - require.NotNil(t, cyborgRole) - authoredCyborgMain := &cyborgRole.PodTemplate.Spec.Containers[0] - require.Equal(t, commonconsts.MainContainerName, authoredCyborgMain.Name) - authoredCyborgMain.Env = append( - authoredCyborgMain.Env, - corev1.EnvVar{Name: "CYBORG_BATCH_SIZE", Value: "2"}, - corev1.EnvVar{Name: "USER_BATCH_REF", Value: "$(CYBORG_BATCH_SIZE)"}, - ) - - t.Log("Keep a sidecar named cyborg separate from the main runtime container") - sidecar := corev1.Container{Name: "cyborg", Image: "sidecar", Args: []string{"observe"}} - cyborgRole.PodTemplate.Spec.Containers = append(cyborgRole.PodTemplate.Spec.Containers, sidecar) - - t.Log("Customize Agent metadata and placement with an image ENTRYPOINT and authored arguments") - agentTemplate := lpxComponent.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate - agentMain := &agentTemplate.Spec.Containers[0] - require.NotEmpty(t, agentMain.Command) - require.NotEmpty(t, agentMain.Args) - agentMain.Command = nil - agentMain.Args = []string{"--instance-model-name", "test/model", "--agent-env-vars", "PIPELINE_ENV=kept"} - agentMain.Env = append(agentMain.Env, - corev1.EnvVar{Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, - corev1.EnvVar{Name: "MAIN_CPU", ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: commonconsts.MainContainerName, Resource: "requests.cpu", - }}}, - ) - agentTemplate.Spec.Tolerations = []corev1.Toleration{ - {Key: "cluster.example/custom", Operator: corev1.TolerationOpExists}, - {Key: "lpu.nvidia.com/node", Operator: corev1.TolerationOpExists}, - } - agentMain.VolumeMounts = append(agentMain.VolumeMounts, - corev1.VolumeMount{Name: "config", MountPath: "/custom", ReadOnly: true}, - ) - customConfigVolume := corev1.Volume{ - Name: "config", - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "authored-runtime-config"}, - DefaultMode: ptr.To[int32](0o440), - }}, - } - agentTemplate.Spec.Volumes = append(agentTemplate.Spec.Volumes, customConfigVolume) + t.Log("Retain the opaque compiler provenance without the removed V1 publication contract") + populateTestGraphBuild(t, manifest, buildID) - t.Log("Project the scheduler-defaulted workload and preserve its conductorless LPU runtime") - registry := newTestDataModelRegistry(t, t.TempDir()) - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) - require.NoError(t, err) - projections := selected.ModelProjections() - require.Len(t, projections, 1) - projection := projections[0] - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) + t.Log("Describe the same one-batch runtime with explicit V2 runtime I/O and prop-sync evidence") + deployment, program := newTestGraphProgram(t, manifest, compilationMode, uint32(len(partitionIDs)*2), 8192) + program.SetNumKvCaches(1) + program.SetNumBatchSplitDivisions(1) + chains, err := deployment.NewSelectedPropSyncChains(1) require.NoError(t, err) - require.Empty(t, plan.ConductorTemplate) - require.Len(t, plan.Agents, 1) - require.Equal(t, 6, plan.Agents[0].Replicas) - request := projection.RequestSpec(plan, "test") - require.Len(t, request.Partitions, 3) - for index, partition := range request.Partitions { - require.Equal(t, int64(index), partition.CompilerPartitionID) - } - - controllerConfig := &configv1alpha1.OperatorConfiguration{ - Infrastructure: configv1alpha1.InfrastructureConfiguration{ETCDAddress: "etcd", NATSAddress: "nats"}, - Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, - LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - } - pcs, extraResources, err := renderPodCliqueSet( - t.Context(), dgd, controllerConfig, &controller_common.RuntimeConfig{}, - nil, selected, plan, newLPXRenderDeployment(t, dgd), - ) + selectedIDs, err := chains.At(0).NewPartitionIds(int32(len(partitionIDs))) require.NoError(t, err) - - t.Log("Verify LPX places the worker while retaining the authored process contract") - var agent, cyborg *grovev1alpha1.PodCliqueTemplateSpec - serving := 0 - for _, clique := range pcs.Spec.Template.Cliques { - if clique.Labels[dynamo.LPXServingLabel] == commonconsts.KubeLabelValueTrue { - serving++ - require.Equal(t, lpxv1alpha1.PodRoleCyborgWorker, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) - require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) - } else { - require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoDiscoveryEnabled) - require.NotContains(t, clique.Labels, commonconsts.KubeLabelDynamoBaseModelHash) - } - switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { - case lpxv1alpha1.PodRoleConductor: - t.Fatalf("implicit V2 hybrid runtime unexpectedly rendered conductor %q", clique.Name) - case lpxv1alpha1.PodRoleAgent: - agent = clique - case lpxv1alpha1.PodRoleCyborgWorker: - cyborg = clique - } - } - require.NotNil(t, agent) - require.NotNil(t, cyborg) - require.Equal(t, 1, serving) - require.Equal(t, lpx.SchedulerName, agent.Spec.PodSpec.SchedulerName) - require.Equal(t, int32(plan.Agents[0].Replicas), agent.Spec.Replicas) - require.Contains(t, cyborg.Spec.StartsAfter, agent.Name) - require.NotContains(t, cyborg.Spec.StartsAfter, "cond") - - main := agent.Spec.PodSpec.Containers[0] - require.Equal(t, "agent", main.Name) - require.Nil(t, main.Command) - require.Equal(t, agentMain.Args, main.Args) - require.NotNil(t, main.SecurityContext) - require.True(t, *main.SecurityContext.Privileged) - require.False(t, main.Stdin) - require.False(t, main.TTY) - require.NotNil(t, main.StartupProbe) - require.NotNil(t, main.ReadinessProbe) - runtimeEnv := make(map[string]string, len(main.Env)) - for _, variable := range main.Env { - runtimeEnv[variable.Name] = variable.Value - } - require.Equal(t, "agent", runtimeEnv["CONTAINER_NAME"]) - require.Equal(t, "agent", envValueSource(main.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) - require.Equal(t, "1024", runtimeEnv["GLUE_RDMA_PATH_MTU"]) - require.Equal(t, "mlx5_0", runtimeEnv["NIC_NAME"]) - require.Equal(t, "19878", runtimeEnv["READINESS_PORT"]) - require.Equal(t, "metadata.annotations['"+lpxv1alpha1.PodModelAnnotation+"']", envValueSource(main.Env, "LPU_MODEL_NAME").FieldRef.FieldPath) - require.Equal(t, "/nfs/node-local-v2-cpu-embeddings", runtimeEnv["LPX_MODEL_PATH"]) - require.Equal(t, "$(LPX_MODEL_PATH)", runtimeEnv["GAS_DIR"]) - lpuConfigName := lpx.LPUConfigMapName(plan.PodCliqueSetName, agent.Annotations[commonconsts.AnnotationExtraResourcesHash]) - require.NotContains(t, runtimeEnv, "TOPOLOGIES") - podIPSource := envValueSource(main.Env, "POD_IP") - require.NotNil(t, podIPSource) - require.NotNil(t, podIPSource.FieldRef) - require.Equal(t, "status.podIP", podIPSource.FieldRef.FieldPath) - require.NotContains(t, runtimeEnv, "PARTITION_ID") - require.NotContains(t, runtimeEnv, "RANK_IN_PARTITION") - - t.Log("Preserve authored Agent tolerations, networking and storage") - require.True(t, agent.Spec.PodSpec.HostIPC) - require.True(t, agent.Spec.PodSpec.HostNetwork) - require.Equal(t, corev1.DNSClusterFirstWithHostNet, agent.Spec.PodSpec.DNSPolicy) - require.Equal(t, agentTemplate.Spec.Tolerations, agent.Spec.PodSpec.Tolerations) - for _, name := range []string{"config", "hugepages", "host-dev", "host-sys", "ssh-secret"} { - require.True(t, slices.ContainsFunc(agent.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == name }), "missing volume %s", name) - } - require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "host-dev" })) - require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "host-sys" })) - require.True(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "hugepages" })) - require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom", ReadOnly: true}) - require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) - require.Contains(t, agent.Spec.PodSpec.Volumes, customConfigVolume) - require.True(t, main.Resources.Requests.Cpu().Equal(resource.MustParse("62"))) - require.True(t, main.Resources.Requests[corev1.ResourceHugePagesPrefix+"2Mi"].Equal(resource.MustParse("4096Mi"))) - - t.Log("Verify Cyborg receives its generated config and InfiniBand bindings instead of fallback PVCs") - require.Contains(t, cyborg.Spec.PodSpec.Containers, sidecar) - require.NotEmpty(t, cyborg.Spec.PodSpec.Containers) - cyborgMain := cyborg.Spec.PodSpec.Containers[0] - require.Equal(t, authoredCyborgMain.Command, cyborgMain.Command) - require.Equal(t, authoredCyborgMain.Args, cyborgMain.Args) - require.Contains(t, cyborgMain.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/configs"}) - require.True(t, slices.ContainsFunc(cyborgMain.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "infiniband" })) - - volumes := make(map[string]corev1.Volume, len(cyborg.Spec.PodSpec.Volumes)) - for _, volume := range cyborg.Spec.PodSpec.Volumes { - volumes[volume.Name] = volume - } - configVolume := volumes["config"] - require.NotNil(t, configVolume.ConfigMap) - decodeConfigName := plan.PodCliqueSetName + "-decode-" + cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash][:16] - require.Equal(t, decodeConfigName, configVolume.ConfigMap.Name) - require.Nil(t, configVolume.PersistentVolumeClaim) - infinibandVolume := volumes["infiniband"] - require.NotNil(t, infinibandVolume.HostPath) - require.Equal(t, "/dev/infiniband", infinibandVolume.HostPath.Path) - require.Nil(t, infinibandVolume.PersistentVolumeClaim) - - t.Log("Preserve authored Cyborg runtime values and their expansion order") - cyborgEnv := make(map[string]string, len(cyborgMain.Env)) - for _, variable := range cyborgMain.Env { - cyborgEnv[variable.Name] = variable.Value - } - require.Equal(t, "19877", cyborgEnv["RDMA_PORT"]) - require.Equal(t, "/tmp/lpu_servers", cyborgEnv["SERVER_HOSTS_FILE"]) - require.Equal(t, "2", cyborgEnv["CYBORG_BATCH_SIZE"]) - require.NotContains(t, cyborgEnv, "CYBORG_SWA_CACHE_IDS") - batchIndex := slices.IndexFunc(cyborgMain.Env, func(variable corev1.EnvVar) bool { - return variable.Name == "CYBORG_BATCH_SIZE" - }) - userIndex := slices.IndexFunc(cyborgMain.Env, func(variable corev1.EnvVar) bool { - return variable.Name == "USER_BATCH_REF" - }) - require.NotEqual(t, -1, batchIndex) - require.NotEqual(t, -1, userIndex) - require.Less(t, batchIndex, userIndex) - - t.Log("Keep partition zero as the collapsed runtime root for all three physical partitions") - lpuConfig := getResource[*corev1.ConfigMap](t, extraResources, lpuConfigName) - require.Equal(t, "0", lpuConfig.Data["partition_ids"]) - - t.Log("Verify generated Cyborg and LPU config resource order and contents") - decodeConfig := getResource[*corev1.ConfigMap](t, extraResources, decodeConfigName) - require.Len(t, extraResources, 2) - require.Equal(t, decodeConfigName, extraResources[0].GetName()) - require.Equal(t, lpuConfigName, extraResources[1].GetName()) - require.Equal(t, dgd.Namespace, decodeConfig.Namespace) - require.Len(t, decodeConfig.Data, 1) - require.Equal(t, []string{"lpx-${GROVE_PCSG_INDEX}-agt-0"}, strings.Split(decodeConfig.Data["lpu_servers"], "\n")) - decodeHash := lpx.LPUConfigMapHash(decodeConfig) - require.Equal(t, decodeHash, cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash]) -} - -func TestGenerateGrovePodCliqueSet_NodeLocalPreservesImageEntrypoint(t *testing.T) { - t.Log("Share the immutable build registry across entrypoint cases") - registry := newTestDataModelRegistry(t, t.TempDir()) - staticEnvNames := []string{ - "POD_IP", "POD_NAME", "POD_NAMESPACE", "POD_UID", "LPU_MODEL_NAME", "CYBORG_FPGA_GPI_REPLICA_INDEX", - "CYBORG_BATCH_SIZE", "CYBORG_FPGA_GPI_IO_FPGA_COUNT", "CYBORG_SWA_CACHE_IDS", "TOKENIZER_DIR", "TOTAL_REPLICAS", - "DYN_SYSTEM_ENABLED", "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", "DYN_SYSTEM_PORT", "DYN_HEALTH_CHECK_ENABLED", - "NIXL_TELEMETRY_ENABLE", "NIXL_TELEMETRY_EXPORTER", "NIXL_TELEMETRY_PROMETHEUS_PORT", "DYN_FORWARDPASS_METRIC_PORT", + for index, partitionID := range partitionIDs { + selectedIDs.Set(index, uint32(partitionID)) } - t.Log("Use ordinary component defaults as the shared environment and port contract") - commonDefaults, err := (&dynamo.BaseComponentDefaults{}).GetBaseContainer(dynamo.ComponentContext{}) - require.NoError(t, err) - workerDefaults, err := dynamo.NewWorkerDefaults().GetBaseContainer(dynamo.ComponentContext{}) + t.Log("Package the LPU partitions and the hybrid CUDA marker in the flat V2 artifact inventory") + artifacts, err := manifest.NewArtifacts() require.NoError(t, err) - roleDefaults := map[string]corev1.Container{ - lpxv1alpha1.PodRoleAgent: commonDefaults, lpxv1alpha1.PodRoleConductor: commonDefaults, - lpxv1alpha1.PodRoleCyborgWorker: workerDefaults, + partitionCount := len(partitionIDs) + if compilationMode == manifestcapnpv2.CompilationMode_lpx { + partitionCount++ } - modes := []struct { - name string - file string - hybrid bool - conductorConfig bool - }{ - {name: "Single V2", file: "single_v2.input.yaml", conductorConfig: true}, - {name: "V2 LPU-only", file: "node-local-v2-lpu-only.input.yaml", conductorConfig: true}, - {name: "V2 hybrid", file: "node-local-v2-hybrid.input.yaml", hybrid: true, conductorConfig: true}, - {name: "HX LPU-only", file: "node-local-v3-hx-lpu-only.input.yaml", conductorConfig: true}, - {name: "HX hybrid", file: "node-local-v3-hx-hybrid.input.yaml", hybrid: true}, - } - intents := []struct { - name string - command []string - args []string - cyborgCommand []string - cyborgArgs []string - hybridOnly bool - configPath *string - configRole int - envFrom bool - customPodSettings bool - }{ - {name: "image entrypoint and image command"}, - {name: "custom pod settings and exec health probes", command: []string{"/opt/custom-runtime"}, customPodSettings: true}, - {name: "default startup without config mount", configPath: ptr.To(""), configRole: 1}, - {name: "default startup with custom read-only config mount", configPath: ptr.To("/custom"), configRole: 1}, - {name: "default Agent without config mount", configPath: ptr.To("")}, - {name: "default Agent with custom read-only config mount", configPath: ptr.To("/custom")}, - {name: "default Agent without config mount and unrelated EnvFrom", configPath: ptr.To(""), envFrom: true}, - {name: "image entrypoint with user arguments", args: []string{"serve"}}, - {name: "explicit executable with image arguments", command: []string{"/opt/custom-runtime"}}, - {name: "explicit executable and arguments", command: []string{"/opt/custom-runtime"}, args: []string{"serve"}}, - {name: "allocation flag in arguments", args: []string{"--allocation", "template-target"}}, - {name: "allocation flag in command", command: []string{"/opt/custom-runtime", "--allocation=template-target"}}, - {name: "allocation environment expansion", args: []string{"--allocation", "$(LPX_ALLOCATION)"}}, - {name: "argument terminator", command: []string{"/opt/custom-runtime"}, args: []string{"--", "literal argument"}}, - {name: "shell command", command: []string{"/bin/sh", "-c"}, args: []string{"exec /opt/custom-runtime --allocation $LPX_ALLOCATION"}}, - { - name: "Cyborg manifest argument remains user-owned", - cyborgCommand: []string{"/opt/cyborg-runtime"}, - cyborgArgs: []string{"--gbuild-manifest", "$(GBUILD_MANIFEST_PATH)"}, - hybridOnly: true, - }, - { - name: "explicit Cyborg shell remains user-owned", - command: []string{"/opt/custom-runtime"}, - args: []string{"serve"}, - cyborgCommand: []string{"/bin/sh", "-c"}, - cyborgArgs: []string{`exec /opt/cyborg-runtime --gbuild-manifest "$GBUILD_MANIFEST_PATH"`}, - hybridOnly: true, - }, - { - name: "detected SGLang Cyborg remains selected no-op", - cyborgCommand: []string{"python3"}, - cyborgArgs: []string{"-m", "dynamo.sglang"}, - hybridOnly: true, - }, - { - name: "detected TensorRT-LLM Cyborg remains selected no-op", - cyborgCommand: []string{"python3"}, - cyborgArgs: []string{"-m", "dynamo.trtllm"}, - hybridOnly: true, - }, - } - - for _, mode := range modes { - t.Run(mode.name, func(t *testing.T) { - payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", mode.file)) - require.NoError(t, err) - - for _, intent := range intents { - if intent.hybridOnly && !mode.hybrid || intent.configPath != nil && intent.configRole == 1 && !mode.conductorConfig { - continue - } - t.Run(intent.name, func(t *testing.T) { - t.Log("Author LPX LPU and GPU runtime settings independently of fixture defaults") - dgd := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, dgd)) - component := dgd.GetComponentByName("lpu") - require.NotNil(t, component) - component.SharedMemorySize = nil - agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) - templates := []*corev1.PodTemplateSpec{agent.PodTemplate, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate} - for roleIndex, template := range templates { - main := &template.Spec.Containers[0] - require.Equal(t, commonconsts.MainContainerName, main.Name) - command := intent.command - args := intent.args - if roleIndex == 1 && intent.cyborgCommand != nil { - command = intent.cyborgCommand - args = intent.cyborgArgs - } - main.Command = slices.Clone(command) - main.Args = slices.Clone(args) - template.Spec.SecurityContext, main.SecurityContext = nil, nil - template.Spec.HostNetwork, template.Spec.HostIPC = false, false - template.Spec.HostUsers, template.Spec.DNSConfig = nil, nil - template.Spec.DNSPolicy, template.Spec.RestartPolicy = "", "" - template.Spec.TerminationGracePeriodSeconds, template.Spec.Tolerations = nil, nil - main.Ports = nil - main.Env = slices.DeleteFunc(main.Env, func(variable corev1.EnvVar) bool { return slices.Contains(staticEnvNames, variable.Name) }) - template.Spec.Volumes = slices.DeleteFunc(template.Spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == commonconsts.KubeValueNameSharedMemory }) - main.VolumeMounts = slices.DeleteFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.MountPath == commonconsts.DefaultSharedMemoryMountPath - }) - requests, limits := main.Resources.Requests, main.Resources.Limits - main.Resources.Requests, main.Resources.Limits = corev1.ResourceList{}, corev1.ResourceList{} - maps.Copy(main.Resources.Requests, requests) - maps.Copy(main.Resources.Limits, limits) - for _, resources := range []corev1.ResourceList{main.Resources.Requests, main.Resources.Limits} { - maps.DeleteFunc(resources, func(name corev1.ResourceName, _ resource.Quantity) bool { - return name != corev1.ResourceName(commonconsts.KubeResourceGPUNvidia) - }) - } - main.StartupProbe, main.LivenessProbe, main.ReadinessProbe = nil, nil, nil - if intent.customPodSettings { - component.SharedMemorySize = ptr.To(resource.MustParse("0")) - template.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(1000)), RunAsGroup: ptr.To(int64(2000)), RunAsNonRoot: ptr.To(true), - FSGroup: ptr.To(int64(3000)), FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeAlways), - SupplementalGroups: []int64{4000}, Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "600"}}, - SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, - } - main.SecurityContext = &corev1.SecurityContext{ - Privileged: ptr.To(false), RunAsUser: ptr.To(int64(1000)), RunAsGroup: ptr.To(int64(2000)), - RunAsNonRoot: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), - } - template.Spec.HostNetwork, template.Spec.HostIPC = true, true - template.Spec.HostUsers = ptr.To(true) - template.Spec.DNSPolicy = corev1.DNSNone - template.Spec.DNSConfig = &corev1.PodDNSConfig{Nameservers: []string{"192.0.2.1"}, Searches: []string{"custom.test"}} - template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure - template.Spec.TerminationGracePeriodSeconds = ptr.To(int64(17)) - template.Spec.Tolerations = []corev1.Toleration{{Key: "custom.example/runtime", Operator: corev1.TolerationOpEqual, Value: "allowed", Effect: corev1.TaintEffectNoSchedule}} - main.Ports = []corev1.ContainerPort{{Name: "custom", ContainerPort: 4321, Protocol: corev1.ProtocolUDP}} - for _, name := range staticEnvNames { - main.Env = append(main.Env, corev1.EnvVar{Name: name, Value: "authored-" + name}) - } - template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{ - Name: commonconsts.KubeValueNameSharedMemory, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("32Mi")), - }}, - }) - main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: commonconsts.KubeValueNameSharedMemory, MountPath: commonconsts.DefaultSharedMemoryMountPath}) - main.Resources.Requests[corev1.ResourceCPU] = resource.MustParse("500m") - main.Resources.Limits[corev1.ResourceCPU] = resource.MustParse("2") - for _, resources := range []corev1.ResourceList{main.Resources.Requests, main.Resources.Limits} { - resources[corev1.ResourceMemory] = resource.MustParse("128Mi") - resources[corev1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse("2Mi") - } - main.StartupProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "started"}}}, - FailureThreshold: 12, PeriodSeconds: 4, TimeoutSeconds: 2, - } - main.LivenessProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "live"}}}, - FailureThreshold: 4, PeriodSeconds: 6, TimeoutSeconds: 3, - } - main.ReadinessProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/custom-health", fmt.Sprint(roleIndex), "ready"}}}, - FailureThreshold: 2, PeriodSeconds: 5, TimeoutSeconds: 1, - } - } - template.Spec.Containers = append(template.Spec.Containers, corev1.Container{ - Name: "observer", Image: "observer:test", Command: []string{"/custom-observer"}, - Env: []corev1.EnvVar{{Name: "POD_IP", Value: "sidecar-owned"}}, - SecurityContext: &corev1.SecurityContext{RunAsUser: ptr.To(int64(2000))}, - }) - main = &template.Spec.Containers[0] - main.Env = append(main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) - if intent.envFrom && roleIndex == 0 { - main.EnvFrom = []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "unrelated-env"}}}} - } - if intent.configPath != nil && roleIndex == intent.configRole { - configIndex := slices.IndexFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.Name == "config" }) - require.NotEqual(t, -1, configIndex) - require.Equal(t, "/configs", main.VolumeMounts[configIndex].MountPath) - if *intent.configPath == "" { - main.VolumeMounts = slices.Delete(main.VolumeMounts, configIndex, configIndex+1) - } else { - main.VolumeMounts[configIndex].MountPath = *intent.configPath - main.VolumeMounts[configIndex].ReadOnly = true - } - } - } - - before := dgd.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) - require.NoError(t, err) - pcs, _, err := renderPodCliqueSet( - t.Context(), - dgd, - &configv1alpha1.OperatorConfiguration{}, - &controller_common.RuntimeConfig{}, - nil, - selected, - mustPlanSelectedLPX(t, dgd, selected), - newLPXRenderDeployment(t, dgd), - ) - require.Equal(t, before, dgd) - require.NoError(t, err) - - t.Log("Verify shared defaults remain overridable while runtime-specific omissions stay empty") - var agents, conductors, cyborgs int - for _, clique := range pcs.Spec.Template.Cliques { - pod := clique.Spec.PodSpec - main := pod.Containers[0] - authoredPod := templates[1].Spec - if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { - authoredPod = templates[0].Spec - } - authored := authoredPod.Containers[0] - if !intent.customPodSettings { - defaults := roleDefaults[clique.Annotations[lpxv1alpha1.PodRoleAnnotation]] - authoredPod.RestartPolicy = corev1.RestartPolicyAlways - authoredPod.TerminationGracePeriodSeconds = ptr.To(int64(60)) - authored.Ports = defaults.Ports - authored.Env = append(slices.Clone(defaults.Env), corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, - }}) - authoredPod.Volumes = append(slices.Clone(authoredPod.Volumes), corev1.Volume{ - Name: commonconsts.KubeValueNameSharedMemory, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, SizeLimit: ptr.To(resource.MustParse("8Gi")), - }}, - }) - authored.VolumeMounts = append(slices.Clone(authored.VolumeMounts), corev1.VolumeMount{ - Name: commonconsts.KubeValueNameSharedMemory, MountPath: commonconsts.DefaultSharedMemoryMountPath, - }) - } - require.Equal(t, authored.Command, main.Command, "%s command", clique.Name) - require.True(t, slices.Equal(authored.Args, main.Args), "%s args", clique.Name) - require.Equal(t, authored.StartupProbe, main.StartupProbe, "%s startup", clique.Name) - require.Equal(t, authored.LivenessProbe, main.LivenessProbe, "%s liveness", clique.Name) - require.Equal(t, authored.ReadinessProbe, main.ReadinessProbe, "%s readiness", clique.Name) - require.Equal(t, authored.SecurityContext, main.SecurityContext, "%s container security", clique.Name) - require.Equal(t, authoredPod.SecurityContext, pod.SecurityContext, "%s pod security", clique.Name) - require.Equal(t, authoredPod.HostNetwork, pod.HostNetwork, "%s host network", clique.Name) - require.Equal(t, authoredPod.HostIPC, pod.HostIPC, "%s host IPC", clique.Name) - require.Equal(t, authoredPod.HostUsers, pod.HostUsers, "%s host users", clique.Name) - require.Equal(t, authoredPod.DNSPolicy, pod.DNSPolicy, "%s DNS policy", clique.Name) - require.Equal(t, authoredPod.DNSConfig, pod.DNSConfig, "%s DNS config", clique.Name) - require.Equal(t, authoredPod.RestartPolicy, pod.RestartPolicy, "%s restart", clique.Name) - require.Equal(t, authoredPod.TerminationGracePeriodSeconds, pod.TerminationGracePeriodSeconds, "%s grace period", clique.Name) - require.Equal(t, authoredPod.Tolerations, pod.Tolerations, "%s tolerations", clique.Name) - require.Equal(t, authored.Ports, main.Ports, "%s ports", clique.Name) - require.Equal(t, authoredPod.InitContainers, pod.InitContainers, "%s init containers", clique.Name) - require.Equal(t, authoredPod.Containers[1:], pod.Containers[1:], "%s sidecars", clique.Name) - actualEnv := slices.DeleteFunc(slices.Clone(main.Env), func(variable corev1.EnvVar) bool { return !slices.Contains(staticEnvNames, variable.Name) }) - expectedEnv := slices.DeleteFunc(slices.Clone(authored.Env), func(variable corev1.EnvVar) bool { return !slices.Contains(staticEnvNames, variable.Name) }) - require.ElementsMatch(t, expectedEnv, actualEnv, "%s static environment", clique.Name) - actualResources := main.Resources.DeepCopy() - delete(actualResources.Requests, "lpu.nvidia.com/lpu") - delete(actualResources.Limits, "lpu.nvidia.com/lpu") - delete(actualResources.Requests, "nvidia.com/lpu") - delete(actualResources.Limits, "nvidia.com/lpu") - require.True(t, apiequality.Semantic.DeepEqual(authored.Resources, *actualResources), "%s resources: expected %v, got %v", clique.Name, authored.Resources, *actualResources) - actualVolumes := slices.DeleteFunc(slices.Clone(pod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) - expectedVolumes := slices.DeleteFunc(slices.Clone(authoredPod.Volumes), func(volume corev1.Volume) bool { return volume.Name != commonconsts.KubeValueNameSharedMemory }) - require.ElementsMatch(t, expectedVolumes, actualVolumes, "%s shared memory volumes", clique.Name) - require.ElementsMatch(t, authored.VolumeMounts, main.VolumeMounts, "%s mounts", clique.Name) - switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { - case lpxv1alpha1.PodRoleAgent: - agents++ - require.False(t, slices.ContainsFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "tmp" })) - case lpxv1alpha1.PodRoleConductor: - conductors++ - require.Contains(t, main.Env, corev1.EnvVar{ - Name: "LPX_ALLOCATION", Value: strings.Join(clique.Spec.StartsAfter, ","), - }) - require.NotContains(t, main.Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "forged-allocation"}) - case lpxv1alpha1.PodRoleCyborgWorker: - cyborgs++ - require.Contains(t, main.Env, corev1.EnvVar{ - Name: "GBUILD_MANIFEST_PATH", Value: filepath.Join("/nfs", component.LPX.BuildID, "manifest.v2.capnp.bin"), - }) - } - } - require.Positive(t, agents) - if mode.hybrid { - require.Zero(t, conductors) - require.Equal(t, 1, cyborgs) - } else { - require.Equal(t, 1, conductors) - require.Zero(t, cyborgs) - } - }) - } - }) - } -} - -func envValueSource(envs []corev1.EnvVar, name string) *corev1.EnvVarSource { - for index := range envs { - if envs[index].Name == name { - return envs[index].ValueFrom - } - } - return nil -} - -func getResource[T any](t *testing.T, resources []client.Object, name string) T { - t.Helper() - i := slices.IndexFunc(resources, func(resource client.Object) bool { return resource.GetName() == name }) - require.GreaterOrEqual(t, i, 0) - return resources[i].(T) -} - -func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { - t.Log("Load an authored hybrid engine and override its independent leader template") - payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) + partitions, err := artifacts.NewPartitions(int32(partitionCount)) require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, yaml.Unmarshal(payload, source)) - leader := lpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate - leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ - {Name: "config", MountPath: "/custom-config", ReadOnly: true}, - {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, - {Name: "model-storage", MountPath: "/nfs"}, - } - leader.Spec.Containers[0].Env = []corev1.EnvVar{ - {Name: "TOKENIZER_DIR", Value: "/custom-tokenizer"}, - {Name: "TOTAL_REPLICAS", Value: "9"}, - {Name: "SERVER_HOSTS_FILE", Value: "/custom-servers"}, - } - for _, name := range []string{"CYBORG_BATCH_SIZE", "CYBORG_FPGA_GPI_IO_FPGA_COUNT", "CYBORG_SWA_CACHE_IDS"} { - leader.Spec.Containers[0].Env = append(leader.Spec.Containers[0].Env, corev1.EnvVar{ - Name: name, ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "runtime-settings"}, Key: name, - }}, - }) + for index, partitionID := range partitionIDs { + partition := partitions.At(index) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_lpu) + ref.SetPartitionId(uint32(partitionID)) + detail, err := partition.Detail().NewLpu() + require.NoError(t, err) + require.NoError(t, detail.SetPath(fmt.Sprintf("part-%d", partitionID))) + require.NoError(t, detail.SetTopology("URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA")) + detail.SetNumChips(16) + detail.SetDevicesPerNode(8) } - authoredConfig := corev1.Volume{ - Name: "config", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + if compilationMode == manifestcapnpv2.CompilationMode_lpx { + partition := partitions.At(len(partitionIDs)) + ref, err := partition.NewPartition() + require.NoError(t, err) + ref.SetDeviceType(manifestcapnpv2.DeviceType_cuda) + ref.SetPartitionId(uint32(len(partitionIDs))) + _, err = partition.Detail().NewCuda() + require.NoError(t, err) } - leader.Spec.Volumes = append(leader.Spec.Volumes, authoredConfig) - t.Log("Render the complete LPX workload with ordinary pod spec overrides") - selected, err := lpx.ResolveSelectedWorkload(t.Context(), source, newTestDataModelRegistry(t, t.TempDir())) - require.NoError(t, err) - plan := mustPlanSelectedLPX(t, source, selected) - pcs, _, err := renderPodCliqueSet(t.Context(), source, - &configv1alpha1.OperatorConfiguration{}, - &controller_common.RuntimeConfig{}, nil, selected, - plan, newLPXRenderDeployment(t, source)) + t.Log("Publish the required V2 compiler manifest") + payload, err := message.Marshal() require.NoError(t, err) - cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { - return clique.Name == plan.CyborgTemplate - }) - require.GreaterOrEqual(t, cliqueIndex, 0) - podSpec := pcs.Spec.Template.Cliques[cliqueIndex].Spec.PodSpec - - t.Log("Keep one config volume alongside the authored InfiniBand volume") - require.Contains(t, podSpec.Volumes, leader.Spec.Volumes[0]) - var configVolumes []corev1.Volume - for _, volume := range podSpec.Volumes { - if volume.Name == authoredConfig.Name { - configVolumes = append(configVolumes, volume) - } - } - require.Equal(t, []corev1.Volume{authoredConfig}, configVolumes) - - t.Log("Preserve authored mounts and runtime environment without interpreting literal or valueFrom settings") - main := podSpec.Containers[0] - require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "config", MountPath: "/custom-config", ReadOnly: true}) - require.Contains(t, main.VolumeMounts, corev1.VolumeMount{Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}) - require.False(t, slices.ContainsFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" })) - for _, variable := range leader.Spec.Containers[0].Env { - require.Contains(t, main.Env, variable) - } - - t.Log("Preserve the authored command and host-file override without adding a launcher") - require.Equal(t, leader.Spec.Containers[0].Command, main.Command) - require.Equal(t, leader.Spec.Containers[0].Args, main.Args) + require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), payload, 0o600)) } diff --git a/deploy/operator/internal/controller/lpx/status.go b/deploy/operator/internal/controller/lpx/status.go new file mode 100644 index 000000000000..c7685d31435b --- /dev/null +++ b/deploy/operator/internal/controller/lpx/status.go @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + "unicode/utf8" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + maxConditionMessageSize = 32768 + // SchedulingFailed is durable expiry/retry evidence, separate from aggregate Ready. + schedulingFailedCondition = "SchedulingFailed" + pipelineRequestDeadlineExceededReason = "LPXSchedulingDeadlineExceeded" +) + +// updateStatus persists the updated status on the deployment. +func (r *graphReconciler) updateStatus(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, previous *v1alpha1.LPXGraphDeploymentStatus) error { + // Every component inherits the parent ready condition. + ready := meta.IsStatusConditionTrue(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) + for name, component := range deployment.Status.Components { + component.Ready = ready + deployment.Status.Components[name] = component + } + + if apiequality.Semantic.DeepEqual(previous, &deployment.Status) { + return nil + } + + if err := r.Status().Update(ctx, deployment); err != nil { + return fmt.Errorf("persist LPXGraphDeployment status: %w", err) + } + + return nil +} + +// setReadyCondition updates the deployment's Ready condition. +func setReadyCondition(deployment *v1alpha1.LPXGraphDeployment, state v1beta1.DGDState, message string) { + status, reason := metav1.ConditionFalse, v1alpha1.LPXReadyReasonPending + switch state { + case v1beta1.DGDStateSuccessful: + status, reason = metav1.ConditionTrue, v1alpha1.LPXReadyReasonReady + case v1beta1.DGDStateFailed: + reason = v1alpha1.LPXReadyReasonFailed + } + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: status, ObservedGeneration: deployment.Generation, + Reason: reason, Message: truncateConditionMessage(message), + }) +} + +// setPipelineRequestReadyCondition reports the first current failure, otherwise the first +// non-Bound request, and returns whether every desired request is Bound. +// deployment is non-nil; absent requests and stale receipts are still pending. +func setPipelineRequestReadyCondition(deployment *v1alpha1.LPXGraphDeployment, requests map[string]*lpxv1alpha1.LPUPipelineRequest) bool { + // Failures outrank pending receipts; stable name order keeps diagnostics consistent. + allBound := true + var pending *lpxv1alpha1.LPUPipelineRequestStatus + for _, name := range slices.Sorted(maps.Keys(requests)) { + request := requests[name] + // Only receipts for the request's current generation can prove Bound or failure. + var status *lpxv1alpha1.LPUPipelineRequestStatus + if request.Status != nil && request.Status.ObservedGeneration != nil && *request.Status.ObservedGeneration == request.Generation { + status = request.Status + } + + if status != nil { + switch status.Phase { + case lpxv1alpha1.RequestPhaseBound: + continue + case lpxv1alpha1.RequestPhasePending, lpxv1alpha1.RequestPhaseNoFit, + lpxv1alpha1.RequestPhasePlanned, lpxv1alpha1.RequestPhaseReserving, + lpxv1alpha1.RequestPhaseBinding, lpxv1alpha1.RequestPhaseDegraded, + lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: + // Keep the first pending receipt while checking the remaining requests for failure. + default: + setPipelineRequestPhaseCondition(deployment, status) + return false + } + } + + if allBound { + pending = status + } + allBound = false + } + + if !allBound { + setPipelineRequestPhaseCondition(deployment, pending) + } + return allBound +} + +// setPipelineRequestPhaseCondition formats a non-Bound receipt without copying it. +// deployment is non-nil; nil status means no current scheduler receipt exists. +func setPipelineRequestPhaseCondition(deployment *v1alpha1.LPXGraphDeployment, status *lpxv1alpha1.LPUPipelineRequestStatus) { + // A missing or stale receipt cannot establish scheduler progress. + if status == nil { + setReadyCondition(deployment, v1beta1.DGDStatePending, "LPUPipelineRequest is published") + return + } + + // Scheduler diagnostics describe only the selected current receipt. + var ( + state = v1beta1.DGDStatePending + summary string + ) + + switch status.Phase { + case lpxv1alpha1.RequestPhasePending: + summary = "LPX scheduler is waiting to plan the current request" + case lpxv1alpha1.RequestPhaseNoFit: + summary = "LPX scheduler found no placement for the current request" + case lpxv1alpha1.RequestPhaseUnsupported: + state = v1beta1.DGDStateFailed + summary = "LPX scheduler cannot support the current request" + case lpxv1alpha1.RequestPhasePlanned: + summary = "LPX scheduler committed a placement plan for the current request" + case lpxv1alpha1.RequestPhaseReserving: + summary = "LPX scheduler is reserving the planned LPU allocation" + case lpxv1alpha1.RequestPhaseBinding: + summary = "LPX scheduler is binding the planned workload Pods" + case lpxv1alpha1.RequestPhaseDegraded: + summary = "LPX scheduler is repairing or releasing a degraded placement" + case lpxv1alpha1.RequestPhaseReleasing: + summary = "LPX scheduler is releasing the current plan" + case lpxv1alpha1.RequestPhaseReleased: + summary = "LPX scheduler released the current plan" + default: + state = v1beta1.DGDStateFailed + summary = fmt.Sprintf("LPX scheduler reported unknown phase %q", status.Phase) + } + + setReadyCondition(deployment, state, pipelineRequestDiagnosticMessage(summary, status.Diagnostics)) +} + +// isSchedulingFailedConditionCurrent reports whether the current generation must remain +// failed until a later input revision explicitly authorizes another publication. +func isSchedulingFailedConditionCurrent(deployment *v1alpha1.LPXGraphDeployment) bool { + failed := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + return failed != nil && failed.Status == metav1.ConditionTrue && failed.Reason == pipelineRequestDeadlineExceededReason && + failed.ObservedGeneration >= deployment.Generation +} + +// schedulingFailureCoversPipelineRequests reports whether every expired scheduling cycle +// predates the recorded failure, even after an input edit. A later cycle needs +// its own durable failure before cleanup; cleanup must not consume a retry edit. +func schedulingFailureCoversPipelineRequests( + deployment *v1alpha1.LPXGraphDeployment, + requests []*lpxv1alpha1.LPUPipelineRequest, +) bool { + failed := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + if failed == nil || failed.Status != metav1.ConditionTrue || failed.Reason != pipelineRequestDeadlineExceededReason { + return false + } + if failed.LastTransitionTime.IsZero() { + return false + } + for _, request := range requests { + startedAt, known := pipelineRequestSchedulingStartedAt(request) + if !known || !startedAt.Before(failed.LastTransitionTime.Time) { + return false + } + } + return true +} + +// setSchedulingFailedCondition updates a non-nil deployment's in-memory failure. +// renew starts a new durable fence; otherwise the original failure time and +// generation survive retries. Reconcile persists this before a later pass deletes LPRs. +func setSchedulingFailedCondition(deployment *v1alpha1.LPXGraphDeployment, renew bool) { + observedGeneration := deployment.Generation + prior := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + if prior != nil && prior.Status == metav1.ConditionTrue && !renew { + observedGeneration = prior.ObservedGeneration + } + + // Even True-to-True renewal needs a fresh timestamp to cover the newly expired cycle. + if renew { + meta.RemoveStatusCondition(&deployment.Status.Conditions, schedulingFailedCondition) + } + + // Aggregate failure and retry authorization have independent condition lifetimes. + const message = "An LPX request exceeded its scheduling deadline" + setReadyCondition(deployment, v1beta1.DGDStateFailed, message) + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: schedulingFailedCondition, Status: metav1.ConditionTrue, + ObservedGeneration: observedGeneration, Reason: pipelineRequestDeadlineExceededReason, Message: message, + }) +} + +// clearPreviousSchedulingFailure clears an old failure only for a later generation. +// deployment is non-nil; callers must not use this after observing a current expiry. +func clearPreviousSchedulingFailure(deployment *v1alpha1.LPXGraphDeployment) { + prior := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + if prior == nil || deployment.Generation <= prior.ObservedGeneration { + return + } + meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: schedulingFailedCondition, Status: metav1.ConditionFalse, + ObservedGeneration: deployment.Generation, Reason: "LPXSchedulingRetryAuthorized", + Message: "The expired request was removed and current input may be published", + }) +} + +func pipelineRequestDiagnosticMessage(summary string, diagnostics []lpxv1alpha1.StatusDiagnostic) string { + if len(diagnostics) == 0 { + return summary + } + var message strings.Builder + message.WriteString(summary) + message.WriteString(". LPX diagnostics: ") + for index, diagnostic := range diagnostics { + if index > 0 { + message.WriteString("; ") + } + message.WriteString(diagnostic.Code) + message.WriteString(" [") + message.WriteString(diagnostic.Subject) + message.WriteString("]: ") + message.WriteString(diagnostic.Detail) + + // Later diagnostics cannot change the retained prefix. + if message.Len() > maxConditionMessageSize { + break + } + } + return truncateConditionMessage(message.String()) +} + +func truncateConditionMessage(message string) string { + if len(message) <= maxConditionMessageSize { + return message + } + const suffix = "..." + last := maxConditionMessageSize - len(suffix) + for last > 0 && !utf8.ValidString(message[:last]) { + last-- + } + return message[:last] + suffix +} diff --git a/deploy/operator/internal/controller/lpx/status_test.go b/deploy/operator/internal/controller/lpx/status_test.go new file mode 100644 index 000000000000..ab8aadb50f68 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/status_test.go @@ -0,0 +1,367 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + "unicode/utf8" + + "gotest.tools/v3/golden" + "sigs.k8s.io/yaml" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestPipelineRequestDeadlineFailureSurvivesReadyChanges(t *testing.T) { + t.Log("A scheduling cycle begins after Ready last became false") + dgd := loadTestDGD(t, lpx.PipelineSingle, "build-v2") + child := newLPXTestDeployment(t, dgd) + setReadyCondition(child, v1beta1.DGDStatePending, "Waiting for scheduling") + transition := metav1.NewTime(time.Now().Add(-time.Hour).UTC().Truncate(time.Second)) + child.Status.Conditions[0].LastTransitionTime = transition + r := newLPXTestReconciler(t, nil, child, dgd) + request := newTestPipelineRequest(child, newTestPodCliqueSet(child), "expired", time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending) + previous := child.Status.DeepCopy() + + t.Log("Persist the failure independently of Ready's unchanged boolean transition") + setSchedulingFailedCondition(child, true) + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Len(t, child.Status.Conditions, 2) + require.True(t, transition.Equal(&child.Status.Conditions[0].LastTransitionTime)) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, child.Status.Conditions[0].Reason) + schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition) + require.NotNil(t, schedulingFailed) + require.True(t, schedulingFailed.LastTransitionTime.After(transition.Time)) + require.Equal(t, metav1.ConditionTrue, schedulingFailed.Status) + require.Equal(t, child.Generation, schedulingFailed.ObservedGeneration) + require.Equal(t, pipelineRequestDeadlineExceededReason, schedulingFailed.Reason) + require.Nil(t, meta.FindStatusCondition(child.Status.Conditions, "Failed")) + require.True(t, schedulingFailureCoversPipelineRequests(child, []*lpxv1alpha1.LPUPipelineRequest{request})) + failure := schedulingFailed.DeepCopy() + + t.Log("Other pending, failed, and ready observations cannot authorize a scheduling retry") + for _, state := range []v1beta1.DGDState{v1beta1.DGDStatePending, v1beta1.DGDStateFailed, v1beta1.DGDStateSuccessful} { + previous = child.Status.DeepCopy() + setReadyCondition(child, state, "An unrelated observation") + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Len(t, child.Status.Conditions, 2) + require.Equal(t, failure, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) + require.True(t, isSchedulingFailedConditionCurrent(child)) + require.True(t, schedulingFailureCoversPipelineRequests(child, []*lpxv1alpha1.LPUPipelineRequest{request})) + } + + t.Log("A later reconcile restores the deadline diagnostic without renewing cleanup authorization") + previous = child.Status.DeepCopy() + setSchedulingFailedCondition(child, false) + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Len(t, child.Status.Conditions, 2) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, child.Status.Conditions[0].Reason) + require.Equal(t, failure, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) +} + +func TestLPXReadyConditionClassifiesState(t *testing.T) { + for _, test := range []struct { + state v1beta1.DGDState + status metav1.ConditionStatus + reason string + }{ + {v1beta1.DGDStatePending, metav1.ConditionFalse, v1alpha1.LPXReadyReasonPending}, + {v1beta1.DGDStateFailed, metav1.ConditionFalse, v1alpha1.LPXReadyReasonFailed}, + {v1beta1.DGDStateSuccessful, metav1.ConditionTrue, v1alpha1.LPXReadyReasonReady}, + } { + t.Run(string(test.state), func(t *testing.T) { + t.Log("One Ready condition carries the condition and the complete diagnostic") + child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 7}} + setReadyCondition(child, test.state, "current observation") + require.Len(t, child.Status.Conditions, 1) + ready := child.Status.Conditions[0] + require.Equal(t, "Ready", ready.Type) + require.Equal(t, test.status, ready.Status) + require.Equal(t, test.reason, ready.Reason) + require.Equal(t, "current observation", ready.Message) + require.Equal(t, child.Generation, ready.ObservedGeneration) + }) + } +} + +func TestLPXStatusSkipsUnchangedObservations(t *testing.T) { + for _, state := range []v1beta1.DGDState{v1beta1.DGDStatePending, v1beta1.DGDStateSuccessful, v1beta1.DGDStateFailed} { + t.Run(string(state), func(t *testing.T) { + t.Log("Start from a persisted status with stable condition transition times") + child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "graph", Namespace: "default", Generation: 1, ResourceVersion: "1"}} + setReadyCondition(child, state, "current observation") + transition := metav1.NewTime(time.Now().Add(-time.Hour).UTC().Truncate(time.Second)) + for index := range child.Status.Conditions { + child.Status.Conditions[index].LastTransitionTime = transition + } + child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ + "engine": {Ready: state == v1beta1.DGDStateSuccessful}, + } + r := &graphReconciler{Client: newLPXTestClient(t, child)} + previous := child.Status.DeepCopy() + writes := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "status", subresource) + writes++ + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + }) + + t.Log("Message-only observations preserve transition time and an unchanged final status skips the write") + different := v1beta1.DGDStateSuccessful + if state == different { + different = v1beta1.DGDStatePending + } + setReadyCondition(child, state, "not externally visible") + setReadyCondition(child, state, "current observation") + require.Zero(t, writes) + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.Zero(t, writes) + require.Equal(t, previous, &child.Status) + + t.Log("A real transition is persisted once and updates component readiness") + setReadyCondition(child, different, "new observation") + require.Zero(t, writes) + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.Equal(t, 1, writes) + require.True(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").LastTransitionTime.After(transition.Time)) + require.Equal(t, different == v1beta1.DGDStateSuccessful, child.Status.Components["engine"].Ready) + }) + } +} + +// Update with: go test ./internal/controller/lpx -run TestPipelineRequestReadyConditionGolden -args -update +func TestPipelineRequestReadyConditionGolden(t *testing.T) { + t.Log("Load scheduler receipts and preserve exact condition diagnostics in the golden") + payload, err := os.ReadFile("testdata/pipeline-request-statuses.yaml") + require.NoError(t, err) + var statuses []lpxv1alpha1.LPUPipelineRequestStatus + require.NoError(t, yaml.UnmarshalStrict(payload, &statuses)) + conditions := make(map[string]metav1.Condition, len(statuses)) + for _, status := range statuses { + t.Run(string(status.Phase), func(t *testing.T) { + t.Log("Project the current receipt without altering it or changing Ready's transition time") + deployment := &v1alpha1.LPXGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 7}, + Status: v1alpha1.LPXGraphDeploymentStatus{Conditions: []metav1.Condition{{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)), + }}}, + } + request := &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{Generation: 7}, Status: status.DeepCopy()} + before := request.DeepCopy() + requests := map[string]*lpxv1alpha1.LPUPipelineRequest{"observed": request} + require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + require.Len(t, deployment.Status.Conditions, 1) + conditions[string(status.Phase)] = *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) + + t.Log("A failure outranks missing receipts and repeated map observations stay stable") + requests["first-open"] = &lpxv1alpha1.LPUPipelineRequest{} + require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + wantReason := v1alpha1.LPXReadyReasonPending + if status.Phase == lpxv1alpha1.RequestPhaseUnsupported || status.Phase == "future" { + wantReason = v1alpha1.LPXReadyReasonFailed + } + condition := *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) + require.Equal(t, wantReason, condition.Reason) + for range 10 { + require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + require.Equal(t, condition, *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition)) + } + require.Equal(t, before, request) + }) + } + actual, err := json.MarshalIndent(conditions, "", " ") + require.NoError(t, err) + golden.Assert(t, string(actual)+"\n", "pipeline-request-ready.golden.json") +} + +func TestPipelineRequestReadyConditionRequiresCurrentReceipts(t *testing.T) { + for _, tc := range []struct { + name string + phase lpxv1alpha1.RequestPhase + observed *int64 + runtime v1beta1.DGDState + allBound bool + }{ + {name: "Bound preserves runtime readiness", phase: lpxv1alpha1.RequestPhaseBound, observed: ptr.To(int64(7)), runtime: v1beta1.DGDStateSuccessful, allBound: true}, + {name: "Bound still needs runtime readiness", phase: lpxv1alpha1.RequestPhaseBound, observed: ptr.To(int64(7)), runtime: v1beta1.DGDStatePending, allBound: true}, + {name: "stale Bound", phase: lpxv1alpha1.RequestPhaseBound, observed: ptr.To(int64(6)), runtime: v1beta1.DGDStateSuccessful}, + {name: "unobserved Bound", phase: lpxv1alpha1.RequestPhaseBound, runtime: v1beta1.DGDStateSuccessful}, + {name: "stale failure", phase: lpxv1alpha1.RequestPhaseUnsupported, observed: ptr.To(int64(6)), runtime: v1beta1.DGDStatePending}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Combine scheduler freshness with the previously observed runtime condition") + child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 7}} + setReadyCondition(child, tc.runtime, "Runtime observation") + before := child.Status.DeepCopy() + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{Generation: 7}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{ + Phase: tc.phase, ObservedGeneration: tc.observed, + Diagnostics: []lpxv1alpha1.StatusDiagnostic{{Detail: "Old diagnostic"}}, + }, + } + require.Equal(t, tc.allBound, setPipelineRequestReadyCondition(child, map[string]*lpxv1alpha1.LPUPipelineRequest{"request": request})) + if tc.allBound { + require.Equal(t, before, &child.Status) + } else { + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.Equal(t, v1alpha1.LPXReadyReasonPending, ready.Reason) + require.NotContains(t, ready.Message, "Old diagnostic") + } + }) + } +} + +func TestSchedulingFailureCoversPipelineRequests(t *testing.T) { + failedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + conditionStatus metav1.ConditionStatus + reason string + generation int64 + noCondition bool + noFailureTime bool + start *metav1.Time + wantCurrent bool + wantCovered bool + }{ + {name: "no condition", noCondition: true}, + {name: "false condition", conditionStatus: metav1.ConditionFalse, reason: pipelineRequestDeadlineExceededReason, generation: 2}, + {name: "unrelated failure", conditionStatus: metav1.ConditionTrue, reason: "OtherFailure", generation: 2}, + {name: "previous generation covers older cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 1, start: ptr.To(metav1.NewTime(failedAt.Add(-time.Minute))), wantCovered: true}, + {name: "previous generation does not cover newer cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 1, start: ptr.To(metav1.NewTime(failedAt.Add(time.Second)))}, + {name: "missing failure timestamp", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, noFailureTime: true, wantCurrent: true}, + {name: "older cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt.Add(-time.Minute))), wantCurrent: true, wantCovered: true}, + {name: "same timestamp", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt)), wantCurrent: true}, + {name: "newer cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt.Add(time.Second))), wantCurrent: true}, + {name: "unknown cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, wantCurrent: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("A durable failure authorizes cleanup of older cycles independently of retry generation") + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}} + if !tc.noCondition { + condition := metav1.Condition{ + Type: schedulingFailedCondition, Status: tc.conditionStatus, Reason: tc.reason, + ObservedGeneration: tc.generation, LastTransitionTime: metav1.NewTime(failedAt), + } + if tc.noFailureTime { + condition.LastTransitionTime = metav1.Time{} + } + deployment.Status.Conditions = []metav1.Condition{condition} + } + request := &lpxv1alpha1.LPUPipelineRequest{Status: &lpxv1alpha1.LPUPipelineRequestStatus{SchedulingStartedAt: tc.start}} + require.Equal(t, tc.wantCurrent, isSchedulingFailedConditionCurrent(deployment)) + require.Equal(t, tc.wantCovered, schedulingFailureCoversPipelineRequests(deployment, []*lpxv1alpha1.LPUPipelineRequest{request})) + }) + } +} + +func TestSetSchedulingFailedCondition(t *testing.T) { + for _, tc := range []struct { + name string + previousGeneration int64 + renew bool + wantGeneration int64 + }{ + {name: "first failure", renew: true, wantGeneration: 2}, + {name: "current failure unchanged", previousGeneration: 2, wantGeneration: 2}, + {name: "previous failure unchanged", previousGeneration: 1, wantGeneration: 1}, + {name: "new expired cycle", previousGeneration: 2, renew: true, wantGeneration: 2}, + {name: "new generation expires", previousGeneration: 1, renew: true, wantGeneration: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Keep the recorded generation and timestamp unless a newly expired cycle requires renewal") + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}} + before := metav1.NewTime(time.Now().Add(-time.Hour)) + if tc.previousGeneration != 0 { + deployment.Status.Conditions = []metav1.Condition{{ + Type: schedulingFailedCondition, Status: metav1.ConditionTrue, Reason: pipelineRequestDeadlineExceededReason, + ObservedGeneration: tc.previousGeneration, LastTransitionTime: before, + }} + } + setSchedulingFailedCondition(deployment, tc.renew) + condition := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + require.Equal(t, tc.wantGeneration, condition.ObservedGeneration) + require.Equal(t, metav1.ConditionTrue, condition.Status) + if tc.renew { + require.True(t, condition.LastTransitionTime.After(before.Time)) + } else { + require.Equal(t, before, condition.LastTransitionTime) + } + }) + } +} + +func TestClearPreviousSchedulingFailure(t *testing.T) { + for _, tc := range []struct { + name string + recordedGeneration int64 + wantCleared bool + }{ + {name: "no failure"}, + {name: "previous generation", recordedGeneration: 1, wantCleared: true}, + {name: "same generation", recordedGeneration: 2}, + {name: "newer observed generation", recordedGeneration: 3}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Only a later desired generation authorizes retrying expired pipeline requests") + deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}} + if tc.recordedGeneration > 0 { + deployment.Status.Conditions = []metav1.Condition{{ + Type: schedulingFailedCondition, Status: metav1.ConditionTrue, + Reason: pipelineRequestDeadlineExceededReason, ObservedGeneration: tc.recordedGeneration, + }} + } + before := deployment.Status.DeepCopy() + clearPreviousSchedulingFailure(deployment) + if !tc.wantCleared { + require.Equal(t, *before, deployment.Status) + return + } + condition := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, deployment.Generation, condition.ObservedGeneration) + require.Equal(t, "LPXSchedulingRetryAuthorized", condition.Reason) + }) + } +} + +func TestPipelineRequestDiagnosticMessageLimit(t *testing.T) { + for _, tc := range []struct{ name, detail string }{ + {name: "ASCII", detail: strings.Repeat("a", 4096)}, + {name: "multibyte", detail: strings.Repeat("界", 4096/len("界"))}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Truncate oversized scheduler diagnostics to the condition limit with an ASCII suffix") + diagnostics := make([]lpxv1alpha1.StatusDiagnostic, 32) + for i := range diagnostics { + diagnostics[i] = lpxv1alpha1.StatusDiagnostic{Code: "code", Subject: "subject", Detail: tc.detail} + } + message := pipelineRequestDiagnosticMessage("LPX scheduler diagnostic summary", diagnostics) + require.LessOrEqual(t, len(message), maxConditionMessageSize) + require.True(t, utf8.ValidString(message)) + require.True(t, strings.HasSuffix(message, "...")) + }) + } +} diff --git a/deploy/operator/internal/controller/lpx/testdata/dgd.yaml b/deploy/operator/internal/controller/lpx/testdata/dgd.yaml new file mode 100644 index 000000000000..ae4ce8de2f75 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/testdata/dgd.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: selected-source + namespace: test + uid: source-uid + generation: 3 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: lpx + type: lpx + replicas: 1 + lpx: + buildId: build-v2 + roles: + - name: conductor + podTemplate: + spec: + containers: + - name: main + image: lpu-runtime + command: [/bin/nova] + volumeMounts: &runtime-mounts + - {name: model-storage, mountPath: /models} + - {name: config, mountPath: /configs} + - {name: host-dev, mountPath: /dev} + - {name: host-sys, mountPath: /sys} + - {name: ssh-secret, mountPath: /ssh-pk, readOnly: true} + - {name: hugepages, mountPath: /dev/hugepages} + - {name: single-v2-ssh-key, mountPath: /tmp/dynamo-lpu-ssh} + volumes: &runtime-volumes + - {name: model-storage, emptyDir: {}} + - {name: host-dev, hostPath: {path: /dev, type: Directory}} + - {name: host-sys, hostPath: {path: /sys, type: Directory}} + - {name: hugepages, emptyDir: {medium: HugePages}} + - {name: ssh-secret, secret: {secretName: ssh-secret, defaultMode: 420}} + - {name: single-v2-ssh-key, emptyDir: {}} + - name: agent + podTemplate: + spec: + volumes: *runtime-volumes + containers: + - name: main + image: lpu-runtime + command: [/bin/quasar-entrypoint] + volumeMounts: *runtime-mounts diff --git a/deploy/operator/internal/controller/lpx/testdata/hybrid-conductor.yaml b/deploy/operator/internal/controller/lpx/testdata/hybrid-conductor.yaml new file mode 100644 index 000000000000..acbd028d15bc --- /dev/null +++ b/deploy/operator/internal/controller/lpx/testdata/hybrid-conductor.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: conductor +replicas: 1 +podTemplate: + spec: + resourceClaims: + - {name: gpu, resourceClaimTemplateName: gpu} + containers: + - name: main + image: cyborg-runtime + resources: + claims: + - {name: gpu} + volumeMounts: + - {name: config, mountPath: /configs} + - {name: model-storage, mountPath: /models} + volumes: + - {name: model-storage, emptyDir: {}} diff --git a/deploy/operator/internal/controller/lpx/testdata/pipeline-request-ready.golden.json b/deploy/operator/internal/controller/lpx/testdata/pipeline-request-ready.golden.json new file mode 100644 index 000000000000..bfcc37adcf93 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/testdata/pipeline-request-ready.golden.json @@ -0,0 +1,82 @@ +{ + "binding": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler is binding the planned workload Pods. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "degraded": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler is repairing or releasing a degraded placement. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "future": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Failed", + "message": "LPX scheduler reported unknown phase \"future\". LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "noFit": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler found no placement for the current request. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "pending": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler is waiting to plan the current request. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "planned": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler committed a placement plan for the current request. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "released": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler released the current plan. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "releasing": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler is releasing the current plan. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "reserving": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Pending", + "message": "LPX scheduler is reserving the planned LPU allocation. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + }, + "unsupported": { + "type": "Ready", + "status": "False", + "observedGeneration": 7, + "lastTransitionTime": "2026-01-01T00:00:00Z", + "reason": "Failed", + "message": "LPX scheduler cannot support the current request. LPX diagnostics: ExactDiagnostic [requests/current]: the exact scheduler detail" + } +} diff --git a/deploy/operator/internal/controller/lpx/testdata/pipeline-request-statuses.yaml b/deploy/operator/internal/controller/lpx/testdata/pipeline-request-statuses.yaml new file mode 100644 index 000000000000..aa3a31eab2bd --- /dev/null +++ b/deploy/operator/internal/controller/lpx/testdata/pipeline-request-statuses.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- phase: pending + observedGeneration: 7 + diagnostics: &diagnostics + - code: ExactDiagnostic + subject: requests/current + detail: the exact scheduler detail +- phase: noFit + observedGeneration: 7 + diagnostics: *diagnostics +- phase: unsupported + observedGeneration: 7 + diagnostics: *diagnostics +- phase: planned + observedGeneration: 7 + diagnostics: *diagnostics +- phase: reserving + observedGeneration: 7 + diagnostics: *diagnostics +- phase: binding + observedGeneration: 7 + diagnostics: *diagnostics +- phase: degraded + observedGeneration: 7 + diagnostics: *diagnostics +- phase: releasing + observedGeneration: 7 + diagnostics: *diagnostics +- phase: released + observedGeneration: 7 + diagnostics: *diagnostics +- phase: future + observedGeneration: 7 + diagnostics: *diagnostics diff --git a/deploy/operator/internal/controller/lpx/util.go b/deploy/operator/internal/controller/lpx/util.go new file mode 100644 index 000000000000..1e95cd848065 --- /dev/null +++ b/deploy/operator/internal/controller/lpx/util.go @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "fmt" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// getDynamoGraphDeployment validates the child's DGD using a cached reader. +// reader and deployment must be non-nil. A nil result means the DGD is missing, +// deleting, no longer selects LPX, or has not delivered its revision. +func getDynamoGraphDeployment(ctx context.Context, reader client.Reader, deployment *v1alpha1.LPXGraphDeployment) (*v1beta1.DynamoGraphDeployment, error) { + logger := log.FromContext(ctx) + + owner := metav1.GetControllerOf(deployment) + if owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != v1beta1.DynamoGraphDeploymentGVK.Kind { + return nil, fmt.Errorf("LPXGraphDeployment requires a DynamoGraphDeployment controller owner") + } + + dgd := &v1beta1.DynamoGraphDeployment{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: owner.Name}, dgd); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("cannot get DynamoGraphDeployment %q for LPXGraphDeployment: %w", owner.Name, err) + } + if dgd.UID != owner.UID { + return nil, fmt.Errorf("DynamoGraphDeployment %q no longer has the referenced UID", owner.Name) + } + + // The parent deletes the child on deselection; GC owns the remaining workload. + if !dgd.DeletionTimestamp.IsZero() || !dgd.HasLPXComponent() { + return nil, nil + } + + restartToken := dynamo.LPXRestartToken(dgd, deployment.Annotations[dynamo.LPXRestartAnnotation]) + revision, err := dynamo.LPXInputRevision(dgd, restartToken) + if err != nil { + return nil, fmt.Errorf("compute DynamoGraphDeployment input revision: %w", err) + } + + if deployment.Spec.InputRevision != revision { + logger.V(4).Info("LPXDynamoGraphDeployment is outdated", "expectedRevision", revision, "inputRevision", deployment.Spec.InputRevision) + return nil, nil + } + + if actualRestartToken := deployment.Annotations[dynamo.LPXRestartAnnotation]; actualRestartToken != restartToken { + logger.V(4).Info("LPXDynamoGraphDeployment is restarting", "expectedRestartToken", restartToken, "restartToken", actualRestartToken) + return nil, nil + } + + return dgd, nil +} + +// getPodCliqueSet returns the deployment's owned PCS, or nil when absent. +// reader and deployment must be non-nil; reader must use the cache. +func getPodCliqueSet(ctx context.Context, reader client.Reader, deployment *v1alpha1.LPXGraphDeployment) (*grovev1alpha1.PodCliqueSet, error) { + pcs := &grovev1alpha1.PodCliqueSet{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: dynamo.PCSNameForLPX(deployment)}, pcs); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(pcs, deployment) { + return nil, fmt.Errorf("PodCliqueSet %q is not controlled by this LPXGraphDeployment", pcs.Name) + } + return pcs, nil +} + +// getPodCliqueScalingGroup observes the single engine group under PCS ordinal zero. +// reader and pcs must be non-nil; reader must use the cache, and pcs must already +// be owned by this LPXGD. A non-nil result is controlled by that PCS and not +// deleting; nil means the group is absent from the cache or is deleting. +// A group controlled by another PCS is an ownership error. +// Nil does not mean zero replicas, and a zero-replica group is returned normally. +func getPodCliqueScalingGroup(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet) (*grovev1alpha1.PodCliqueScalingGroup, error) { + // Engine replicas belong to one PCSG, not to additional top-level PCS ordinals. + configs := pcs.Spec.Template.PodCliqueScalingGroupConfigs + if len(configs) != 1 { + return nil, fmt.Errorf("LPX PodCliqueSet %q requires exactly one scaling-group template", pcs.Name) + } + name := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, configs[0].Name) + + // Establish the group's identity and lifetime once for all downstream operations. + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: name}, pcsg); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(pcsg, pcs) { + return nil, fmt.Errorf("PodCliqueScalingGroup %q is not controlled by PodCliqueSet %q", pcsg.Name, pcs.Name) + } + + if !pcsg.DeletionTimestamp.IsZero() { + return nil, nil + } + + return pcsg, nil +} diff --git a/deploy/operator/internal/controller/lpx/util_test.go b/deploy/operator/internal/controller/lpx/util_test.go new file mode 100644 index 000000000000..ac111d4678fb --- /dev/null +++ b/deploy/operator/internal/controller/lpx/util_test.go @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package lpx + +import ( + "context" + "errors" + "testing" + + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestGetPodCliqueScalingGroup(t *testing.T) { + for _, tc := range []struct { + name string + templates int + missing bool + previousPCS bool + deleting bool + replicas int32 + wantGroup bool + wantError string + }{ + {name: "current group", templates: 1, replicas: 3, wantGroup: true}, + {name: "zero replicas is still an observed group", templates: 1, wantGroup: true}, + {name: "missing group", templates: 1, missing: true}, + {name: "previous PCS incarnation", templates: 1, previousPCS: true, wantError: `PodCliqueScalingGroup "pcs-0-engines" is not controlled by PodCliqueSet "pcs"`}, + {name: "deleting previous PCS incarnation", templates: 1, previousPCS: true, deleting: true, wantError: `PodCliqueScalingGroup "pcs-0-engines" is not controlled by PodCliqueSet "pcs"`}, + {name: "deleting group", templates: 1, deleting: true}, + {name: "missing template", wantError: "requires exactly one scaling-group template"}, + {name: "multiple templates", templates: 2, wantError: "requires exactly one scaling-group template"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe the one group under PCS ordinal zero and establish ownership once") + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: "pcs", Namespace: "default", UID: "pcs-uid"}} + pcs.Spec.Template.PodCliqueScalingGroupConfigs = make([]grovev1alpha1.PodCliqueScalingGroupConfig, tc.templates) + for index := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { + pcs.Spec.Template.PodCliqueScalingGroupConfigs[index].Name = "engines" + } + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pcs-0-engines", Namespace: pcs.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: tc.replicas}, + } + if tc.previousPCS { + group.OwnerReferences[0].UID = "previous-pcs" + } + if tc.deleting { + group.Finalizers = []string{"example.com/cleanup"} + now := metav1.Now() + group.DeletionTimestamp = &now + } + var objects []client.Object + if !tc.missing { + objects = append(objects, group) + } + reader := newLPXTestClient(t, objects...) + + t.Log("Only a current non-deleting group supplies capacity, including an explicit zero") + observed, err := getPodCliqueScalingGroup(t.Context(), reader, pcs) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + require.Nil(t, observed) + return + } + require.NoError(t, err) + if tc.wantGroup { + require.NotNil(t, observed) + require.Equal(t, group.Name, observed.Name) + require.Equal(t, tc.replicas, observed.Spec.Replicas) + require.True(t, metav1.IsControlledBy(observed, pcs)) + } else { + require.Nil(t, observed) + } + }) + } +} + +func TestLPXCacheMissesPreservePublishedWorkload(t *testing.T) { + for _, missing := range []string{"source", "group"} { + t.Run(missing, func(t *testing.T) { + t.Log("Hide one dependency from the cache while its objects still exist") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, desired) + before := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, desired.requests[0].Name) + base := r.Client.(client.WithWatch) + writes := 0 + r.Client = interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + kind := "" + switch object.(type) { + case *v1beta1.DynamoGraphDeployment: + kind = "source" + case *grovev1alpha1.PodCliqueScalingGroup: + kind = "group" + } + if kind == missing { + return apierrors.NewNotFound(grovev1alpha1.SchemeGroupVersion.WithResource(kind).GroupResource(), key.Name) + } + return delegated.Get(ctx, key, object, opts...) + }, + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + writes++ + return errors.New("unexpected create during cache miss") + }, + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + writes++ + return errors.New("unexpected deletion during cache miss") + }, + }) + + t.Log("Wait without treating a missing cache observation as deletion authority") + for range 2 { + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result, "dependency watches resume reconciliation") + } + require.Zero(t, writes) + require.Equal(t, before, getTestPipelineRequest(t, t.Context(), base, before.Namespace, before.Name)) + }) + } +} + +func TestLPXForeignRequestNameIsNeverAdopted(t *testing.T) { + t.Log("Occupy the deterministic request name with another PCS's request") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + foreign := desired.requests[0].DeepCopy() + foreignPCS := findLPXTestPodCliqueSet(t, objects).DeepCopy() + foreignPCS.UID = "foreign-pcs" + foreign.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(foreignPCS, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} + require.NoError(t, r.Create(t.Context(), foreign)) + before := getTestPipelineRequest(t, t.Context(), r.Client, foreign.Namespace, foreign.Name) + + t.Log("Finish PCS synchronization before testing the conflicting request name") + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result) + require.Equal(t, before, getTestPipelineRequest(t, t.Context(), r.Client, foreign.Namespace, foreign.Name)) + + t.Log("An indexed miss followed by AlreadyExists waits without adoption or deletion") + for range 2 { + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result, "LPR events resume reconciliation; name collisions are never adopted") + require.Equal(t, before, getTestPipelineRequest(t, t.Context(), r.Client, foreign.Namespace, foreign.Name)) + } +} + +func TestLPXDeselectionWaitsForParentDeletion(t *testing.T) { + t.Log("Keep a published workload while its source removes the LPX component") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + before := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, desired.requests[0].Name) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + dgd.Spec.Components = nil + require.NoError(t, r.Update(t.Context(), dgd)) + + t.Log("Wait for the parent's deletion and GC without hashing empty LPX inputs or deleting the PCS") + for range 2 { + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result, "the source and child watches observe deselection") + } + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.True(t, pcs.DeletionTimestamp.IsZero()) + require.Equal(t, before, getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, before.Name)) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) +} + +func TestLPXDeletionUsesSnapshotPreconditions(t *testing.T) { + t.Log("Observe the PCS before a concurrent update") + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ + Name: "pcs", Namespace: "default", UID: "pcs-uid", ResourceVersion: "1", + }} + r := &graphReconciler{Client: newLPXTestClient(t, pcs)} + changed := pcs.DeepCopy() + changed.Labels = map[string]string{"example.com/concurrent": "update"} + require.NoError(t, r.Update(t.Context(), changed)) + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + options := (&client.DeleteOptions{}).ApplyOptions(opts) + require.Equal(t, metav1.DeletePropagationForeground, *options.PropagationPolicy) + require.Equal(t, pcs.UID, *options.Preconditions.UID) + require.Equal(t, pcs.ResourceVersion, *options.Preconditions.ResourceVersion) + return delegated.Delete(ctx, object, opts...) + }, + }) + + t.Log("Foreground deletion cannot remove a resource changed since observation") + err := r.deletePodCliqueSet(t.Context(), pcs) + require.True(t, apierrors.IsConflict(err), "expected a resource-version conflict: %v", err) + stored := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), stored)) + require.Equal(t, changed, stored) +} diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 94117cea7dde..6e59c7ddca2f 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -10,8 +10,8 @@ import ( "fmt" "maps" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -28,49 +28,68 @@ import ( commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" ) -const lpxSourceOwnerIndex = "lpx.sourceController" +const ( + dgdControllerOwnerIndex = "lpx.dgdController" + podCliqueSetKind = "PodCliqueSet" + // pipelineRequestPCSOwnerUIDIndex indexes controlling PCS UIDs, never reusable PCS names. + pipelineRequestPCSOwnerUIDIndex = ".metadata.controller" +) func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { - // Source events address exact controller owners, not source-named materializations. - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences); err != nil { - return fmt.Errorf("register LPX source owner index: %w", err) + // DGD events address exact controller owners, not DGD-named materializations. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.LPXGraphDeployment{}, dgdControllerOwnerIndex, dgdControllerOwnerKey); err != nil { + return fmt.Errorf("register LPX DGD owner index: %w", err) } + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). - For(&nvidiacomv1alpha1.LPXGraphDeployment{}, builder.WithPredicates(lpxDeploymentPrimaryPredicate())). + For(&v1alpha1.LPXGraphDeployment{}, builder.WithPredicates(lpxGraphDeploymentPredicate())). Named("lpxgraphdeployment"). - Watches(&nvidiacomv1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(r.mapLPXSourceToRequests), builder.WithPredicates(lpxSourcePredicate())). - WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.runtimeConfig)) - // Primary and source events are sufficient while disabled. - if r.unavailableReason() != "" { + Watches(&v1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(r.mapDGDToLPXGraphDeployments), builder.WithPredicates(dgdPredicate())). + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.config, r.runtimeConfig)) + + // Primary and DGD events are sufficient while disabled. + if !r.enabled { return ctrlBuilder.Complete(r) } // Active integration observes its workload and scheduling dependencies. + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &lpxv1alpha1.LPUPipelineRequest{}, pipelineRequestPCSOwnerUIDIndex, pipelineRequestOwnerUID); err != nil { + return fmt.Errorf("register LPR owner UID index: %w", err) + } + return ctrlBuilder.Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). Owns(&grovev1alpha1.PodCliqueSet{}). - Watches(&grovev1alpha1.PodClique{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxPodCliqueEventPredicates())). - Watches(&grovev1alpha1.PodCliqueScalingGroup{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests), builder.WithPredicates(lpxScalingGroupEventPredicates())). - Watches(&lpxv1alpha1.LPUPipelineRequest{}, handler.EnqueueRequestsFromMapFunc(mapLPXChildToRequests)). + Watches(&grovev1alpha1.PodClique{}, handler.EnqueueRequestsFromMapFunc(mapChildToLPXGraphDeployment), builder.WithPredicates(podCliquePredicate())). + Watches(&grovev1alpha1.PodCliqueScalingGroup{}, handler.EnqueueRequestsFromMapFunc(mapChildToLPXGraphDeployment), builder.WithPredicates(podCliqueScalingGroupPredicate())). + Watches(&lpxv1alpha1.LPUPipelineRequest{}, handler.EnqueueRequestsFromMapFunc(mapChildToLPXGraphDeployment)). Complete(r) } -func lpxSourceOwnerReferences(obj client.Object) []string { +func pipelineRequestOwnerUID(obj client.Object) []string { owner := metav1.GetControllerOf(obj) - if owner == nil || owner.APIVersion != nvidiacomv1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" { + if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.Kind != podCliqueSetKind { + return nil + } + return []string{string(owner.UID)} +} + +func dgdControllerOwnerKey(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != v1beta1.DynamoGraphDeploymentGVK.Kind { return nil } return []string{owner.Name + "/" + string(owner.UID)} } -func (r *graphReconciler) mapLPXSourceToRequests(ctx context.Context, obj client.Object) []ctrl.Request { - deployments := &nvidiacomv1alpha1.LPXGraphDeploymentList{} +func (r *graphReconciler) mapDGDToLPXGraphDeployments(ctx context.Context, obj client.Object) []ctrl.Request { + deployments := &v1alpha1.LPXGraphDeploymentList{} ownerKey := obj.GetName() + "/" + string(obj.GetUID()) - if err := r.List(ctx, deployments, client.InNamespace(obj.GetNamespace()), client.MatchingFields{lpxSourceOwnerIndex: ownerKey}); err != nil { - ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX deployments for source", "source", client.ObjectKeyFromObject(obj), "sourceUID", obj.GetUID()) + if err := r.List(ctx, deployments, client.InNamespace(obj.GetNamespace()), client.MatchingFields{dgdControllerOwnerIndex: ownerKey}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "Unable to list LPX deployments for DGD", "dgd", client.ObjectKeyFromObject(obj), "dgdUID", obj.GetUID()) return nil } requests := make([]ctrl.Request, 0, len(deployments.Items)) @@ -80,8 +99,17 @@ func (r *graphReconciler) mapLPXSourceToRequests(ctx context.Context, obj client return requests } -// lpxDeploymentPrimaryPredicate observes every field that grants publication authority. -func lpxDeploymentPrimaryPredicate() predicate.Predicate { +// mapChildToLPXGraphDeployment receives non-nil, namespaced Grove informer objects. +func mapChildToLPXGraphDeployment(_ context.Context, obj client.Object) []ctrl.Request { + deploymentName := obj.GetAnnotations()[lpx.DeploymentNameAnnotation] + if deploymentName == "" { + return nil + } + return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: deploymentName}}} +} + +// lpxGraphDeploymentPredicate observes every field that grants publication authority. +func lpxGraphDeploymentPredicate() predicate.Predicate { return predicate.Or( commoncontroller.GenerationOrDeletionChangedPredicate(), predicate.AnnotationChangedPredicate{}, @@ -91,35 +119,56 @@ func lpxDeploymentPrimaryPredicate() predicate.Predicate { ) } -// mapLPXChildToRequests receives non-nil, namespaced Grove informer objects. -func mapLPXChildToRequests(_ context.Context, obj client.Object) []ctrl.Request { - deploymentName := obj.GetAnnotations()[dynamolpx.DeploymentNameAnnotation] - if deploymentName == "" { - return nil +// dgdPredicate ignores ordinary scaling/status traffic but wakes the child +// for relevant intent, DGD identity/deletion, and persisted restart selection. +// Its registered informer supplies non-nil DGDs. +func dgdPredicate() predicate.Predicate { + filter := predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.(*v1beta1.DynamoGraphDeployment).HasLPXComponent() + }) + filter.UpdateFunc = func(e event.UpdateEvent) bool { + oldDGD := e.ObjectOld.(*v1beta1.DynamoGraphDeployment) + newDGD := e.ObjectNew.(*v1beta1.DynamoGraphDeployment) + if !oldDGD.HasLPXComponent() && !newDGD.HasLPXComponent() { + return false + } + if oldDGD.UID != newDGD.UID || !apiequality.Semantic.DeepEqual(oldDGD.DeletionTimestamp, newDGD.DeletionTimestamp) { + return true + } + // Spec edits advance generation; only the selected restart depends on status. + // Avoid conversion and hashing for ordinary status-only events. + oldRestart, newRestart := dynamo.LPXRestartToken(oldDGD, ""), dynamo.LPXRestartToken(newDGD, "") + if oldDGD.Generation == newDGD.Generation && oldRestart == newRestart && + maps.Equal(oldDGD.Labels, newDGD.Labels) && maps.Equal(oldDGD.Annotations, newDGD.Annotations) { + return false + } + oldRevision, oldErr := dynamo.LPXInputRevision(oldDGD, oldRestart) + newRevision, newErr := dynamo.LPXInputRevision(newDGD, newRestart) + return oldErr != nil || newErr != nil || oldRevision != newRevision } - return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: obj.GetNamespace(), Name: deploymentName}}} + return filter } -func lpxPodCliqueMaterializationEvent(obj client.Object) bool { +func isLPXPodClique(obj client.Object) bool { clique := obj.(*grovev1alpha1.PodClique) role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] - return clique.Annotations[dynamolpx.WorkloadDigestAnnotation] != "" && + return clique.Annotations[lpx.WorkloadDigestAnnotation] != "" && (role == lpxv1alpha1.PodRoleAgent || role == lpxv1alpha1.PodRoleConductor || role == lpxv1alpha1.PodRoleCyborgWorker) } -func lpxPodCliqueScalingGroupMaterializationEvent(obj client.Object) bool { +func isLPXPodCliqueScalingGroup(obj client.Object) bool { group := obj.(*grovev1alpha1.PodCliqueScalingGroup) - return group.Annotations[dynamolpx.WorkloadDigestAnnotation] != "" + return group.Annotations[lpx.WorkloadDigestAnnotation] != "" } -func lpxPodCliqueEventPredicates() predicate.Funcs { +func podCliquePredicate() predicate.Funcs { return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { return lpxPodCliqueMaterializationEvent(e.Object) }, - DeleteFunc: func(e event.DeleteEvent) bool { return lpxPodCliqueMaterializationEvent(e.Object) }, + CreateFunc: func(e event.CreateEvent) bool { return isLPXPodClique(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return isLPXPodClique(e.Object) }, UpdateFunc: func(e event.UpdateEvent) bool { oldClique := e.ObjectOld.(*grovev1alpha1.PodClique) newClique := e.ObjectNew.(*grovev1alpha1.PodClique) - if !lpxPodCliqueMaterializationEvent(oldClique) && !lpxPodCliqueMaterializationEvent(newClique) { + if !isLPXPodClique(oldClique) && !isLPXPodClique(newClique) { return false } return commoncontroller.PodCliqueStatusChangeIsSignificant(oldClique, newClique) || @@ -134,14 +183,14 @@ func lpxPodCliqueEventPredicates() predicate.Funcs { } } -func lpxScalingGroupEventPredicates() predicate.Funcs { +func podCliqueScalingGroupPredicate() predicate.Funcs { return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { return lpxPodCliqueScalingGroupMaterializationEvent(e.Object) }, - DeleteFunc: func(e event.DeleteEvent) bool { return lpxPodCliqueScalingGroupMaterializationEvent(e.Object) }, + CreateFunc: func(e event.CreateEvent) bool { return isLPXPodCliqueScalingGroup(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return isLPXPodCliqueScalingGroup(e.Object) }, UpdateFunc: func(e event.UpdateEvent) bool { oldGroup := e.ObjectOld.(*grovev1alpha1.PodCliqueScalingGroup) newGroup := e.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) - if !lpxPodCliqueScalingGroupMaterializationEvent(oldGroup) && !lpxPodCliqueScalingGroupMaterializationEvent(newGroup) { + if !isLPXPodCliqueScalingGroup(oldGroup) && !isLPXPodCliqueScalingGroup(newGroup) { return false } return commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldGroup, newGroup) || @@ -154,33 +203,3 @@ func lpxScalingGroupEventPredicates() predicate.Funcs { GenericFunc: func(event.GenericEvent) bool { return false }, } } - -// lpxSourcePredicate ignores ordinary scaling/status traffic but wakes the child -// for relevant intent, source identity/deletion, and persisted restart selection. -// Its registered informer supplies non-nil DGDs. -func lpxSourcePredicate() predicate.Predicate { - filter := predicate.NewPredicateFuncs(func(obj client.Object) bool { - return obj.(*nvidiacomv1beta1.DynamoGraphDeployment).HasLPXComponent() - }) - filter.UpdateFunc = func(e event.UpdateEvent) bool { - oldSource := e.ObjectOld.(*nvidiacomv1beta1.DynamoGraphDeployment) - newSource := e.ObjectNew.(*nvidiacomv1beta1.DynamoGraphDeployment) - if !oldSource.HasLPXComponent() && !newSource.HasLPXComponent() { - return false - } - if oldSource.UID != newSource.UID || !apiequality.Semantic.DeepEqual(oldSource.DeletionTimestamp, newSource.DeletionTimestamp) { - return true - } - // Spec edits advance generation; only the selected restart depends on status. - // Avoid conversion and hashing for ordinary status-only events. - oldRestart, newRestart := dynamo.LPXRestartToken(oldSource, ""), dynamo.LPXRestartToken(newSource, "") - if oldSource.Generation == newSource.Generation && oldRestart == newRestart && - maps.Equal(oldSource.Labels, newSource.Labels) && maps.Equal(oldSource.Annotations, newSource.Annotations) { - return false - } - oldRevision, oldErr := dynamo.LPXInputRevision(oldSource, oldRestart) - newRevision, newErr := dynamo.LPXInputRevision(newSource, newRestart) - return oldErr != nil || newErr != nil || oldRevision != newRevision - } - return filter -} diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 554a71edadb2..210747adaf09 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -9,39 +9,37 @@ import ( "bytes" "context" "errors" + "maps" "testing" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/predicate" ) -func TestLPXDeploymentPrimaryPredicateTracksOwnerIdentity(t *testing.T) { +func TestLPXGraphDeploymentPredicate(t *testing.T) { t.Log("Build one current private handoff and its primary-resource predicate") - deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ + deployment := &v1alpha1.LPXGraphDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: "graph", Namespace: "workloads", Generation: 1, OwnerReferences: []metav1.OwnerReference{{Name: "graph", UID: "source-uid", Controller: ptr.To(true)}}, }, } - filter := lpxDeploymentPrimaryPredicate() + filter := lpxGraphDeploymentPredicate() t.Log("Ignore status-only updates that do not change publication authority") statusOnly := deployment.DeepCopy() @@ -54,23 +52,20 @@ func TestLPXDeploymentPrimaryPredicateTracksOwnerIdentity(t *testing.T) { require.True(t, filter.Update(event.UpdateEvent{ObjectOld: deployment, ObjectNew: ownerChanged})) } -func TestLPXSourceWatchMapsControllerOwnedMaterializations(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, nvidiacomv1alpha1.AddToScheme(scheme)) - require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) - source := &nvidiacomv1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "workloads", UID: "source-uid"}} - owned := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ - Name: "materialization", Namespace: source.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, +func TestDGDWatchMapsControllerOwnedDeployments(t *testing.T) { + t.Log("Index children by their controller owner using the shared test client") + dgd := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "workloads", UID: "source-uid"}} + owned := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + Name: "materialization", Namespace: dgd.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, v1beta1.DynamoGraphDeploymentGVK)}, }} unrelated := owned.DeepCopy() unrelated.Name = "unrelated" unrelated.OwnerReferences[0].UID = "other-source-uid" - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(owned, unrelated). - WithIndex(&nvidiacomv1alpha1.LPXGraphDeployment{}, lpxSourceOwnerIndex, lpxSourceOwnerReferences).Build() + kube := newLPXTestClient(t, owned, unrelated) reconciler := &graphReconciler{Client: kube} - require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(owned)}}, reconciler.mapLPXSourceToRequests(t.Context(), source)) + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(owned)}}, reconciler.mapDGDToLPXGraphDeployments(t.Context(), dgd)) t.Log("Report failed owner lookups with the exact source identity, without inventing a child name") lookupErr := errors.New("owner index unavailable") @@ -81,162 +76,174 @@ func TestLPXSourceWatchMapsControllerOwnedMaterializations(t *testing.T) { }) var logs bytes.Buffer ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.WriteTo(&logs))) - require.Empty(t, reconciler.mapLPXSourceToRequests(ctx, source)) + require.Empty(t, reconciler.mapDGDToLPXGraphDeployments(ctx, dgd)) require.Contains(t, logs.String(), lookupErr.Error()) require.Contains(t, logs.String(), `"name":"source"`) require.Contains(t, logs.String(), `"namespace":"workloads"`) - require.Contains(t, logs.String(), `"sourceUID":"source-uid"`) + require.Contains(t, logs.String(), `"dgdUID":"source-uid"`) } -func TestLPXWorkloadEventPredicates(t *testing.T) { +func TestGroveEventPredicates(t *testing.T) { + metadata := metav1.ObjectMeta{ + Name: "grove-child", Namespace: "workloads", + Annotations: map[string]string{ + lpx.DeploymentNameAnnotation: "materialization", + lpx.WorkloadDigestAnnotation: "sha256:workload", + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + }, + } for _, test := range []struct { name string object client.Object + observed client.Object predicate predicate.Predicate - observe func(client.Object) }{ - {"clique", &grovev1alpha1.PodClique{}, lpxPodCliqueEventPredicates(), func(o client.Object) { o.(*grovev1alpha1.PodClique).Status.ReadyReplicas++ }}, - {"scaling group", &grovev1alpha1.PodCliqueScalingGroup{}, lpxScalingGroupEventPredicates(), func(o client.Object) { - o.(*grovev1alpha1.PodCliqueScalingGroup).Status.ObservedGeneration = ptr.To(int64(2)) - }}, + { + "clique", &grovev1alpha1.PodClique{ObjectMeta: metadata}, + &grovev1alpha1.PodClique{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueStatus{ReadyReplicas: 1}}, + podCliquePredicate(), + }, + { + "scaling group", &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metadata}, + &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ObservedGeneration: ptr.To(int64(2))}}, + podCliqueScalingGroupPredicate(), + }, } { t.Run(test.name, func(t *testing.T) { - t.Log("Observe creation, deletion and all metadata used to fence materialization") - old := test.object - old.SetName("grove-child") - old.SetNamespace("workloads") - old.SetAnnotations(map[string]string{ - dynamolpx.DeploymentNameAnnotation: "materialization", - dynamolpx.WorkloadDigestAnnotation: "sha256:workload", - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - }) - require.True(t, test.predicate.Create(event.CreateEvent{Object: old})) - require.True(t, test.predicate.Delete(event.DeleteEvent{Object: old})) - require.False(t, test.predicate.Generic(event.GenericEvent{Object: old})) + t.Log("Only LPX workload creation, deletion and readiness changes wake the controller") + require.True(t, test.predicate.Create(event.CreateEvent{Object: test.object})) + require.True(t, test.predicate.Delete(event.DeleteEvent{Object: test.object})) + require.False(t, test.predicate.Generic(event.GenericEvent{Object: test.object})) + require.True(t, test.predicate.Update(event.UpdateEvent{ObjectOld: test.object, ObjectNew: test.observed})) require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization"}}}, - mapLPXChildToRequests(t.Context(), old), "Grove children need no source DGD label for routing") + mapChildToLPXGraphDeployment(t.Context(), test.object), "Grove children need no DGD label for routing") - t.Log("Ordinary readiness traffic does not wake the LPX controller") - unrelated := old.DeepCopyObject().(client.Object) + unrelated := test.object.DeepCopyObject().(client.Object) unrelated.SetAnnotations(nil) + observedUnrelated := test.observed.DeepCopyObject().(client.Object) + observedUnrelated.SetAnnotations(nil) require.False(t, test.predicate.Create(event.CreateEvent{Object: unrelated})) - observed := unrelated.DeepCopyObject().(client.Object) - test.observe(observed) - require.False(t, test.predicate.Update(event.UpdateEvent{ObjectOld: unrelated, ObjectNew: observed})) - - t.Log("All compiled roles and PodClique template-hash convergence remain observable") - if clique, ok := old.(*grovev1alpha1.PodClique); ok { - for _, role := range []string{lpxv1alpha1.PodRoleAgent, lpxv1alpha1.PodRoleConductor, lpxv1alpha1.PodRoleCyborgWorker} { - current := clique.DeepCopy() - current.Annotations[lpxv1alpha1.PodRoleAnnotation] = role - require.True(t, test.predicate.Create(event.CreateEvent{Object: current})) - } - current := clique.DeepCopy() - current.Status.CurrentPodTemplateHash = ptr.To("current-template") - require.True(t, test.predicate.Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: current})) - } + require.False(t, test.predicate.Delete(event.DeleteEvent{Object: unrelated})) + require.False(t, test.predicate.Update(event.UpdateEvent{ObjectOld: unrelated, ObjectNew: observedUnrelated})) - t.Log("Wake on role readiness, independently observed group generation and materialization fences") + t.Log("Metadata changes wake reconciliation except for resource-version-only noise") + workloadAnnotations := maps.Clone(metadata.Annotations) + workloadAnnotations[lpx.WorkloadDigestAnnotation] = "sha256:updated" for _, change := range []struct { - name string - mutate func(client.Object) - want bool + name string + metadata metav1.ObjectMeta + want bool }{ - {"unchanged", func(client.Object) {}, false}, - {"status noise", func(o client.Object) { o.SetResourceVersion("2") }, false}, - {"status observed", test.observe, true}, - {"stamp lost", func(o client.Object) { o.SetAnnotations(nil) }, true}, - {"workload changed", func(o client.Object) { o.GetAnnotations()[dynamolpx.WorkloadDigestAnnotation] = "sha256:updated" }, true}, - {"labels changed", func(o client.Object) { o.SetLabels(map[string]string{"changed": "true"}) }, true}, - {"owner changed", func(o client.Object) { o.SetOwnerReferences([]metav1.OwnerReference{{UID: "new-owner"}}) }, true}, - {"deleting", func(o client.Object) { now := metav1.Now(); o.SetDeletionTimestamp(&now) }, true}, - {"spec generation changed", func(o client.Object) { o.SetGeneration(o.GetGeneration() + 1) }, true}, + {"unchanged", metadata, false}, + {"resource version", metav1.ObjectMeta{Annotations: metadata.Annotations, ResourceVersion: "2"}, false}, + {"stamp lost", metav1.ObjectMeta{}, true}, + {"workload", metav1.ObjectMeta{Annotations: workloadAnnotations}, true}, + {"labels", metav1.ObjectMeta{Annotations: metadata.Annotations, Labels: map[string]string{"changed": "true"}}, true}, + {"owner", metav1.ObjectMeta{Annotations: metadata.Annotations, OwnerReferences: []metav1.OwnerReference{{UID: "new-owner"}}}, true}, + {"deleting", metav1.ObjectMeta{Annotations: metadata.Annotations, DeletionTimestamp: ptr.To(metav1.Now())}, true}, + {"generation", metav1.ObjectMeta{Annotations: metadata.Annotations, Generation: 1}, true}, } { t.Run(change.name, func(t *testing.T) { - updated := old.DeepCopyObject().(client.Object) - change.mutate(updated) - require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: updated})) - require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: updated, ObjectNew: old})) + updated := test.object.DeepCopyObject().(client.Object) + updated.SetAnnotations(change.metadata.Annotations) + updated.SetLabels(change.metadata.Labels) + updated.SetOwnerReferences(change.metadata.OwnerReferences) + updated.SetDeletionTimestamp(change.metadata.DeletionTimestamp) + updated.SetGeneration(change.metadata.Generation) + updated.SetResourceVersion(change.metadata.ResourceVersion) + require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: test.object, ObjectNew: updated})) + require.Equal(t, change.want, test.predicate.Update(event.UpdateEvent{ObjectOld: updated, ObjectNew: test.object})) }) } }) } } -func TestLPXSourceWatchIgnoresOrdinaryChurn(t *testing.T) { - t.Log("Ordinary capacity and DGD status edits must not rerun LPX compilation") - source := newLPXTestSource(dynamolpx.PipelineSingle, "build-v2") - source.Spec.Components = append(source.Spec.Components, nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", Replicas: ptr.To(int32(1))}) - changed := source.DeepCopy() - changed.Generation++ - changed.Spec.Components[1].Replicas = ptr.To(int32(4)) - changed.Status.State = nvidiacomv1beta1.DGDStatePending - predicate := lpxSourcePredicate() - require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - - t.Log("Ignore ordinary creation and deletion, but keep LPX creation, deletion and deselection") - ordinary := source.DeepCopy() +func TestPodCliquePredicateRoles(t *testing.T) { + t.Log("Observe compiled LPX roles and template-hash convergence") + filter := podCliquePredicate() + for _, test := range []struct { + role string + want bool + }{ + {lpxv1alpha1.PodRoleAgent, true}, + {lpxv1alpha1.PodRoleConductor, true}, + {lpxv1alpha1.PodRoleCyborgWorker, true}, + {"ordinary", false}, + } { + t.Run(test.role, func(t *testing.T) { + clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + lpx.WorkloadDigestAnnotation: "sha256:workload", + lpxv1alpha1.PodRoleAnnotation: test.role, + }}} + require.Equal(t, test.want, filter.Create(event.CreateEvent{Object: clique})) + current := clique.DeepCopy() + current.Status.CurrentPodTemplateHash = ptr.To("current-template") + require.Equal(t, test.want, filter.Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: current})) + }) + } +} + +func TestDGDPredicate(t *testing.T) { + t.Log("Observe LPX input changes, not ordinary capacity and status churn") + dgd := loadTestDGD(t, lpx.PipelineSingle, "build-v2") + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", Replicas: ptr.To(int32(1))}) + filter := dgdPredicate() + + ordinary := dgd.DeepCopy() ordinary.Spec.Components = ordinary.Spec.Components[1:] ordinary.Generation++ - require.False(t, predicate.Create(event.CreateEvent{Object: ordinary})) - require.False(t, predicate.Delete(event.DeleteEvent{Object: ordinary})) - require.True(t, predicate.Create(event.CreateEvent{Object: source})) - require.True(t, predicate.Delete(event.DeleteEvent{Object: source})) - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: ordinary})) - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: source})) - - t.Log("LPX input changes and persisted restart selection wake the child") - changed.Spec.Components[0].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "new-runtime" - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed = source.DeepCopy() - changed.Generation++ - changed.Spec.Components[0].TopologyConstraint = &nvidiacomv1beta1.TopologyConstraint{PackDomain: "rack"} - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed = source.DeepCopy() - changed.Generation++ - changed.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed = source.DeepCopy() - changed.Spec.Restart = &nvidiacomv1beta1.Restart{ID: "restart"} - require.False(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed.Status.Restart = &nvidiacomv1beta1.RestartStatus{ObservedID: "restart", Phase: nvidiacomv1beta1.RestartPhaseRestarting, InProgress: []string{"lpx"}} - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed = source.DeepCopy() - changed.UID = "replacement" - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - - t.Log("Inherited scheduler metadata still wakes the child without a new source generation") - changed = source.DeepCopy() - changed.Labels = map[string]string{"priorityClassName": "inference"} - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) - changed = source.DeepCopy() - changed.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - require.True(t, predicate.Update(event.UpdateEvent{ObjectOld: source, ObjectNew: changed})) -} + require.False(t, filter.Create(event.CreateEvent{Object: ordinary})) + require.False(t, filter.Delete(event.DeleteEvent{Object: ordinary})) + require.True(t, filter.Create(event.CreateEvent{Object: dgd})) + require.True(t, filter.Delete(event.DeleteEvent{Object: dgd})) + + statusOnly := dgd.DeepCopy() + statusOnly.Status.State = v1beta1.DGDStatePending + capacity := dgd.DeepCopy() + capacity.Generation++ + capacity.Spec.Components[1].Replicas = ptr.To(int32(4)) + capacity.Status.State = v1beta1.DGDStatePending + image := dgd.DeepCopy() + image.Generation++ + image.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "new-runtime" + componentTopology := dgd.DeepCopy() + componentTopology.Generation++ + componentTopology.Spec.Components[0].TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} + graphTopology := dgd.DeepCopy() + graphTopology.Generation++ + graphTopology.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "fabric"} + restart := dgd.DeepCopy() + restart.Spec.Restart = &v1beta1.Restart{ID: "restart"} + selectedRestart := restart.DeepCopy() + selectedRestart.Status.Restart = &v1beta1.RestartStatus{ObservedID: "restart", Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{"lpx"}} + replacement := dgd.DeepCopy() + replacement.UID = "replacement" + labels := dgd.DeepCopy() + labels.Labels = map[string]string{"priorityClassName": "inference"} + annotations := dgd.DeepCopy() + annotations.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) -func BenchmarkLPXSourceWatchStatusChurn(b *testing.B) { - source := &nvidiacomv1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: "inference", Namespace: "workloads", Generation: 1}, - Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ - Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{{ - ComponentName: "lpx", ComponentType: nvidiacomv1beta1.ComponentTypeLPX, - LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "model/build"}, - Roles: []nvidiacomv1beta1.ComponentRoleSpec{{Name: nvidiacomv1beta1.ComponentRoleLPXConductor}, { - Name: nvidiacomv1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "lpu-runtime:test"}}}, - }, - }}, - }}, - }, - } - updated := source.DeepCopy() - updated.Status.State = nvidiacomv1beta1.DGDStatePending - update := event.UpdateEvent{ObjectOld: source, ObjectNew: updated} - filter := lpxSourcePredicate() - b.ReportAllocs() - for b.Loop() { - if filter.Update(update) { - b.Fatal("status-only updates must not enqueue reconciliation") - } + for _, test := range []struct { + name string + changed *v1beta1.DynamoGraphDeployment + want bool + }{ + {"status only", statusOnly, false}, + {"ordinary capacity and status", capacity, false}, + {"LPX deselection", ordinary, true}, + {"runtime image", image, true}, + {"component topology", componentTopology, true}, + {"graph topology", graphTopology, true}, + {"unselected restart", restart, false}, + {"selected restart", selectedRestart, true}, + {"DGD replacement", replacement, true}, + {"inherited labels", labels, true}, + {"inherited annotations", annotations, true}, + } { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, filter.Update(event.UpdateEvent{ObjectOld: dgd, ObjectNew: test.changed})) + require.Equal(t, test.want, filter.Update(event.UpdateEvent{ObjectOld: test.changed, ObjectNew: dgd})) + }) } } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index a1ce2feb5614..2fb2025b1a1c 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "maps" "slices" @@ -21,7 +22,6 @@ import ( dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -105,32 +105,8 @@ func LPXRestartToken(source *v1beta1.DynamoGraphDeployment, previous string) str return previous } -// ValidateLPXSource rejects foreign, stale, or fabricated child handoffs before -// the LPX controller acquires a build or publishes a workload. -// Both objects must be non-nil. -func ValidateLPXSource(deployment *v1alpha1.LPXGraphDeployment, source *v1beta1.DynamoGraphDeployment) error { - owner := metav1.GetControllerOf(deployment) - if deployment.Namespace != source.Namespace || - owner == nil || owner.APIVersion != v1beta1.GroupVersion.String() || owner.Kind != "DynamoGraphDeployment" || - owner.Name != source.Name || owner.UID != source.UID { - return fmt.Errorf("LPXGraphDeployment requires its exact source DGD and controller owner") - } - if provider := source.Annotations[commonconsts.KubeAnnotationWorkloadProvider]; provider != "" && provider != commonconsts.WorkloadProviderGrove { - return fmt.Errorf("LPX requires the Grove workload provider") - } - if !source.DeletionTimestamp.IsZero() { - return fmt.Errorf("source DGD is being deleted") - } - restart := LPXRestartToken(source, deployment.Annotations[LPXRestartAnnotation]) - revision, err := LPXInputRevision(source, restart) - if err != nil { - return err - } - if deployment.Spec.InputRevision != revision || deployment.Annotations[LPXRestartAnnotation] != restart { - return fmt.Errorf("LPXGraphDeployment is waiting for the current source input revision") - } - return nil -} +// ErrLPXSourcePending means the source and child snapshots await a matching handoff. +var ErrLPXSourcePending = errors.New("LPXGraphDeployment is waiting for the current source input revision") // lpxInputRevisionPayload is the normalized source intent hashed for inputRevision. // Keep complete selection and rendering inputs so nested role templates and build diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index d5c71809b143..c2023e3fa554 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -36,13 +36,14 @@ func TestBackendDetectionSkipsNonWorkers(t *testing.T) { } } -func TestRenderSelectedLPXRolePodSettings(t *testing.T) { +func TestRenderSelectedLPXRolePreservesTemplate(t *testing.T) { tests := []struct { name string authored *corev1.PodSecurityContext sharedMemorySize *resource.Quantity volumes []corev1.Volume mounts []corev1.VolumeMount + container corev1.Container }{ { name: "no authored security context", @@ -76,6 +77,22 @@ func TestRenderSelectedLPXRolePodSettings(t *testing.T) { mounts: []corev1.VolumeMount{{Name: "shared-memory", MountPath: "/dev/shm"}}, }, {name: "explicit zero shared memory size", sharedMemorySize: ptr.To(resource.MustParse("0"))}, + {name: "image entrypoint with arguments", container: corev1.Container{Args: []string{"serve"}}}, + {name: "explicit command with image arguments", container: corev1.Container{Command: []string{"/custom-runtime"}}}, + {name: "shell startup", container: corev1.Container{ + Command: []string{"/bin/sh", "-c"}, Args: []string{"exec /custom-runtime --allocation $LPX_ALLOCATION"}, + }}, + {name: "SGLang startup without backend injection", container: corev1.Container{ + Command: []string{"python3"}, Args: []string{"-m", "dynamo.sglang"}, + }}, + {name: "TensorRT-LLM startup without backend injection", container: corev1.Container{ + Command: []string{"python3"}, Args: []string{"-m", "dynamo.trtllm"}, + }}, + {name: "authored health probes", container: corev1.Container{ + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/health", "started"}}}, FailureThreshold: 12}, + LivenessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/health", "live"}}}, PeriodSeconds: 6}, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/health", "ready"}}}, TimeoutSeconds: 2}, + }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -83,10 +100,12 @@ func TestRenderSelectedLPXRolePodSettings(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "hybrid", Namespace: "test"}, } + container := test.container.DeepCopy() + container.Name, container.Image = "main", "cyborg:test" component := &v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "engine", ComponentType: v1beta1.ComponentTypeDecode, SharedMemorySize: test.sharedMemorySize, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "main", Image: "cyborg:test"}}, + Containers: []corev1.Container{*container}, SecurityContext: test.authored, }}, } @@ -107,6 +126,14 @@ func TestRenderSelectedLPXRolePodSettings(t *testing.T) { require.Equal(t, test.authored, template.Spec.SecurityContext) require.ElementsMatch(t, test.volumes, template.Spec.Volumes) require.ElementsMatch(t, test.mounts, template.Spec.Containers[0].VolumeMounts) + + t.Log("Leave startup and health checks template-owned, including omission") + main := template.Spec.Containers[0] + require.Equal(t, container.Command, main.Command) + require.Equal(t, container.Args, main.Args) + require.Equal(t, container.StartupProbe, main.StartupProbe) + require.Equal(t, container.LivenessProbe, main.LivenessProbe) + require.Equal(t, container.ReadinessProbe, main.ReadinessProbe) }) } } diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 47e862d68b25..de7abac052f4 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -7,7 +7,6 @@ import ( "context" "fmt" - v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovecommon "github.com/ai-dynamo/grove/operator/api/common" @@ -23,7 +22,9 @@ import ( // EvaluateLPXGroveReadiness observes one complete engine, including every role // in every replica. Ordinary DGD components are deliberately not evaluated here. -func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet) (GroveReadiness, error) { +// source is non-nil. pcs and group may be nil while materializing; otherwise the +// caller has verified ownership, deletion state, and the single-group template. +func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, group *grovev1alpha1.PodCliqueScalingGroup) (GroveReadiness, error) { component := lpx.ServingComponent(source) status := v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, RuntimeNamespace: source.GetDynamoNamespaceForComponent(component)} // Draft instances are counted from their own complete Agent cliques. @@ -51,8 +52,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source pending := func(message string) GroveReadiness { return result(false, v1beta1.DGDReadyReasonSomeResourcesNotReady, message) } - if pcs == nil || !pcs.DeletionTimestamp.IsZero() || !metav1.IsControlledBy(pcs, deployment) || - len(pcs.Spec.Template.PodCliqueScalingGroupConfigs) != 1 { + if pcs == nil { return pending("Waiting for the exact LPX PodCliqueSet"), nil } hash := getAcceptedPCSRevisionHash(pcs) @@ -62,16 +62,12 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source config := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] groupName := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name) status.ComponentNames = []string{groupName} - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: groupName}, group); err != nil { - if apierrors.IsNotFound(err) { - return pending("Waiting for the LPX scaling group"), nil - } - return GroveReadiness{}, err + if group == nil { + return pending("Waiting for the LPX scaling group"), nil } status.Replicas, status.UpdatedReplicas = group.Status.Replicas, group.Status.UpdatedReplicas status.AvailableReplicas = ptr.To(group.Status.AvailableReplicas) - if !metav1.IsControlledBy(group, pcs) || !group.DeletionTimestamp.IsZero() || group.Status.ObservedGeneration == nil || *group.Status.ObservedGeneration != group.Generation { + if group.Status.ObservedGeneration == nil || *group.Status.ObservedGeneration != group.Generation { return pending("Waiting for the exact observed LPX scaling group"), nil } status.ScheduledReplicas = ptr.To(group.Status.ScheduledReplicas) diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go index d15e2df88b2f..525d2150c961 100644 --- a/deploy/operator/internal/dynamo/grove_lpx_test.go +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -6,7 +6,6 @@ package dynamo import ( "testing" - v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,7 +33,7 @@ func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{draft, serving}}, } - readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, &v1alpha1.LPXGraphDeployment{}, nil) + readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, nil, nil) require.NoError(t, err) require.NotEqual(t, source.GetDynamoNamespaceForComponent(&draft), source.GetDynamoNamespaceForComponent(&serving)) require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 60b575443116..7f9f8567c85c 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -77,6 +77,7 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { wantOffsets []int }{ {name: "legacy 14 endpoints", partitions: 14, wantOffsets: []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26}}, + {name: "one chain across all partitions", partitions: 3, chains: [][]uint32{{0, 1, 2}}, wantOffsets: []int{0}}, {name: "chain followed by endpoint", partitions: 4, chains: [][]uint32{{1, 2}}, wantOffsets: []int{0, 2, 6}}, {name: "two chains followed by endpoint", partitions: 5, chains: [][]uint32{{0, 1}, {2, 3}}, wantOffsets: []int{0, 4, 8}}, } { diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index fd023f6242cb..4b5dc16e9325 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -35,7 +35,7 @@ func TestPlanMaterializationBounds(t *testing.T) { {name: "hybrid", models: 1, projection: hybridProjection}, {name: "LPU-only", models: 1, projection: lpuOnlyProjection}, {name: "SpecDecode", models: 2, projection: lpuOnlyProjection}, - {name: "maximum draft fanout", models: maxSpecDecodeNumDrafts + 1, projection: lpuOnlyProjection}, + {name: "maximum draft fanout", models: MaxSpecDecodeNumDrafts + 1, projection: lpuOnlyProjection}, } { t.Run(test.name, func(t *testing.T) { t.Log("Build independent projections without using component names in identities") diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index 9d50504f623c..b2be3910870a 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -6,10 +6,8 @@ package lpx import ( - "slices" "testing" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -122,74 +120,3 @@ func lpuResourceLists(spec *corev1.PodSpec) []corev1.ResourceList { } return lists } - -func TestConfigureNodeLocalRuntimeBindings(t *testing.T) { - for _, role := range []string{"conductor", "agent"} { - t.Run(role, func(t *testing.T) { - t.Log("Author main-container references and resource requirements") - pod := corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: commonconsts.MainContainerName, - Env: []corev1.EnvVar{ - {Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, - {Name: "MAIN_CPU", ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: commonconsts.MainContainerName, Resource: "limits.cpu", - }}}, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, - Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, - }, - }}, - Volumes: []corev1.Volume{{Name: "resources", VolumeSource: corev1.VolumeSource{ - DownwardAPI: &corev1.DownwardAPIVolumeSource{Items: []corev1.DownwardAPIVolumeFile{{ - Path: "main-memory", ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: commonconsts.MainContainerName, Resource: "limits.memory", - }, - }}}, - }}}, - } - before := pod.DeepCopy() - - t.Log("Bind the selected role and retarget every authored main-container reference") - if role == "conductor" { - configureNodeLocalConductorRuntime(&pod, "agt") - } else { - configureAgentIdentity(&pod) - } - container := pod.Containers[0] - require.Equal(t, role, container.Name) - require.Equal(t, role, testContainerEnvValue(container.Env, "CONTAINER_NAME")) - require.Equal(t, role, testContainerEnvSource(container.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) - require.Equal(t, role, pod.Volumes[0].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) - require.Equal(t, before.Containers[0].Resources, container.Resources) - - t.Log("Bind placement only to the conductor") - if role == "conductor" { - require.Equal(t, "agt", testContainerEnvValue(container.Env, "LPX_ALLOCATION")) - } - }) - } -} - -func testExecProbe(command string) *corev1.Probe { - return &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{command}}}, - } -} - -func testContainerEnvValue(env []corev1.EnvVar, name string) string { - index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) - if index < 0 { - return "" - } - return env[index].Value -} - -func testContainerEnvSource(env []corev1.EnvVar, name string) *corev1.EnvVarSource { - index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) - if index < 0 { - return nil - } - return env[index].ValueFrom -} diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 2fa61eb13af9..eb362459e124 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -224,7 +224,7 @@ func RenderSelectedNodeLocal( } } - // The LPX-only PCS owns one scaling group for the complete engine. + // Seed the complete-engine group at its immutable minimum; live capacity is managed through /scale. members := selectedTemplateNames if hybrid { members = append(members, plan.CyborgTemplate) @@ -233,7 +233,7 @@ func RenderSelectedNodeLocal( Name: lpxScalingGroupTemplateName, CliqueNames: members, Annotations: map[string]string{WorkloadDigestAnnotation: workloadDigest}, - Replicas: ptr.To(plan.Replicas), + Replicas: ptr.To(ptr.Deref(input.MinAvailable, 1)), MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), }} diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index daa812641045..b5b7d1ef0c22 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -6,6 +6,7 @@ package lpx import ( + "slices" "strings" "testing" @@ -62,19 +63,34 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { t.Log("Cover XT configuration mounts and HX hybrid model-storage paths") hybrid := newV3CompilerFixture() hybrid.compilationMode = manifestcapnp.CompilationMode_lpx + xtSnapshot := acquireTestSnapshot(t, writeV2CompilerFixture(t)) + hxSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) tests := []struct { - name string - family lpxv1alpha1.TargetFamily - pipeline Pipeline - snapshot *BuildSnapshot + name string + family lpxv1alpha1.TargetFamily + pipeline Pipeline + snapshot *BuildSnapshot + configPath string }{ { name: "XT config mount", family: lpxv1alpha1.TargetFamilyXt8888, pipeline: PipelineSingle, - snapshot: acquireTestSnapshot(t, writeV2CompilerFixture(t)), + snapshot: xtSnapshot, configPath: "/custom", + }, + { + name: "XT omitted config mount", family: lpxv1alpha1.TargetFamilyXt8888, pipeline: PipelineSingle, + snapshot: xtSnapshot, + }, + { + name: "HX omitted config mount", family: lpxv1alpha1.TargetFamilyHx16x8x2x3, pipeline: PipelineSingle, + snapshot: hxSnapshot, + }, + { + name: "HX custom config mount", family: lpxv1alpha1.TargetFamilyHx16x8x2x3, pipeline: PipelineSingle, + snapshot: hxSnapshot, configPath: "/custom", }, { name: "HX hybrid storage", family: lpxv1alpha1.TargetFamilyHx16x8x2x3, pipeline: PipelineLPX, - snapshot: acquireTestSnapshot(t, writeCompilerFixture(t, hybrid)), + snapshot: acquireTestSnapshot(t, writeCompilerFixture(t, hybrid)), configPath: "/configs", }, } for _, test := range tests { @@ -101,10 +117,25 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { }}, Spec: renderTestPodSpec(), } - if test.family == lpxv1alpha1.TargetFamilyXt8888 { + template.Spec.Tolerations = []corev1.Toleration{ + {Key: "cluster.example/custom", Operator: corev1.TolerationOpExists}, + {Key: "lpu.nvidia.com/node", Operator: corev1.TolerationOpExists}, + } + if test.family == lpxv1alpha1.TargetFamilyXt8888 && test.configPath != "" { + template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{ + Name: lpuConfigVolumeName, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "authored-config"}, DefaultMode: ptr.To[int32](0440), + }}, + }) + } + template.Spec.Containers[0].VolumeMounts = slices.DeleteFunc(template.Spec.Containers[0].VolumeMounts, + func(mount corev1.VolumeMount) bool { return mount.Name == lpuConfigVolumeName }) + if test.configPath != "" { template.Spec.Containers[0].VolumeMounts = append(template.Spec.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) - } else { + corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: test.configPath}) + } + if test.pipeline == PipelineLPX { template.Spec.Containers[0].VolumeMounts[0].MountPath = "/model-cache" pcs.Spec.Template.Cliques[0].Spec.PodSpec.Containers[0].VolumeMounts[1].MountPath = "/model-cache" } @@ -132,14 +163,16 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { require.Equal(t, projection.Digest().String(), agent.Annotations[WorkloadDigestAnnotation]) require.Equal(t, projection.CompilerSnapshotDigest(), agent.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) + t.Log("Preserve authored placement and volumes") + require.Equal(t, template.Spec.Tolerations, agent.Spec.PodSpec.Tolerations) + require.Subset(t, agent.Spec.PodSpec.Volumes, template.Spec.Volumes) + t.Log("Resolve runtime mounts using the authored storage path") - if test.family == lpxv1alpha1.TargetFamilyXt8888 { + if test.pipeline != PipelineLPX { conductor := namedClique(t, rendered, "cond") require.Equal(t, "kept", conductor.Annotations["user"]) - require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/custom"}) - require.Contains(t, conductor.Spec.PodSpec.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: lpuConfigVolumeName, MountPath: "/configs"}) + require.Equal(t, template.Spec.Containers[0].VolumeMounts, conductor.Spec.PodSpec.Containers[0].VolumeMounts) + require.Equal(t, template.Spec.Containers[0].VolumeMounts, agent.Spec.PodSpec.Containers[0].VolumeMounts) } else { cyborg := namedClique(t, rendered, "cond").Spec.PodSpec.Containers[0] require.Contains(t, cyborg.VolumeMounts, corev1.VolumeMount{Name: "model-storage", MountPath: "/model-cache"}) diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go index a9e68fde5ca0..aa70f11eeef0 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go @@ -11,6 +11,7 @@ import ( "fmt" "slices" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" @@ -32,17 +33,15 @@ type BuildSnapshotSource interface { AcquireBuildSnapshot(context.Context, string) (*BuildSnapshot, error) } -// ResolveSelectedWorkload validates and projects the selected LPX workload in dgd. -// dgd and source must be non-nil. The function reads but does not mutate dgd. +// ResolveSelectedWorkload projects admitted LPX intent against immutable builds. +// dgd and source must be non-nil; dgd must select LPX and have passed webhook validation. +// The function reads but does not mutate dgd. func ResolveSelectedWorkload( ctx context.Context, dgd *dynamov1beta1.DynamoGraphDeployment, source BuildSnapshotSource, ) (*SelectedWorkload, error) { - components, allErrs := validateSelectedIntent(dgd) - if err := allErrs.ToAggregate(); err != nil { - return nil, err - } + components := Components(dgd) projections := make([]*ModelProjection, 0, len(components)) var snapshot NormalizedBuildSnapshot var pipeline Pipeline @@ -190,3 +189,27 @@ func validateSelectedConductor( return nil } + +// validateRolePodSpecContainerNames checks both lists that share the Pod's name space. +// spec and fldPath must be non-nil. +func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path, reservedName string) field.ErrorList { + // Both lists must avoid the materialized role container name. + allErrs := field.ErrorList{} + for _, group := range []struct { + name string + containers []corev1.Container + }{ + {"containers", spec.Containers}, + {"initContainers", spec.InitContainers}, + } { + for containerIndex, container := range group.containers { + if container.Name == reservedName { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child(group.name).Index(containerIndex).Child("name"), + fmt.Sprintf("LPX reserves %q for the materialized role container", container.Name), + )) + } + } + } + return allErrs +} diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go index fef9980997c1..33075413e469 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go @@ -328,42 +328,6 @@ func TestResolveSelectedWorkloadPreservesAuthoredLaunch(t *testing.T) { require.Equal(t, before, dgd) } -func TestResolveSelectedWorkloadRejectsInvalidRolesBeforeBuildAcquisition(t *testing.T) { - t.Log("Author placement and container identity errors on both roles") - agent, conductor := testLPXPodTemplate("agent"), testLPXPodTemplate("conductor") - agent.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} - agent.Spec.Containers[0].Args = []string{"--allocation=forged", "--"} - agent.Spec.Hostname = "custom-host" - agent.Spec.Containers = append(agent.Spec.Containers, corev1.Container{Name: "agent", Image: "sidecar"}) - agent.Spec.InitContainers = append(agent.Spec.InitContainers, corev1.Container{Name: "agent", Image: "setup"}) - conductor.Spec.NodeName = "chosen-node" - conductor.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{ - MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule, - }} - dgd := newSelectedTestDGD(t, "selected", testLPXComponent("lpx", "build", - v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: agent}, - v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: conductor}, - )) - - t.Log("Aggregate every actionable error at its authored path before acquiring a build") - _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) - for _, message := range []string{ - "spec.components[0].roles[0].podTemplate.spec.hostname: Forbidden: LPX owns role addressing and placement", - `spec.components[0].roles[0].podTemplate.spec.containers[1].name: Forbidden: LPX reserves "agent" for the materialized role container`, - `spec.components[0].roles[0].podTemplate.spec.initContainers[0].name: Forbidden: LPX reserves "agent" for the materialized role container`, - "spec.components[0].roles[1].podTemplate.spec.nodeName: Forbidden: LPX owns role addressing and placement", - "spec.components[0].roles[1].podTemplate.spec.topologySpreadConstraints: Forbidden: LPX owns role placement", - } { - require.ErrorContains(t, err, message) - } - - t.Log("Reject missing main containers for both roles before acquiring a build") - agent.Spec.Containers, conductor.Spec.Containers = nil, nil - _, err = ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) - require.ErrorContains(t, err, `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX agent component requires a "main" runtime container`) - require.ErrorContains(t, err, `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX conductor component requires a "main" runtime container`) -} - func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t *testing.T) { t.Log("Acquire LPU-only and hybrid builds that select different runtime container identities") lpuSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) @@ -464,8 +428,6 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t before := dgd.DeepCopy() t.Log("Defer conductor-name checks until the immutable build has been acquired") - require.Empty(t, ValidateAgentContainerNames(dgd)) - require.Empty(t, ValidateSelectedIntent(dgd)) _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) require.ErrorIs(t, err, ErrBuildSnapshotAcquisition) diff --git a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go b/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go similarity index 76% rename from deploy/operator/internal/dynamo/lpx/runtime_startup_test.go rename to deploy/operator/internal/dynamo/lpx/runtime_agent_test.go index 1e7b242d8bf7..6c1f81270a7e 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_startup_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go @@ -10,8 +10,10 @@ import ( "slices" "testing" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/ptr" ) @@ -35,6 +37,7 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { }{ {name: "image defaults"}, {name: "image ENTRYPOINT with args", args: []string{"--instance-model-name", "custom-model", "--agent-env-vars=KEEP=1"}}, + {name: "explicit command with image arguments", command: []string{"/custom-launcher"}}, {name: "explicit command", command: []string{"/custom-launcher", "wrapper-option"}, args: []string{"argument with spaces", "literal $HOME", ""}}, } { t.Run(role.name+"/"+startup.name, func(t *testing.T) { @@ -50,6 +53,9 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { { Name: "main", Image: "custom-runtime", Command: startup.command, Args: startup.args, Env: []corev1.EnvVar{{Name: "LPX_ALLOCATION", Value: "forged-allocation"}}, + EnvFrom: []corev1.EnvFromSource{{ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "user-environment"}, + }}}, }, }} if startup.args != nil { @@ -120,6 +126,7 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { authored.Env = slices.DeleteFunc(slices.Clone(authored.Env), func(variable corev1.EnvVar) bool { return variable.Name == "LPX_ALLOCATION" }) } require.Equal(t, authored.Env, container.Env) + require.Equal(t, authored.EnvFrom, container.EnvFrom) require.Equal(t, authored.StartupProbe, container.StartupProbe) require.Equal(t, authored.ReadinessProbe, container.ReadinessProbe) require.Equal(t, authored.LivenessProbe, container.LivenessProbe) @@ -231,3 +238,74 @@ func TestModelPathsRejectInvalidReferences(t *testing.T) { }) } } + +func TestConfigureNodeLocalRuntimeBindings(t *testing.T) { + for _, role := range []string{"conductor", "agent"} { + t.Run(role, func(t *testing.T) { + t.Log("Author main-container references and resource requirements") + pod := corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, + Env: []corev1.EnvVar{ + {Name: "CONTAINER_NAME", Value: commonconsts.MainContainerName}, + {Name: "MAIN_CPU", ValueFrom: &corev1.EnvVarSource{ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, Resource: "limits.cpu", + }}}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }}, + Volumes: []corev1.Volume{{Name: "resources", VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{Items: []corev1.DownwardAPIVolumeFile{{ + Path: "main-memory", ResourceFieldRef: &corev1.ResourceFieldSelector{ + ContainerName: commonconsts.MainContainerName, Resource: "limits.memory", + }, + }}}, + }}}, + } + before := pod.DeepCopy() + + t.Log("Bind the selected role and retarget every authored main-container reference") + if role == "conductor" { + configureNodeLocalConductorRuntime(&pod, "agt") + } else { + configureAgentIdentity(&pod) + } + container := pod.Containers[0] + require.Equal(t, role, container.Name) + require.Equal(t, role, testContainerEnvValue(container.Env, "CONTAINER_NAME")) + require.Equal(t, role, testContainerEnvSource(container.Env, "MAIN_CPU").ResourceFieldRef.ContainerName) + require.Equal(t, role, pod.Volumes[0].DownwardAPI.Items[0].ResourceFieldRef.ContainerName) + require.Equal(t, before.Containers[0].Resources, container.Resources) + + t.Log("Bind placement only to the conductor") + if role == "conductor" { + require.Equal(t, "agt", testContainerEnvValue(container.Env, "LPX_ALLOCATION")) + } + }) + } +} + +func testExecProbe(command string) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{command}}}, + } +} + +func testContainerEnvValue(env []corev1.EnvVar, name string) string { + index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) + if index < 0 { + return "" + } + return env[index].Value +} + +func testContainerEnvSource(env []corev1.EnvVar, name string) *corev1.EnvVarSource { + index := slices.IndexFunc(env, func(value corev1.EnvVar) bool { return value.Name == name }) + if index < 0 { + return nil + } + return env[index].ValueFrom +} diff --git a/deploy/operator/internal/dynamo/lpx/selection.go b/deploy/operator/internal/dynamo/lpx/selection.go index 7708ee5a5db4..dbcbcefef525 100644 --- a/deploy/operator/internal/dynamo/lpx/selection.go +++ b/deploy/operator/internal/dynamo/lpx/selection.go @@ -9,10 +9,12 @@ import ( "fmt" dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "k8s.io/utils/ptr" ) const ( + // MaxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. + MaxSpecDecodeNumDrafts = 8 + draftStageName = "draft" runtimeModelDraft = "draft" runtimeModelTarget = "target" @@ -58,21 +60,6 @@ func ServingComponent(dgd *dynamov1beta1.DynamoGraphDeployment) *dynamov1beta1.D return nil } -// SelectedModelNames returns build-backed runtime model identities, in canonical -// stage and draft-instance order, for a supported runtime. -// dgd must be non-nil and is not mutated. -func SelectedModelNames(dgd *dynamov1beta1.DynamoGraphDeployment) ([]string, error) { - components := Components(dgd) - if err := validateLPXComposition(dgd, len(components)).ToAggregate(); err != nil { - return nil, fmt.Errorf("%w: %w", ErrUnsupportedRuntime, err) - } - var names []string - for index, component := range components { - names = append(names, expandedSelectedModelNames(index, len(components), int(ptr.Deref(component.Replicas, 1)))...) - } - return names, nil -} - func expandedSelectedModelNames(index, componentCount, draftCount int) []string { // Component membership retains Nova's existing default/draft/target identities. if componentCount == 1 { diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go index 2fd8cd42f594..7f020fe923d3 100644 --- a/deploy/operator/internal/dynamo/lpx/selection_test.go +++ b/deploy/operator/internal/dynamo/lpx/selection_test.go @@ -10,14 +10,8 @@ import ( dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { @@ -69,121 +63,10 @@ func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { } } -func TestSelectedModelNames(t *testing.T) { - t.Log("Construct a SpecDecode selection with draft fanout") - draftReplicas := int32(3) - dgd := &dynamov1beta1.DynamoGraphDeployment{Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ - Components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{ - testLPXComponent("small", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}), - testLPXComponent("large", "target-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}), - }, - }} - dgd.Spec.Components[0].Replicas = &draftReplicas - - t.Log("Expand the configured draft replica count into logical model names") - names, err := SelectedModelNames(dgd) - require.NoError(t, err) - require.Equal(t, []string{"draft0", "draft1", "draft2", "target"}, names) - - t.Log("Default an omitted draft replica count to one logical draft") - dgd.Spec.Components[0].Replicas = nil - names, err = SelectedModelNames(dgd) - require.NoError(t, err) - require.Equal(t, []string{"draft0", "target"}, names) - - for _, invalid := range []int32{0, 9} { - t.Logf("Reject invalid draft replica count %d", invalid) - dgd.Spec.Components[0].Replicas = &invalid - _, err = SelectedModelNames(dgd) - require.ErrorIs(t, err, ErrUnsupportedRuntime) - } - t.Log("Derive the default runtime identity for a single serving component") - dgd.Spec.Components = dgd.Spec.Components[1:] - names, err = SelectedModelNames(dgd) - require.NoError(t, err) - require.Equal(t, []string{"default"}, names) - require.Same(t, &dgd.Spec.Components[0], ServingComponent(dgd)) -} - -func TestSelectedModelNamesRequiresSharedConductorOwner(t *testing.T) { - t.Log("Read two agent-only components without a shared conductor owner") - dgd := newSelectedTestDGD(t, "missing-conductor", - testLPXComponent("small", "build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}), - testLPXComponent("large", "build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}), - ) - before := dgd.DeepCopy() - - t.Log("Reject the incomplete runtime without inventing a conductor") - names, err := SelectedModelNames(dgd) - require.ErrorIs(t, err, ErrUnsupportedRuntime) - require.Nil(t, names) - require.Nil(t, ServingComponent(dgd)) - require.Equal(t, before, dgd) -} - -func TestSelectedMinAvailableOwnership(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - draftMin *int32 - targetMin *int32 - singleton bool - wantErr bool - }{ - {name: "omitted draft"}, - {name: "default draft", draftMin: ptr.To(int32(1))}, - {name: "nondefault draft", draftMin: ptr.To(int32(2)), wantErr: true}, - {name: "target", targetMin: ptr.To(int32(1))}, - {name: "singleton", targetMin: ptr.To(int32(2)), singleton: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Put a non-LPX component before the target and reverse the authored LPX order") - target := testLPXComponent("target", "target-build", - dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, - dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("target")}) - target.MinAvailable = test.targetMin - if test.singleton { - target.Replicas = ptr.To(int32(2)) - } - dgd := newSelectedTestDGD(t, "min-available", - dynamov1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", MinAvailable: ptr.To(int32(1))}, target) - if !test.singleton { - draft := testLPXComponent("draft", "draft-build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("draft")}) - draft.Replicas = ptr.To(int32(2)) - draft.MinAvailable = test.draftMin - dgd.Spec.Components = append(dgd.Spec.Components, draft) - } - - for _, phase := range []string{"authored", "defaulted"} { - t.Run(phase, func(t *testing.T) { - if phase == "defaulted" { - t.Log("Apply the real Grove CREATE defaulter before selected-LPX validation") - ctx := features.WithGate(t.Context(), features.Gates{Grove: true}) - ctx = admission.NewContextWithRequest(ctx, admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Create, - Kind: metav1.GroupVersionKind(dynamov1beta1.DynamoGraphDeploymentGVK), - }}) - require.NoError(t, defaulting.NewDGDDefaulter("test").Default(ctx, dgd)) - if !test.singleton { - require.Equal(t, ptr.To(ptr.Deref(test.draftMin, 1)), dgd.Spec.Components[2].MinAvailable) - } - } - before := dgd.DeepCopy() - - t.Log("Reject only nondefault draft availability at its authored index without mutation") - errs := ValidateSelectedIntent(dgd) - require.Equal(t, before, dgd) - if test.wantErr { - require.Len(t, errs, 1) - require.Equal(t, field.ErrorTypeForbidden, errs[0].Type) - require.Equal(t, "spec.components[2].minAvailable", errs[0].Field) - require.Contains(t, errs[0].Detail, "must be omitted or 1") - return - } - require.Empty(t, errs) - }) - } - }) - } +func TestExpandedSelectedModelNames(t *testing.T) { + t.Log("Expand admitted component replicas into runtime model names") + require.Equal(t, []string{"draft0", "draft1", "draft2"}, expandedSelectedModelNames(0, 2, 3)) + require.Equal(t, []string{"draft0"}, expandedSelectedModelNames(0, 2, 1)) + require.Equal(t, []string{"target"}, expandedSelectedModelNames(1, 2, 1)) + require.Equal(t, []string{"default"}, expandedSelectedModelNames(0, 1, 2)) } diff --git a/deploy/operator/internal/dynamo/lpx/selection_validation.go b/deploy/operator/internal/dynamo/lpx/selection_validation.go deleted file mode 100644 index 8e356100983e..000000000000 --- a/deploy/operator/internal/dynamo/lpx/selection_validation.go +++ /dev/null @@ -1,276 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - "strings" - - dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/ptr" -) - -// maxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. -const maxSpecDecodeNumDrafts = 8 - -// ValidateSelectedIntent validates the snapshot-independent selected-LPX -// requirements beyond admission's structural checks before controller downloads -// and selected-workload projection. -// dgd must be non-nil. -func ValidateSelectedIntent(dgd *dynamov1beta1.DynamoGraphDeployment) field.ErrorList { - _, allErrs := validateSelectedIntent(dgd) - return allErrs -} - -func validateSelectedIntent(dgd *dynamov1beta1.DynamoGraphDeployment) ([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, field.ErrorList) { - specPath := field.NewPath("spec") - if !dgd.HasLPXComponent() { - return nil, field.ErrorList{field.Forbidden( - specPath.Child("components"), - "DynamoGraphDeployment has no LPX component", - )} - } - allErrs := field.ErrorList{} - if dgd.Spec.TopologyConstraint != nil { - allErrs = append(allErrs, field.Forbidden( - specPath.Child("topologyConstraint"), - "selected LPX does not support Grove topologyConstraint", - )) - } - if dgd.Spec.ProviderOverride != nil { - allErrs = append(allErrs, field.Forbidden( - specPath.Child("providerOverride"), - "selected LPX does not support Grove topology overrides on the deployment", - )) - } - components := Components(dgd) - allErrs = append(allErrs, validateLPXComposition(dgd, len(components))...) - allErrs = append(allErrs, ValidateAgentContainerNames(dgd)...) - - // Keep errors tied to authored component and role indices rather than runtime order. - for index := range dgd.Spec.Components { - component := &dgd.Spec.Components[index] - if !component.IsLPX() { - continue - } - componentPath := field.NewPath("spec", "components").Index(index) - allErrs = append(allErrs, validateSelectedLPXComponent(component, componentPath)...) - } - return components, allErrs -} - -// validateLPXComposition bounds the shared runtime before model expansion or acquisition. -func validateLPXComposition(dgd *dynamov1beta1.DynamoGraphDeployment, componentCount int) field.ErrorList { - componentsPath := field.NewPath("spec", "components") - if componentCount < 1 || componentCount > 2 { - return field.ErrorList{field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")} - } - allErrs := field.ErrorList{} - for index := range dgd.Spec.Components { - component := &dgd.Spec.Components[index] - if !component.IsLPX() { - continue - } - componentPath := componentsPath.Index(index) - conductor := component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) - if componentCount != 2 { - continue - } - // Draft fanout is per model instance; the target owns one shared serving runtime. - replicas := ptr.Deref(component.Replicas, 1) - if conductor == nil { - if replicas < 1 || replicas > maxSpecDecodeNumDrafts { - allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "draft replicas must be between 1 and 8")) - } - if component.ModelRef != nil { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("modelRef"), "the shared target owns the serving endpoint")) - } - // Grove persists its immutable default of 1 even for drafts. - if ptr.Deref(component.MinAvailable, 1) != 1 { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("minAvailable"), "draft minAvailable must be omitted or 1; the shared target owns minimum availability")) - } - } else if replicas != 1 { - allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "shared target replicas must be one")) - } - } - allErrs = append(allErrs, ValidateConductorRoles(&dgd.Spec, componentsPath, componentsPath.Index)...) - return allErrs -} - -// validateSelectedLPXComponent validates inputs required by the existing runtime. -// Shared role admission owns structure; composition validation bounds materialization. -func validateSelectedLPXComponent(component *dynamov1beta1.DynamoComponentDeploymentSharedSpec, componentPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if component.Experimental != nil && component.Experimental.Checkpoint != nil && component.Experimental.Checkpoint.Enabled { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("experimental", "checkpoint"), - fmt.Sprintf("selected LPX component %q does not support checkpointing", component.ComponentName))) - } - if component.TopologyConstraint != nil { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("topologyConstraint"), - fmt.Sprintf("selected LPX component %q does not support Grove topologyConstraint", component.ComponentName))) - } - if component.ProviderOverride != nil { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("providerOverride"), "LPX component does not support Grove topology overrides")) - } - if component.IsMultinode() { - allErrs = append(allErrs, field.Forbidden(componentPath.Child("multinode"), "LPX component cannot use multinode")) - } - config := component.LPX - configPath := componentPath.Child("lpx") - if strings.TrimSpace(config.BuildID) == "" { - allErrs = append(allErrs, field.Required(configPath.Child("buildId"), "LPX component requires a buildId")) - } - for index, role := range component.Roles { - rolePath := componentPath.Child("roles").Index(index) - if role.PodTemplate == nil { - continue - } - allErrs = append(allErrs, validateSelectedRuntimeContainer(role.PodTemplate, "LPX "+role.Name, rolePath)...) - allErrs = append(allErrs, validateLPXRolePlacement(&role.PodTemplate.Spec, rolePath.Child("podTemplate", "spec"), role.Name == dynamov1beta1.ComponentRoleLPXAgent)...) - } - return allErrs -} - -// ValidateConductorRoles requires one explicit serving template across LPX components. -// Draft components remain Agent-only. Arguments must be non-nil; spec is not mutated. -// componentsPath and componentPath locate the collection and each component in the source API. -func ValidateConductorRoles(spec *dynamov1beta1.DynamoGraphDeploymentSpec, componentsPath *field.Path, componentPath func(int) *field.Path) field.ErrorList { - // Inspect authored roles so admission and preflight enforce the same snapshot-independent shape. - allErrs := field.ErrorList{} - hasLPX, conductorCount := false, 0 - for componentIndex := range spec.Components { - component := &spec.Components[componentIndex] - if !component.IsLPX() { - continue - } - hasLPX = true - for roleIndex, role := range component.Roles { - if role.Name != dynamov1beta1.ComponentRoleLPXConductor { - continue - } - conductorCount++ - if role.PodTemplate == nil { - rolePath := componentPath(componentIndex).Child("roles").Index(roleIndex) - allErrs = append(allErrs, field.Required(rolePath.Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) - } - } - } - if hasLPX && conductorCount != 1 { - allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must declare exactly one conductor role")) - } - return allErrs -} - -// ValidateAgentContainerNames reserves the Agent identity in authored LPX agent -// templates. Conductor identities depend on the immutable build's execution mode. -// dgd must be non-nil and is not mutated. -func ValidateAgentContainerNames(dgd *dynamov1beta1.DynamoGraphDeployment) field.ErrorList { - // Every LPX agent template renames main independently of the selected build. - allErrs := field.ErrorList{} - for componentIndex := range dgd.Spec.Components { - component := &dgd.Spec.Components[componentIndex] - if !component.IsLPX() { - continue - } - - // Keep collision errors on each authored agent template. - for roleIndex, role := range component.Roles { - if role.Name != dynamov1beta1.ComponentRoleLPXAgent || role.PodTemplate == nil { - continue - } - podSpecPath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec") - allErrs = append(allErrs, validateRolePodSpecContainerNames(&role.PodTemplate.Spec, podSpecPath, lpuAgentContainerName)...) - } - } - return allErrs -} - -// validateRolePodSpecContainerNames checks both lists that share the Pod's name space. -// spec and fldPath must be non-nil. -func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path, reservedName string) field.ErrorList { - // Both lists must avoid the materialized role container name. - allErrs := field.ErrorList{} - for _, group := range []struct { - name string - containers []corev1.Container - }{ - {"containers", spec.Containers}, - {"initContainers", spec.InitContainers}, - } { - for containerIndex, container := range group.containers { - if container.Name == reservedName { - allErrs = append(allErrs, field.Forbidden( - fldPath.Child(group.name).Index(containerIndex).Child("name"), - fmt.Sprintf("LPX reserves %q for the materialized role container", container.Name), - )) - } - } - } - return allErrs -} - -// validateLPXRolePlacement protects controller-owned addressing and scheduling. -// Required node affinity remains a supported input to LPX Agent placement. -func validateLPXRolePlacement(spec *corev1.PodSpec, fldPath *field.Path, agent bool) field.ErrorList { - allErrs := field.ErrorList{} - if spec.SchedulerName != "" && spec.SchedulerName != corev1.DefaultSchedulerName { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulerName"), "LPX owns role scheduler selection")) - } - for _, entry := range []struct{ name, value string }{ - {"hostname", spec.Hostname}, {"subdomain", spec.Subdomain}, {"nodeName", spec.NodeName}, - } { - if entry.value != "" { - allErrs = append(allErrs, field.Forbidden(fldPath.Child(entry.name), "LPX owns role addressing and placement")) - } - } - if len(spec.TopologySpreadConstraints) != 0 { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("topologySpreadConstraints"), "LPX owns role placement")) - } - if !agent { - return allErrs - } - if len(spec.NodeSelector) != 0 { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("nodeSelector"), "LPX exclusively owns Agent node selection")) - } - if a := spec.Affinity; a != nil && - ((a.NodeAffinity != nil && len(a.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0) || - (a.PodAffinity != nil && (len(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0)) || - (a.PodAntiAffinity != nil && (len(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0))) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("affinity"), "node-local LPX supports only required nodeAffinity")) - } - if len(spec.SchedulingGates) != 0 { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulingGates"), "Grove and LPX own Agent scheduling gates")) - } - if len(spec.ResourceClaims) != 0 { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("resourceClaims"), "node-local LPX Agents cannot use ResourceClaims")) - } - return allErrs -} - -// validateSelectedRuntimeContainer validates a present template at its authored role path. -func validateSelectedRuntimeContainer( - template *corev1.PodTemplateSpec, - role string, - rolePath *field.Path, -) field.ErrorList { - containersPath := rolePath.Child("podTemplate", "spec", "containers") - for index := range template.Spec.Containers { - container := &template.Spec.Containers[index] - if container.Name != commonconsts.MainContainerName { - continue - } - return nil - } - return field.ErrorList{ - field.Required( - containersPath, - fmt.Sprintf("%s component requires a %q runtime container", role, commonconsts.MainContainerName), - ), - } -} diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index eb9dec585e21..42146c7fa849 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -28,7 +28,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 + name: dgd-lpx-v2-vllm-94cd-decode- namespace: inference-engine --- data: @@ -166,7 +166,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 + name: dgd-lpx-v2-vllm-94cd-lpu- namespace: inference-engine --- metadata: @@ -188,7 +188,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: e586b76cc8ccd9a1ed2e1b0297032eb1da420881dcfdcb632a4f00f4971308ff + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 @@ -392,14 +392,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 + name: dgd-lpx-v2-vllm-94cd-lpu- name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -571,7 +571,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 + name: dgd-lpx-v2-vllm-94cd-decode- name: config replicas: 1 roleName: cond @@ -590,7 +590,7 @@ spec: - cond minAvailable: 1 name: lpx - replicas: 2 + replicas: 1 terminationDelay: 15m0s status: availableReplicas: 0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index af2053019458..88da944b83a4 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -11,7 +11,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f + name: node-local-v2-hybrid-9d06-decode- namespace: test --- data: @@ -30,7 +30,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 + name: node-local-v2-hybrid-9d06-lpu- namespace: test --- metadata: @@ -52,7 +52,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 @@ -207,14 +207,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 + name: node-local-v2-hybrid-9d06-lpu- name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -360,7 +360,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f + name: node-local-v2-hybrid-9d06-decode- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 399e0cf48437..0efcf8a5352c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -25,7 +25,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 + name: node-local-v2-lpu-only-33b1-lpu- namespace: test --- metadata: @@ -47,7 +47,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc @@ -176,14 +176,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 + name: node-local-v2-lpu-only-33b1-lpu- name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor @@ -348,7 +348,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 + name: node-local-v2-lpu-only-33b1-lpu- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 5c264153ac1d..f90a5b94f840 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -59,7 +59,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 + name: node-local-v2-specdecode-63df-lpu- namespace: test --- metadata: @@ -81,7 +81,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b0943511c591eb7201003374eacf778c4545a959d7be251547399296e35ee1ae @@ -214,14 +214,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 + name: node-local-v2-specdecode-63df-lpu- name: config replicas: 4 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d19c12c440463a3ebbc8c86f3fe882a33e7d5f48de493146aaf6d0acfa4fdba7 @@ -354,14 +354,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 + name: node-local-v2-specdecode-63df-lpu- name: config replicas: 4 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b06e7ae90a96bd569bf228eb8b1d4b1192c4b0df3af7e33202d1fc1965a5247e @@ -494,14 +494,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 + name: node-local-v2-specdecode-63df-lpu- name: config replicas: 4 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 scheduling.lpu.nvidia.com/role: conductor @@ -673,7 +673,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 + name: node-local-v2-specdecode-63df-lpu- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index b065484cd8b8..e3fe7b5fc710 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -17,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd + name: node-local-v3-hx-hybrid-be18-lpu- namespace: test --- metadata: @@ -39,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 @@ -194,7 +194,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd + name: node-local-v3-hx-hybrid-be18-lpu- name: config replicas: 1 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 339a323cad30..57a214ecbe6e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -17,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd + name: node-local-v3-hx-lpu-only-d43c-lpu- namespace: test --- metadata: @@ -39,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 @@ -182,14 +182,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd + name: node-local-v3-hx-lpu-only-d43c-lpu- name: config replicas: 1 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/role: conductor @@ -373,7 +373,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd + name: node-local-v3-hx-lpu-only-d43c-lpu- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index c63f47f48b24..0a5e46e8c048 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -38,7 +38,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 + name: node-local-v3-hx-specdecode-1f7a-lpu- namespace: test --- metadata: @@ -60,7 +60,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6b42344f7916f4778ffdef73249a7395bbc7a082466cc457953af5de5544c80f @@ -207,14 +207,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 + name: node-local-v3-hx-specdecode-1f7a-lpu- name: config replicas: 1 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b94a35c4a019e3213f4497e105522208d2a2707e8469ec13c22d46b99d53c227 @@ -361,14 +361,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 + name: node-local-v3-hx-specdecode-1f7a-lpu- name: config replicas: 1 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8440780b5cdc30ae4bbcf80fda3cce06027325884a8b69751577112153ab2cf5 @@ -515,14 +515,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 + name: node-local-v3-hx-specdecode-1f7a-lpu- name: config replicas: 1 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 scheduling.lpu.nvidia.com/role: conductor @@ -714,7 +714,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 + name: node-local-v3-hx-specdecode-1f7a-lpu- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 650b368ae969..0642a6b31ddd 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -100,7 +100,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 + name: dgd-single-v2-3a98-lpu- namespace: inference-engine --- metadata: @@ -122,7 +122,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 @@ -255,14 +255,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 + name: dgd-single-v2-3a98-lpu- name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 + nvidia.com/extra-resources-hash: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/role: conductor @@ -446,7 +446,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 + name: dgd-single-v2-3a98-lpu- name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index a7e113621b0b..ef64a3805a1c 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -218,10 +218,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( } allErrs = append(allErrs, v.validateDynamoGraphDeploymentSpec(&dgd.Spec, field.NewPath("spec"), specOpts)...) - // Reserve the Agent name before build-dependent conductor validation runs in the LPX controller. - allErrs = append(allErrs, dynamolpx.ValidateAgentContainerNames(dgd)...) - - // LPX children require the durable Grove route before runtime validation can be deferred. + // LPX children require the Grove route for materialization. if dgd.HasLPXComponent() && !grovePathway { allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "components"), grovePathwayRequirement)) } @@ -305,8 +302,18 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( allErrs := field.ErrorList{} + // LPX cardinality determines the root overrides and shared draft/target contract. + lpxComponentCount := 0 + for i := range spec.Components { + if spec.Components[i].IsLPX() { + lpxComponentCount++ + } + } + // Validate the root provider fragment against the selected graph workload program. - if spec.ProviderOverride != nil { + if spec.ProviderOverride != nil && lpxComponentCount > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("providerOverride"), "LPX does not support Grove topology overrides on the deployment")) + } else if spec.ProviderOverride != nil { allErrs = append(allErrs, v.validateProviderOverride( spec.ProviderOverride, fldPath.Child("providerOverride"), @@ -326,12 +333,32 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( if len(spec.Components) == 0 { allErrs = append(allErrs, field.Required(componentsPath, "must have at least one component")) } + if lpxComponentCount > 2 { + allErrs = append(allErrs, field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")) + } components := componentsByName(spec.Components) - hasLPXComponent := false + hasLPXComponent := lpxComponentCount > 0 for i := range spec.Components { component := &spec.Components[i] componentPath := componentsPath.Index(i) - hasLPXComponent = hasLPXComponent || component.IsLPX() + + // Draft fanout expands models; the target owns the shared endpoint and availability. + if lpxComponentCount == 2 && component.IsLPX() { + replicas := k8sptr.Deref(component.Replicas, 1) + if component.ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) == nil { + if replicas > dynamolpx.MaxSpecDecodeNumDrafts { + allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "draft replicas must be between 1 and 8")) + } + if component.ModelRef != nil { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("modelRef"), "the shared target owns the serving endpoint")) + } + if k8sptr.Deref(component.MinAvailable, 1) != 1 { + allErrs = append(allErrs, field.Forbidden(componentPath.Child("minAvailable"), "draft minAvailable must be omitted or 1; the shared target owns minimum availability")) + } + } else if replicas != 1 { + allErrs = append(allErrs, field.Invalid(componentPath.Child("replicas"), replicas, "shared target replicas must be one")) + } + } // Externally managed components validate their generated names in their own controller. if opts.grovePathway && !component.ManagedByExternalController() { @@ -404,13 +431,34 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( return conductorComponentsPath.Key(spec.Components[index].ComponentName) } } - allErrs = append(allErrs, dynamolpx.ValidateConductorRoles(spec, conductorComponentsPath, componentPath)...) + conductorCount := 0 + for index := range spec.Components { + component := &spec.Components[index] + if !component.IsLPX() { + continue + } + for roleIndex, role := range component.Roles { + if role.Name != nvidiacomv1beta1.ComponentRoleLPXConductor { + continue + } + conductorCount++ + if role.PodTemplate == nil { + allErrs = append(allErrs, field.Required(componentPath(index).Child("roles").Index(roleIndex).Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) + } + } + } + if hasLPXComponent && conductorCount != 1 { + allErrs = append(allErrs, field.Forbidden(conductorComponentsPath, "LPX components must declare exactly one conductor role")) + } if spec.Restart != nil { allErrs = append(allErrs, v.validateRestart(spec.Restart, fldPath.Child("restart"), components)...) } constraintPath := fldPath.Child("topologyConstraint") + if hasLPXComponent && spec.TopologyConstraint != nil { + allErrs = append(allErrs, field.Forbidden(constraintPath, "LPX does not support Grove topologyConstraint")) + } hasComponentConstraint := false for i := range spec.Components { if spec.Components[i].TopologyConstraint != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index aa9570351910..d35be8b82e82 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -127,7 +127,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{`spec.experimental.kvTransferPolicy: Forbidden: is not supported when an LPX component is selected`}, }, { - name: "LPX topology and checkpoint support is deferred to the child controller", + name: "LPX rejects topology and checkpoint configuration", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.TopologyConstraint = &nvidiacomv1beta1.SpecTopologyConstraint{ ClusterTopologyName: "grove-topology", PackDomain: "rack", @@ -138,6 +138,11 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true}, } }), + wantWebhookErrs: []string{ + "spec.components[0].topologyConstraint: Forbidden: LPX does not support Grove topologyConstraint", + "spec.components[0].experimental.checkpoint: Forbidden: checkpoint functionality is supported only for worker, prefill, and decode components", + "spec.topologyConstraint: Forbidden: LPX does not support Grove topologyConstraint", + }, }, { name: "LPX admits an ordinary component checkpoint", @@ -298,7 +303,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantSchemaErr: "spec.components[0].lpx.buildId: Invalid value: \"\": spec.components[0].lpx.buildId in body should be at least 1 chars long", }, { - name: "v1alpha1 LPX root template remains rejected when runtime validation is deferred", + name: "v1alpha1 LPX rejects root templates and whitespace build IDs together", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { component := dgd.Spec.Services["lpx"] component.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "runtime"}} @@ -306,13 +311,15 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), wantWebhookErrs: []string{ "spec.components[0].podTemplate: Forbidden: LPX Pod templates belong to roles", + "spec.components[0].lpx.buildId: Required value: LPX component requires a buildId", }, }, { - name: "v1alpha1 LPX whitespace build ID validation is deferred to the child controller", + name: "v1alpha1 LPX rejects whitespace build IDs", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].LPX.BuildID = " " }), + wantWebhookErrs: []string{"spec.components[0].lpx.buildId: Required value: LPX component requires a buildId"}, }, // Conductor startup is explicit on both source API versions and on every update. { @@ -415,6 +422,121 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, // Selected LPX workload shapes. + { + name: "LPX rejects multinode configuration", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Multinode = &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2} + }), + wantWebhookErrs: []string{"spec.components[0].multinode: Forbidden: multinode is supported only for worker, prefill, or decode components"}, + }, + { + name: "LPX does not grandfather invalid multinode configuration", + seedWithoutWebhook: true, + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Multinode = &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2} + }), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Multinode = &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2} + dgd.Labels = map[string]string{"updated": "true"} + }), + wantWebhookErrs: []string{"spec.components[0].multinode: Forbidden: multinode is supported only for worker, prefill, or decode components"}, + }, + { + name: "LPX rejects a whitespace-only build reference", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].LPX.BuildID = " " + }), + wantWebhookErrs: []string{"spec.components[0].lpx.buildId: Required value: LPX component requires a buildId"}, + }, + { + name: "v1alpha1 LPX rejects a whitespace-only build reference on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].LPX.BuildID = " " + }), + wantWebhookErrs: []string{"spec.components[0].lpx.buildId: Required value: LPX component requires a buildId"}, + }, + { + name: "LPX requires main containers in both roles", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + for index := range dgd.Spec.Components[0].Roles { + dgd.Spec.Components[0].Roles[index].PodTemplate.Spec.Containers[0].Name = "sidecar" + } + }), + wantWebhookErrs: []string{ + `spec.components[0].roles[0].podTemplate.spec.containers: Required value: LPX agent component requires a "main" runtime container`, + `spec.components[0].roles[1].podTemplate.spec.containers: Required value: LPX conductor component requires a "main" runtime container`, + }, + }, + { + name: "LPX rejects conductor placement on UPDATE", + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + spec := &dgd.Spec.Components[0].Roles[1].PodTemplate.Spec + spec.NodeName = "chosen-node" + spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule}} + }), + wantWebhookErrs: []string{ + "spec.components[0].roles[1].podTemplate.spec.nodeName: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[1].podTemplate.spec.topologySpreadConstraints: Forbidden: LPX owns role placement", + }, + }, + { + name: "LPX rejects three model components with one conductor", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, nil) + extra := dgd.Spec.Components[0].DeepCopy() + extra.ComponentName = "extra-draft" + dgd.Spec.Components = append(dgd.Spec.Components, *extra) + }), + wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair"}, + }, + { + name: "LPX rejects zero draft replicas", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(0))) + }), + wantCELErr: "spec.components[0]: Invalid value: replicas must be positive when type is lpx", + }, + { + name: "LPX admits maximum draft fanout with default minimum availability", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(8))) + dgd.Spec.Components[0].MinAvailable = k8sptr.To(int32(1)) + }), + }, + { + name: "LPX rejects draft endpoint and minimum availability at authored indices", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, k8sptr.To(int32(3))) + dgd.Spec.Components[0], dgd.Spec.Components[1] = dgd.Spec.Components[1], dgd.Spec.Components[0] + draft := &dgd.Spec.Components[1] + draft.MinAvailable = k8sptr.To(int32(2)) + draft.ModelRef = &nvidiacomv1beta1.ModelReference{Name: "draft-model"} + }), + wantWebhookErrs: []string{ + "spec.components[1].modelRef: Forbidden: the shared target owns the serving endpoint", + "spec.components[1].minAvailable: Forbidden: draft minAvailable must be omitted or 1; the shared target owns minimum availability", + }, + }, + { + name: "LPX admits singleton minimum availability", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Replicas = k8sptr.To(int32(3)) + dgd.Spec.Components[0].MinAvailable = k8sptr.To(int32(2)) + }), + }, + { + name: "v1alpha1 LPX rejects shared target scaling on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, nil) + }), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + setAlphaLPXSpecDec(dgd, nil) + dgd.Spec.Services["target"].Replicas = k8sptr.To(int32(2)) + }), + wantWebhookErrs: []string{"spec.components[1].replicas: Invalid value: 2: shared target replicas must be one"}, + }, { name: "LPX roles reject provider overrides", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { @@ -431,19 +553,27 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{conductorRoleErr}, }, { - name: "LPX provider override support on update is deferred to the child controller", + name: "LPX rejects provider overrides on UPDATE", oldDeployment: betaLPXDGDForAdmission(nil), deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.ProviderOverride = groveProviderOverride("", `{"spec":{"template":{"topologyConstraint":{"pack":{"required":"rack"}}}}}`) dgd.Spec.Components[0].ProviderOverride = groveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) }), + wantWebhookErrs: []string{ + "spec.providerOverride: Forbidden: LPX does not support Grove topology overrides on the deployment", + "spec.components[0].providerOverride: Forbidden: LPX component does not support Grove topology overrides", + }, }, { - name: "v1alpha1 LPX provider override support is deferred after conversion", + name: "v1alpha1 LPX rejects provider overrides after conversion", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.ProviderOverride = alphaGroveProviderOverride("", `{"spec":{"template":{"topologyConstraint":{"pack":{"required":"rack"}}}}}`) dgd.Spec.Services["lpx"].ProviderOverride = alphaGroveProviderOverride("", `{"topologyConstraint":{"pack":{"required":"rack"}}}`) }), + wantWebhookErrs: []string{ + "spec.providerOverride: Forbidden: LPX does not support Grove topology overrides on the deployment", + "spec.components[0].providerOverride: Forbidden: LPX component does not support Grove topology overrides", + }, }, { name: "LPX admits an ordinary component provider override", @@ -475,7 +605,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "LPX Agent placement validation is deferred to the child controller", + name: "LPX rejects controller-owned Agent placement", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { setBetaLPXSpecDec(dgd, nil) spec := &dgd.Spec.Components[0].Roles[0].PodTemplate.Spec @@ -495,12 +625,26 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: "custom"}} spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "device", ResourceClaimName: k8sptr.To("device-claim")}} }), + wantWebhookErrs: []string{ + "spec.components[0].roles[0].podTemplate.spec.schedulerName: Forbidden: LPX owns role scheduler selection", + "spec.components[0].roles[0].podTemplate.spec.hostname: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[0].podTemplate.spec.subdomain: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[0].podTemplate.spec.nodeName: Forbidden: LPX owns role addressing and placement", + "spec.components[0].roles[0].podTemplate.spec.topologySpreadConstraints: Forbidden: LPX owns role placement", + "spec.components[0].roles[0].podTemplate.spec.nodeSelector: Forbidden: LPX exclusively owns Agent node selection", + "spec.components[0].roles[0].podTemplate.spec.affinity: Forbidden: node-local LPX supports only required nodeAffinity", + "spec.components[0].roles[0].podTemplate.spec.schedulingGates: Forbidden: Grove and LPX own Agent scheduling gates", + "spec.components[0].roles[0].podTemplate.spec.resourceClaims: Forbidden: node-local LPX Agents cannot use ResourceClaims", + }, }, { - name: "LPX SpecDecode draft replica support is deferred to the child controller", + name: "LPX rejects excessive draft fanout", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { setBetaLPXSpecDec(dgd, k8sptr.To(int32(9))) }), + wantWebhookErrs: []string{ + "spec.components[0].replicas: Invalid value: 9: draft replicas must be between 1 and 8", + }, }, { name: "lpx requires a agent role", @@ -554,7 +698,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "LPX PCSG name overflow validation is deferred to the child controller", + name: "LPX admits long component names independently of generated PCSG names", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].ComponentName = longLPXComponentName }), diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index e85d699b67d7..8e7003548e87 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -40,27 +40,85 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -// validateLPXPodTemplateSpec checks image and annotation semantics beyond the Pod schema. -// LPX intent validation additionally checks main-container and placement rules. +// validatePodTemplateSpec checks LPX role templates beyond the admitted Pod schema. // template and fldPath are non-nil. -func (v *sharedValidation) validateLPXPodTemplateSpec(template *corev1.PodTemplateSpec, fldPath *field.Path) field.ErrorList { +func (v *sharedValidation) validatePodTemplateSpec(template *corev1.PodTemplateSpec, fldPath *field.Path, role string) field.ErrorList { allErrs := apivalidation.ValidateAnnotations(template.Annotations, fldPath.Child("metadata", "annotations")) + return append(allErrs, v.validatePodSpec(&template.Spec, fldPath.Child("spec"), role)...) +} + +// validatePodSpec protects LPX-owned container names, addressing, and placement. +// spec and fldPath are non-nil; Kubernetes schema validation runs before this method. +func (v *sharedValidation) validatePodSpec(spec *corev1.PodSpec, fldPath *field.Path, role string) field.ErrorList { + var allErrs field.ErrorList + agent := role == nvidiacomv1beta1.ComponentRoleLPXAgent + if !hasContainerNamed(spec.Containers, consts.MainContainerName) { + allErrs = append(allErrs, field.Required(fldPath.Child("containers"), fmt.Sprintf("LPX %s component requires a %q runtime container", role, consts.MainContainerName))) + } + + // Agent names are build-independent; conductor names depend on the compiled execution mode. for _, group := range []struct { name string containers []corev1.Container }{ - {"containers", template.Spec.Containers}, - {"initContainers", template.Spec.InitContainers}, + {"containers", spec.Containers}, + {"initContainers", spec.InitContainers}, } { for index, container := range group.containers { if strings.TrimSpace(container.Image) == "" { - allErrs = append(allErrs, field.Required(fldPath.Child("spec", group.name).Index(index).Child("image"), "must specify a non-empty image")) + allErrs = append(allErrs, field.Required(fldPath.Child(group.name).Index(index).Child("image"), "must specify a non-empty image")) + } + if agent && container.Name == nvidiacomv1beta1.ComponentRoleLPXAgent { + allErrs = append(allErrs, field.Forbidden(fldPath.Child(group.name).Index(index).Child("name"), fmt.Sprintf("LPX reserves %q for the materialized role container", container.Name))) } } } + + // Every LPX role uses controller-owned addressing and scheduler selection. + if spec.SchedulerName != "" && spec.SchedulerName != corev1.DefaultSchedulerName { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulerName"), "LPX owns role scheduler selection")) + } + for _, entry := range []struct{ name, value string }{ + {"hostname", spec.Hostname}, {"subdomain", spec.Subdomain}, {"nodeName", spec.NodeName}, + } { + if entry.value != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child(entry.name), "LPX owns role addressing and placement")) + } + } + if len(spec.TopologySpreadConstraints) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("topologySpreadConstraints"), "LPX owns role placement")) + } + if !agent { + return allErrs + } + + // Required node affinity is the only authored Agent placement constraint. + if len(spec.NodeSelector) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("nodeSelector"), "LPX exclusively owns Agent node selection")) + } + if a := spec.Affinity; a != nil && + ((a.NodeAffinity != nil && len(a.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0) || + (a.PodAffinity != nil && (len(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0)) || + (a.PodAntiAffinity != nil && (len(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 || len(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0))) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("affinity"), "node-local LPX supports only required nodeAffinity")) + } + if len(spec.SchedulingGates) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("schedulingGates"), "Grove and LPX own Agent scheduling gates")) + } + if len(spec.ResourceClaims) != 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("resourceClaims"), "node-local LPX Agents cannot use ResourceClaims")) + } return allErrs } +// validateLPXConfig validates the build reference. config and fldPath must be non-nil. +func (v *sharedValidation) validateLPXConfig(config *nvidiacomv1beta1.LPXConfig, fldPath *field.Path) field.ErrorList { + if strings.TrimSpace(config.BuildID) == "" { + return field.ErrorList{field.Required(fldPath.Child("buildId"), "LPX component requires a buildId")} + } + return nil +} + // sharedValidation carries request-wide dependencies and accumulation used by // validation for API types shared by multiple resources. type sharedValidation struct { @@ -100,7 +158,9 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( allErrs := field.ErrorList{} // Validate the provider-native fragment in this component context. - if spec.ProviderOverride != nil { + if spec.ProviderOverride != nil && spec.IsLPX() { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("providerOverride"), "LPX component does not support Grove topology overrides")) + } else if spec.ProviderOverride != nil { allErrs = append(allErrs, v.validateProviderOverride( spec.ProviderOverride, fldPath.Child("providerOverride"), @@ -117,6 +177,12 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( if spec.IsLPX() && spec.PodTemplate != nil { allErrs = append(allErrs, field.Forbidden(fldPath.Child("podTemplate"), "LPX Pod templates belong to roles")) } + if spec.IsLPX() && spec.TopologyConstraint != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("topologyConstraint"), "LPX does not support Grove topologyConstraint")) + } + if spec.LPX != nil { + allErrs = append(allErrs, v.validateLPXConfig(spec.LPX, fldPath.Child("lpx"))...) + } // Enforce Grove-only availability semantics before validating later fields. if spec.MinAvailable != nil && !options.grovePathway { @@ -237,7 +303,7 @@ func hasUnsupportedMultinode(spec *nvidiacomv1beta1.DynamoComponentDeploymentSha } // validateMultinodeComponentType rejects unsupported new combinations and -// ratchets identical legacy violations on update. fldPath points to multinode. +// ratchets identical non-LPX legacy violations on update. fldPath points to multinode. func validateMultinodeComponentType( newSpec *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, oldSpec *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, @@ -246,7 +312,7 @@ func validateMultinodeComponentType( if !hasUnsupportedMultinode(newSpec) { return nil } - if oldSpec != nil && oldSpec.ComponentType == newSpec.ComponentType && + if !newSpec.IsLPX() && oldSpec != nil && oldSpec.ComponentType == newSpec.ComponentType && hasUnsupportedMultinode(oldSpec) && apiequality.Semantic.DeepEqual(oldSpec.Multinode, newSpec.Multinode) { return nil @@ -465,7 +531,7 @@ func (v *sharedValidation) validateComponentRoleSpec( // LPX renders role templates directly and has no provider-override lowering. if options.component != nil && options.component.IsLPX() { if role.PodTemplate != nil { - allErrs = append(allErrs, v.validateLPXPodTemplateSpec(role.PodTemplate, fldPath.Child("podTemplate"))...) + allErrs = append(allErrs, v.validatePodTemplateSpec(role.PodTemplate, fldPath.Child("podTemplate"), role.Name)...) } else if role.Name == nvidiacomv1beta1.ComponentRoleLPXAgent { allErrs = append(allErrs, field.Required(fldPath.Child("podTemplate"), "the LPX agent role requires a podTemplate")) } @@ -745,7 +811,6 @@ func (v *sharedValidation) validateComponentCheckpointConfig( "checkpoint functionality requires component type to be explicitly set to worker, prefill, or decode", )) } else if checkpointConfig.Enabled && - componentType != nvidiacomv1beta1.ComponentTypeLPX && !dynamo.IsWorkerComponent(string(componentType)) { allErrs = append(allErrs, field.Forbidden( fldPath, diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 799b269ba98a..5804c83ef3a7 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -76,6 +76,8 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Desired number of complete component instances. For a single-node component, each instance is one pod. For a multinode component, each instance contains all of its roles. Minimum `0`. When `scalingAdapter` is set, this field is owned by the [DynamoGraphDeploymentScalingAdapter](full-api-reference.mdx#dynamographdeploymentscalingadapter) and should not be modified directly. For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid and LPU-only engines allow up to `2496` replicas; generated Pod hostname limits may lower that maximum. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. + + When LPX serving-component `replicas` is omitted, the controller follows the live Grove `PodCliqueScalingGroup` (PCSG) replica count without writing its Scale subresource. This externally managed count is lost when the `PodCliqueSet` (PCS) or PCSG is replaced, including PCS replacement for immutable workload or compiler-provenance changes. The new group starts from the template's `minAvailable` seed (default `1`); the external scaler must reapply its desired capacity. Explicit DGD `replicas` are reapplied by the operator after replacement. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 943c27601787..0e7c728034e2 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -77,6 +77,8 @@ spec: Labels propagated to all child resources. Same precedence rules as `annotations`. + + For LPX, metadata synchronization keeps the PCS labels and clique templates consistent after queue edits. Moving an existing KAI PodGroup to another queue depends on the Grove backend; metadata synchronization does not recreate existing PodGroups. @@ -89,8 +91,13 @@ spec: - Wall-clock limit for each pending LPU pipeline request, measured from the scheduler-published start of its current scheduling cycle until it reports `Bound`, `NoFit`, or `Unsupported`. The deadline is not armed until the scheduler publishes `status.schedulingStartedAt`, and a later scheduling cycle resets that timestamp. Valid values are `1` through `9,223,372,036`; omission means unlimited. - Expiry reports `LPXSchedulingDeadlineExceeded` and deletes the expired request. When `replicas` is explicit, it also lowers the Grove replica count to the surviving prefix; externally managed capacity is left unchanged. A later LPX input revision authorizes another publication. This field is not a solver budget. + Best-effort deadline for each LPU pipeline request's scheduling cycle. The timer starts at the scheduler's `status.schedulingStartedAt`; without that timestamp, no deadline is running. A later scheduling cycle supplies a new timestamp. Valid values are `1` through `9,223,372,036`; omission means unlimited. This field is not a solver budget or an overall deployment readiness timeout. + + Current-generation `Bound`, `NoFit`, and `Unsupported` receipts are exempt. `Degraded`, `Releasing`, and `Released` receipts are also exempt when their committed execution was accepted for the current request generation. `NoFit` can therefore leave the deployment Pending indefinitely, and a `Bound` request can still wait indefinitely for runtime readiness. + + Deadline handling runs after download checks, compiler-registry resolution, and desired-state validation. An outage or invalid desired configuration can delay failure reporting and cleanup indefinitely, even after the deadline has elapsed. The timer does not bound asynchronous cleanup or finalization. + + Expiry records `LPXSchedulingDeadlineExceeded` before cleanup. Cleanup removes complete engine replicas only from a trailing suffix: an expired interior replica blocks deadline cleanup to preserve healthy higher ordinals. Explicit `replicas` authorizes lowering the Grove replica count to the surviving prefix; externally managed capacity remains unchanged. Grove scaling and owner garbage collection handle pod cleanup; the LPX graph controller does not delete pods directly. A later LPX input revision authorizes republication after cleanup. An existing failure remains cleanup authority for cycles that started before it, so retiring those cycles does not consume the retry edit. From 274172b20ad12cb24efb7bb0f6a3057c1b94a1a0 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Tue, 22 Sep 2026 01:28:26 -0700 Subject: [PATCH 111/196] feat: support multiple LPX components Signed-off-by: Steven Davidovitz --- deploy/operator/api/AGENTS.md | 5 - .../dynamocomponentdeployment_types.go | 7 +- .../dynamocomponentdeployment_types.go | 7 +- ...nvidia.com_dynamocomponentdeployments.yaml | 14 +- .../nvidia.com_dynamographdeployments.yaml | 14 +- .../controller/dgd_grove_lpx_restart_test.go | 3 +- .../controller/dgd_grove_lpx_status.go | 3 +- .../dgd_lpx_handoff_envtest_test.go | 2 +- .../controller/dgd_lpx_handoff_test.go | 2 +- .../internal/controller/lpx/AGENTS.md | 66 +- .../internal/controller/lpx/controller.go | 314 +++--- .../controller/lpx/controller_test.go | 967 ++++++++++-------- .../internal/controller/lpx/eviction.go | 58 +- .../internal/controller/lpx/eviction_test.go | 188 ++-- .../controller/lpx/pipeline_request.go | 26 +- .../lpx/pipeline_request_deadline.go | 96 +- .../lpx/pipeline_request_deadline_test.go | 61 +- .../controller/lpx/pipeline_request_test.go | 26 +- .../internal/controller/lpx/render.go | 160 ++- .../internal/controller/lpx/render_test.go | 549 +++++----- .../internal/controller/lpx/status.go | 18 +- .../internal/controller/lpx/status_test.go | 25 +- .../operator/internal/controller/lpx/util.go | 90 +- .../internal/controller/lpx/util_test.go | 203 +++- deploy/operator/internal/dynamo/graph_lpx.go | 112 +- .../internal/dynamo/graph_lpx_test.go | 44 +- deploy/operator/internal/dynamo/grove_lpx.go | 44 +- .../internal/dynamo/grove_lpx_test.go | 24 +- .../internal/dynamo/lpx/components.go | 58 ++ .../internal/dynamo/lpx/components_test.go | 113 ++ .../internal/dynamo/lpx/cyborg_render_test.go | 12 +- .../dynamo/lpx/hybrid_cyborg_bindings.go | 13 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 14 +- .../internal/dynamo/lpx/lpu_configmap.go | 20 +- .../internal/dynamo/lpx/lpu_configmap_test.go | 8 +- .../internal/dynamo/lpx/materialization.go | 91 +- .../dynamo/lpx/materialization_test.go | 86 +- .../internal/dynamo/lpx/model_projection.go | 3 - deploy/operator/internal/dynamo/lpx/render.go | 69 +- .../internal/dynamo/lpx/render_test.go | 32 +- ...lected_workload.go => resolve_workload.go} | 77 +- ...kload_test.go => resolve_workload_test.go} | 98 +- .../internal/dynamo/lpx/selected_workload.go | 41 - .../operator/internal/dynamo/lpx/selection.go | 76 -- .../internal/dynamo/lpx/selection_test.go | 72 -- .../lpu-cyborg-specdec.input.yaml | 93 ++ .../from_dgd_yaml/lpu-cyborg-specdec.yaml | 482 +++++++++ .../lpu-gpu-specdecode.input.yaml | 63 ++ .../from_dgd_yaml/lpu-gpu-specdecode.yaml | 362 +++++++ .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 14 +- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 14 +- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 12 +- .../node-local-v2-specdecode.yaml | 20 +- .../node-local-v3-hx-hybrid.yaml | 8 +- .../node-local-v3-hx-lpu-only.yaml | 12 +- .../node-local-v3-hx-specdecode.yaml | 20 +- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 12 +- .../operator/internal/dynamo/lpx/workload.go | 53 + .../operator/internal/podcache/transform.go | 17 +- .../internal/podcache/transform_test.go | 18 + .../validation/dynamographdeployment.go | 13 +- ...hdeployment_lpx_validation_envtest_test.go | 39 +- .../additional-resources/api-reference-k8s.md | 8 +- .../dynamo-component-deployment.mdx | 2 - .../dynamo-graph-deployment.mdx | 2 +- .../kubernetes-api/full-api-reference.mdx | 8 +- 66 files changed, 3482 insertions(+), 1801 deletions(-) create mode 100644 deploy/operator/internal/dynamo/lpx/components.go create mode 100644 deploy/operator/internal/dynamo/lpx/components_test.go rename deploy/operator/internal/dynamo/lpx/{resolve_selected_workload.go => resolve_workload.go} (75%) rename deploy/operator/internal/dynamo/lpx/{resolve_selected_workload_test.go => resolve_workload_test.go} (78%) delete mode 100644 deploy/operator/internal/dynamo/lpx/selected_workload.go delete mode 100644 deploy/operator/internal/dynamo/lpx/selection.go delete mode 100644 deploy/operator/internal/dynamo/lpx/selection_test.go create mode 100644 deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml create mode 100644 deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml create mode 100644 deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml create mode 100644 deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml create mode 100644 deploy/operator/internal/dynamo/lpx/workload.go diff --git a/deploy/operator/api/AGENTS.md b/deploy/operator/api/AGENTS.md index 4553346f3708..4916d0c481a7 100644 --- a/deploy/operator/api/AGENTS.md +++ b/deploy/operator/api/AGENTS.md @@ -8,8 +8,3 @@ code and conversion tests, or explicitly document why conversion is unaffected. Follow its invariants for hub/spoke conversion, sparse annotation preservation, live-source precedence, structural helper naming, and fuzz coverage. - -Keep public API comments concise and focused on the field's contract, defaults, -and validation constraints. Put reconciliation mechanics and detailed rationale -in comments beside the implementation or in its scoped `AGENTS.md`, not in API -field descriptions. diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 7f4c38f0e5ea..b8ad5e4587c2 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -174,9 +174,10 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components each require an agent role. All LPX components in a DGD - // must declare exactly one conductor role in total. Every LPX role requires - // its own podTemplate. + // LPX components each require an agent role. A DGD may contain independent + // LPX components, each with its own conductor role, or a shared draft and + // target pair with a conductor role only on the target. Every LPX role + // requires its own podTemplate. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 3ed8f26b6a11..9e08f77f3321 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -178,9 +178,10 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - // LPX components each require an agent role. All LPX components in a DGD - // must declare exactly one conductor role in total. Every LPX role requires - // its own podTemplate. + // LPX components each require an agent role. A DGD may contain independent + // LPX components, each with its own conductor role, or a shared draft and + // target pair with a conductor role only on the target. Every LPX role + // requires its own podTemplate. // +optional // +listType=map // +listMapKey=name diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 9a1921c863c6..ca6f43856ca5 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -11465,9 +11465,10 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components each require an agent role. All LPX components in a DGD - must declare exactly one conductor role in total. Every LPX role requires - its own podTemplate. + LPX components each require an agent role. A DGD may contain independent + LPX components, each with its own conductor role, or a shared draft and + target pair with a conductor role only on the target. Every LPX role + requires its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -28655,9 +28656,10 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components each require an agent role. All LPX components in a DGD - must declare exactly one conductor role in total. Every LPX role requires - its own podTemplate. + LPX components each require an agent role. A DGD may contain independent + LPX components, each with its own conductor role, or a shared draft and + target pair with a conductor role only on the target. Every LPX role + requires its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 4432bc77ea7d..79008646b845 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11852,9 +11852,10 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components each require an agent role. All LPX components in a DGD - must declare exactly one conductor role in total. Every LPX role requires - its own podTemplate. + LPX components each require an agent role. A DGD may contain independent + LPX components, each with its own conductor role, or a shared draft and + target pair with a conductor role only on the target. Every LPX role + requires its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. @@ -29295,9 +29296,10 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. - LPX components each require an agent role. All LPX components in a DGD - must declare exactly one conductor role in total. Every LPX role requires - its own podTemplate. + LPX components each require an agent role. A DGD may contain independent + LPX components, each with its own conductor role, or a shared draft and + target pair with a conductor role only on the target. Every LPX role + requires its own podTemplate. items: description: |- ComponentRoleSpec configures one named Pod-producing role inside a compound component. diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go index 1a82d3037a0f..3fdad84bb518 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -9,7 +9,6 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,7 +37,7 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { require.Empty(t, resolver.Resolve(ctx, dgd, []string{"lpx"})) t.Log("An input edit before handoff must keep the restart in progress despite the old Ready receipt") - lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "runtime:changed" + dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "runtime:changed" dgd.Generation++ revision, err := dynamo.LPXInputRevision(dgd, "restart-1") require.NoError(t, err) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 9b38b5ffa4c3..2b27c27a4264 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -25,10 +25,9 @@ func mergeLPXChildStatus( if ordinary.ComponentStatus == nil { ordinary.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) } - serving := lpx.ServingComponent(source) for _, component := range components { kind := v1beta1.ComponentKindPodCliqueScalingGroup - if component != serving { + if component.ComponentRole(v1beta1.ComponentRoleLPXConductor) == nil { kind = v1beta1.ComponentKindPodClique } ordinary.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ComponentKind: kind} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 234a518efd44..1c751215149a 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -351,7 +351,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) t.Log("An LPX template edit advances the real child generation and invalidates the old observation") - dynamolpx.ServingComponent(source).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" + source.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" require.NoError(t, env.Client().Update(t.Context(), source)) updated, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index ec5c3f0ef642..63f6824c2fdf 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -45,7 +45,7 @@ func newLPXHandoffSource(t *testing.T, fixture string) *v1beta1.DynamoGraphDeplo require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(data, source)) - lpx.ServingComponent(source).ComponentName = "lpx" + source.GetComponentByName("lpu").ComponentName = "lpx" source.Generation = 3 return source } diff --git a/deploy/operator/internal/controller/lpx/AGENTS.md b/deploy/operator/internal/controller/lpx/AGENTS.md index 05b13310204f..f604a1ba5829 100644 --- a/deploy/operator/internal/controller/lpx/AGENTS.md +++ b/deploy/operator/internal/controller/lpx/AGENTS.md @@ -36,8 +36,8 @@ SPDX-License-Identifier: Apache-2.0 - The DynamoGraphDeployment (DGD) owns the LPXGraphDeployment (LPXGD). The LPXGD owns its PodCliqueSet (PCS), runtime ConfigMaps, and Services. The PCS owns the - LPUPipelineRequests (LPRs) and Grove hierarchy. One PodCliqueScalingGroup (PCSG) - under PCS ordinal zero holds all engine replicas. + LPUPipelineRequests (LPRs) and Grove hierarchy. Each independent workload has one + PodCliqueScalingGroup (PCSG). A shared draft/target runtime remains one workload. - LPRs have a controlling PCS owner reference with `blockOwnerDeletion: true`. Foreground PCS deletion keeps the owner present until blocking LPRs are gone. This is the supported lifecycle, not a guarantee against external orphaning. @@ -48,7 +48,8 @@ SPDX-License-Identifier: Apache-2.0 - Index LPRs by controlling PCS UID, validating the owner's API version and kind in the index. Do not add a PCS-name index or namespace-wide request scan. - PCS identity derives from the LPXGD UID and stays stable across input edits. - LPR names identify the LPXGD namespace, name, UID, model, and PCSG ordinal. + LPR names identify the LPXGD namespace, name, UID, workload, model, and PCSG + ordinal. - Keep identity and change detection separate: `InputRevision` synchronizes the parent-to-child handoff; workload digests detect immutable workload changes; `CompilerSnapshotDigestAnnotation` records compiler provenance and is part of @@ -80,16 +81,16 @@ SPDX-License-Identifier: Apache-2.0 - Retain matching LPRs and their status across input edits. Collect missing requests in the same ordered pass that constructs desired requests. An immutable mismatch invalidates that result and requires PCS replacement. -- Publish missing requests independently in PCSG ordinal/model order. Partial - publication and idempotent retries are supported; exactly-once batches are - not required. Creation order does not guarantee scheduler order or prevent +- Publish missing requests independently in workload/PCSG ordinal/model order. + Partial publication and idempotent retries are supported; exactly-once + batches are not required. Creation order does not guarantee scheduler order or prevent interior scheduling failures. - Wait for terminating requests to disappear before reusing their names. Do not introduce batch finalizers or cleanup protocols. # Capacity and Deletion -- Keep the top-level PCS replica count at one; engine capacity belongs to the +- Keep the top-level PCS replica count at one; workload capacity belongs to the PCSG. Correct top-level replica drift through normal synchronization, not PCS replacement. - Seed the PCS's scaling-group template from immutable `MinAvailable` (default @@ -102,15 +103,13 @@ SPDX-License-Identifier: Apache-2.0 - External capacity is not persisted across PCS or PCSG replacement. A new group starts from the immutable template seed; its external scaler must reapply the desired count. Explicit DGD replicas are reapplied by this controller. -- Validate the single-group template, group ownership, and deletion state at - observation. A foreign PCSG is an ownership error, not a cache miss. A nil PCSG - means the group is missing or the owned group is deleting; it does not mean - zero capacity or completed pod deletion. -- Initial PCS creation needs no PCSG. With explicit replicas and no existing - LPRs, PCS synchronization may also proceed before the group appears. With - explicit replicas, deadlines may be recorded while the group is unavailable. - Request deletion, publication, and readiness require an observed group; - externally managed capacity also requires it once the PCS exists. +- Index PCSGs by controlling PCS, validating the owner's API version and kind + in the index. Do not add a namespace-wide request scan. +- Initial PCS creation needs no PCSG. Once the PCS exists, `Reconcile` waits + until every configured group has an owned, non-deleting PCSG before proceeding. + The PCSG watch resumes reconciliation. Downstream capacity, request, deadline, + and readiness operations can rely on all configured groups being present. + This also defers PCS updates and replacement until all groups are observed. - Render a complete valid replacement before deleting an existing workload. A replacement error preserves the existing workload, even if its digest differs. - For immutable workload or request changes, foreground-delete the PCS and @@ -133,19 +132,20 @@ SPDX-License-Identifier: Apache-2.0 # Scheduling Cycles -- Scheduling deadlines apply to individual LPR cycles, not all deployment - Pending states, runtime readiness, or asynchronous cleanup. -- Start the deadline at `LPR.status.schedulingStartedAt`. Without that timestamp, - no deadline is running. Do not substitute creation time, managed fields, or a - deployment-level timestamp. The scheduler supplies a new start when a surviving - request begins another cycle, including Bound-to-Pending transitions. +- `attemptDeadlineSeconds` bounds how long an individual LPR remains pending, + including waiting for the scheduler to start. It is not a solver budget or a + deadline for all deployment Pending states, runtime readiness, or asynchronous cleanup. +- Use a nonzero `LPR.status.schedulingStartedAt` when present; otherwise use + `LPR.metadata.creationTimestamp`. The scheduler supplies a new start when a + surviving request begins another cycle, including Bound-to-Pending transitions; + that timestamp takes precedence over the request's original creation time. - Current-generation `Bound`, `NoFit`, and `Unsupported` states are exempt. `Degraded`, `Releasing`, and `Released` are exempt only with a current-generation status and a committed execution accepted for that generation. - Exemption is not readiness: `NoFit` remains pending, and `Bound` still requires Grove runtime readiness. -- The deadline is best effort: download checks, compiler-registry resolution and - desired-state validation precede expiry evaluation. Their failures can delay +- The deadline is best effort: PCSG observation, download checks, compiler-registry + resolution and desired-state validation precede expiry evaluation. Their failures can delay failure reporting and deadline cleanup indefinitely, including after expiry. # Scheduling Failure and Cleanup @@ -156,11 +156,13 @@ SPDX-License-Identifier: Apache-2.0 - Re-evaluate expiry from current request status on every pass. Do not delete a request solely because it expired in an earlier pass. The condition records failure, not a deletion queue; do not persist an expired-request UID list. -- One expired model request makes its entire engine replica eligible, including - sibling model requests. Cleanup may remove only a trailing suffix. - Any interior expired ordinal blocks all deadline scale-down and request - deletion, even if another expired suffix exists. For four replicas, expiry at - `{2,3}` permits scale to two; `{0}` or `{0,3}` permits no cleanup. +- Evaluate deadline cleanup independently for each workload. One expired model + request makes its entire workload replica eligible, including sibling model + requests. Cleanup may remove only a trailing suffix. + Any interior expired ordinal blocks that workload's deadline scale-down and + request deletion, even if another expired suffix exists in the same workload. + For four replicas, expiry at `{2,3}` permits scale to two; `{0}` or `{0,3}` + permits no cleanup. - Continue cleanup beyond an already-lowered group count on subsequent passes. Delete non-expired siblings before expired requests so a partial failure leaves expiry evidence for the next reconciliation. @@ -194,6 +196,9 @@ SPDX-License-Identifier: Apache-2.0 - `Ready.LastTransitionTime` changes only when its boolean status changes. Renewing `SchedulingFailed` deliberately gives it a new timestamp even when it remains true. +- Derive each component's readiness from its own workload's scheduler receipts and + Grove roles. The graph is Ready only when every workload is ready; aggregate + readiness must not overwrite independently observed component readiness. - Persist child status once at the reconciliation boundary, including early errors. Record errors before converting them into bounded deadline retries. Advance `ObservedGeneration` after error-free reconciliation, including pending @@ -202,9 +207,6 @@ SPDX-License-Identifier: Apache-2.0 acknowledgement and persisting status. Status-write failures remain errors. `reconcileWorkload` preserves its error and puts any bounded deadline retry in `ctrl.Result`; only the outer `Reconcile` converts that pair to a successful retry. -- Synchronize propagated PCS and serving Service labels and annotations through - `WithMetadataSync`. Remove previously tracked keys omitted from desired state; - preserve untracked metadata and the shared helper's bookkeeping annotations. - Check downloads before acquiring compiler metadata or deriving deadlines. Reuse fresh successful observations by build URL across revisions, without renewing `LastCheckedAt`. Check new or expired builds. Only a Ready deployment diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 32655331bfad..db2a78abd689 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -7,21 +7,19 @@ import ( "context" "errors" "fmt" + "maps" "slices" "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -118,6 +116,11 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } }(deployment.Status.DeepCopy()) + for name, component := range deployment.Status.Components { + component.Ready = false + deployment.Status.Components[name] = component + } + if !r.enabled { setReadyCondition(deployment, v1beta1.DGDStateFailed, integrationDisabledMessage(r.runtimeConfig)) return ctrl.Result{}, nil @@ -147,132 +150,147 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu } var ( - pcsg *grovev1alpha1.PodCliqueScalingGroup + pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup requests []lpxv1alpha1.LPUPipelineRequest ) if pcs != nil { - // Foreground PCS replacement waits for its blocking LPRs; discover them only by owner UID. - requests, err = r.getPipelineRequests(ctx, pcs) + // Wait for every configured group before reconciling an existing PCS. + pcsgs, err = getPodCliqueScalingGroups(ctx, r.Client, pcs) if err != nil { return ctrl.Result{}, err } - pcsg, err = getPodCliqueScalingGroup(ctx, r.Client, pcs) + if len(pcsgs) != len(pcs.Spec.Template.PodCliqueScalingGroupConfigs) { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for all LPX scaling groups") + return ctrl.Result{}, nil + } + + // Foreground PCS replacement waits for its blocking LPRs; discover them only by owner UID. + requests, err = r.getPipelineRequests(ctx, pcs) if err != nil { return ctrl.Result{}, err } } - return r.reconcileWorkload(ctx, deployment, dgd, pcs, pcsg, requests) + if result, err := r.reconcileModelDownloads(ctx, deployment, dgd); err != nil || result.RequeueAfter > 0 { + return result, err + } + + return r.reconcileWorkloads(ctx, deployment, dgd, pcs, pcsgs, requests) } -// reconcileWorkload consumes owned observations; DGD and deployment are non-nil. -// A nil pcs means initial creation (pcsg is nil and requests is empty). A nil pcsg -// means no current, non-deleting group is observed, not zero capacity. A non-nil -// pcsg is owned by pcs; only its live Spec.Replicas controls existing engine capacity. -// On error, a nonzero RequeueAfter requests a bounded deadline retry; Reconcile -// records the error and persists status before returning that retry to controller-runtime. -func (r *graphReconciler) reconcileWorkload( +// reconcileWorkloads consumes owned observations; DGD and deployment are non-nil. +// A nil pcs means initial creation, with no pcsgs or requests. Otherwise pcsgs +// contains every configured, non-deleting group. On error, Reconcile persists +// status before applying deadline retries. +func (r *graphReconciler) reconcileWorkloads( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, - pcsg *grovev1alpha1.PodCliqueScalingGroup, + pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, requests []lpxv1alpha1.LPUPipelineRequest, ) (result ctrl.Result, err error) { - if result, err := r.reconcileModelDownloads(ctx, deployment, dgd); err != nil || result.RequeueAfter > 0 { - return result, err - } - - workload, err := lpx.ResolveSelectedWorkload(ctx, dgd, r.modelRegistry) + // Resolve and render every workload before deleting the running PCS. + workloads, plans, err := r.resolveWorkloads(ctx, deployment, dgd) if err != nil { return ctrl.Result{}, err } - // Validate the complete replacement before either immutable change can delete the running PCS. - plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) - if err != nil { - return ctrl.Result{}, err - } - desiredPCS, resources, err := r.renderPodCliqueSet(ctx, deployment, dgd, workload, plan) + // Share the rendered identities with capacity management and request publication. + desiredPCS, resources, err := r.renderPodCliqueSet(ctx, deployment, dgd, workloads, plans) if err != nil { return ctrl.Result{}, err } - // Model composition changes require replacement: Grove makes clique membership immutable. - // Replica-only changes, including top-level PCS replica drift, use normal mutable sync. - if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != workload.Digest().String() { + // Immutable composition changes replace the PCS; replica-only changes use /scale. + if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != desiredPCS.Annotations[lpx.WorkloadDigestAnnotation] { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") - return ctrl.Result{}, r.deletePodCliqueSet(ctx, pcs) + return ctrl.Result{}, deletePodCliqueSet(ctx, r, pcs) } - // Explicit DGD replicas authorize scale writes; omission leaves the live count externally managed. - explicitReplicas := lpx.ServingComponent(dgd).Replicas - if explicitReplicas == nil && pcs != nil { - // Do not derive an externally managed request set from an unknown replica count. - if pcsg == nil { - setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the LPX scaling group") - return ctrl.Result{}, nil + // Resolve capacity and requests for the complete graph before any group is changed. + var ( + groupNames = slices.Sorted(maps.Keys(workloads)) + + desiredRequests = make(map[string]*lpxv1alpha1.LPUPipelineRequest) + explicitReplicas = make(map[string]*int32, len(plans)) + missingRequests []*lpxv1alpha1.LPUPipelineRequest + ) + + // Index the graph's observed requests once, retaining scheduler-owned receipts. + requestsByName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(requests)) + for i := range requests { + request := &requests[i] + requestsByName[request.Name] = request + } + + // Resolve each workload in the same order used for request publication. + for _, groupName := range groupNames { + plan := plans[groupName] + workload := workloads[groupName] + pcsg := pcsgs[plan.LPXScalingGroup] + explicitReplicas[plan.LPXScalingGroup] = dgd.GetComponentByName(groupName).Replicas + + // External scalers own live capacity once Grove has created the groups. + if pcs != nil && explicitReplicas[plan.LPXScalingGroup] == nil { + plan.Replicas = pcsg.Spec.Replicas + if err := plan.ValidateReplicaCount(); err != nil { + return ctrl.Result{}, err + } } - plan.Replicas = pcsg.Spec.Replicas - if err := plan.ValidateReplicaCount(); err != nil { - return ctrl.Result{}, err + desired, missing, intentChanged := resolvePipelineRequests(deployment, requestsByName, workload, plan) + + if intentChanged { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") + return ctrl.Result{}, deletePodCliqueSet(ctx, r, pcs) } - } - // Preserve matching scheduler receipts and collect new LPRs in publication order. - desiredRequests, missingRequests, intentChanged := resolvePipelineRequests(deployment, requests, workload, plan) - if intentChanged { - setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") - return ctrl.Result{}, r.deletePodCliqueSet(ctx, pcs) + maps.Copy(desiredRequests, desired) + missingRequests = append(missingRequests, missing...) } - // Schedule deadline wakes without consuming errors needed by the status boundary. + // One graph-wide deadline observation preserves the earliest wakeup on every return. expiredRequests, deadlineAt := pipelineRequestDeadlines(desiredRequests, pipelineRequestDeadlineSeconds(dgd)) defer func() { result = requeueForPipelineRequestDeadline(deadlineAt, result, err) }() - // Apply explicit scale-in before deadline failures can block publication. - if err := r.scaleDownPodCliqueScalingGroup(ctx, pcsg, explicitReplicas); err != nil { - return ctrl.Result{}, err - } - - // Deadline failures stop publication even when no timer or cleanup is needed. - if len(expiredRequests) > 0 || isSchedulingFailedConditionCurrent(deployment) { - // Explicit or external scale-in has already lowered capacity; failure must not prevent its LPR cleanup. - if pcsg != nil { - if err := r.deletePipelineRequests(ctx, pipelineRequestsPendingDeletion(requests, desiredRequests)); err != nil { + // Scale only existing groups with explicitly managed capacity. + for groupName, pcsg := range pcsgs { + if replicas := explicitReplicas[groupName]; replicas != nil { + if err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, *replicas); err != nil { return ctrl.Result{}, err } - - // Do not delete those names again during deadline cleanup or mutate the expired requests' backing slice. - requests = slices.DeleteFunc(slices.Clone(requests), func(request lpxv1alpha1.LPUPipelineRequest) bool { - _, desired := desiredRequests[request.Name] - return !desired - }) } - return r.reconcilePipelineRequestDeadline(ctx, deployment, pcsg, requests, expiredRequests, explicitReplicas != nil) } - clearPreviousSchedulingFailure(deployment) + // Remove requests only after all scale-down writes have succeeded. + removed := pipelineRequestsPendingDeletion(requests, desiredRequests) + if err := r.deletePipelineRequests(ctx, removed); err != nil { + return ctrl.Result{}, err + } - // Existing LPRs require an observed group so scale-down precedes their deletion. - // Without LPRs, initial PCS synchronization can proceed before Grove creates the group. - if pcsg == nil && len(requests) > 0 { - setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the LPX scaling group") - return ctrl.Result{}, nil + // Failure blocks publication while each workload independently cleans up expired suffixes. + if len(expiredRequests) > 0 || isSchedulingFailedConditionCurrent(deployment) { + retained := slices.DeleteFunc(slices.Clone(requests), func(request lpxv1alpha1.LPUPipelineRequest) bool { + _, desired := desiredRequests[request.Name] + return !desired + }) + return r.reconcileSchedulingFailure(ctx, deployment, pcsgs, explicitReplicas, retained, expiredRequests) } - // Deletion is terminal for this pass; LPR events resume publication after cleanup. - if removed := pipelineRequestsPendingDeletion(requests, desiredRequests); len(removed) > 0 { + acknowledgeSchedulingRetry(deployment) + + // Terminating requests must settle before synchronization or scale-out. + if len(removed) > 0 { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for removed LPX requests to finish deletion") - return ctrl.Result{}, r.deletePipelineRequests(ctx, removed) + return ctrl.Result{}, nil } - modified, _, err := commoncontroller.SyncObservedResource(ctx, r, deployment, pcs, desiredPCS, commoncontroller.WithPreservedListOrder(), commoncontroller.WithMetadataSync()) + modified, _, err := commoncontroller.SyncObservedResource(ctx, r, deployment, pcs, desiredPCS, commoncontroller.WithPreservedListOrder()) if err != nil { if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the PodCliqueSet cache observation") @@ -282,67 +300,42 @@ func (r *graphReconciler) reconcileWorkload( return ctrl.Result{}, err } - if err := r.reconcileRuntimeResources(ctx, deployment, dgd, resources); err != nil { + if err := r.reconcileRuntimeResources(ctx, deployment, resources); err != nil { return ctrl.Result{}, err } - // Publication requires an observed PCS spec and its current, non-deleting group. - if modified || pcsg == nil { + // Observe the created or updated PCS before scaling or publishing requests. + if pcs == nil || modified { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for Grove to observe the workload") return ctrl.Result{}, nil } // Scale-out proceeds after deleting old names, without waiting for Pods. - if err := r.scalePodCliqueScalingGroup(ctx, pcsg, explicitReplicas); err != nil { - return ctrl.Result{}, err + for _, groupName := range groupNames { + plan := plans[groupName] + if replicas := explicitReplicas[plan.LPXScalingGroup]; replicas != nil { + if err := scalePodCliqueScalingGroup(ctx, r, pcsgs[plan.LPXScalingGroup], *replicas); err != nil { + return ctrl.Result{}, err + } + } } - // Publication is terminal for this pass; readiness needs watched scheduler receipts. if len(missingRequests) > 0 { return ctrl.Result{}, r.reconcilePipelineRequests(ctx, deployment, pcs, missingRequests) } defer func() { + // Old configmaps remain available until all workloads are Ready. if err == nil { err = r.deleteUnusedConfigMaps(ctx, deployment, resources) } }() - return r.reconcileReadiness(ctx, deployment, dgd, pcs, pcsg, desiredRequests) -} - -// scaleDownPodCliqueScalingGroup lowers capacity before asynchronous request cleanup. -// A nil pcsg defers scaling until the group is observed; a non-nil pcsg is owned -// and not deleting. Nil replicas leaves capacity externally managed. Scale-out -// must wait until terminating request names disappear. -func (r *graphReconciler) scaleDownPodCliqueScalingGroup(ctx context.Context, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas *int32) error { - if pcsg == nil || replicas == nil || *replicas >= pcsg.Spec.Replicas { - return nil - } - return r.scalePodCliqueScalingGroup(ctx, pcsg, replicas) + return r.reconcileReadiness(ctx, deployment, dgd, pcs, pcsgs, plans, desiredRequests) } -// scalePodCliqueScalingGroup updates the non-nil, already-owned PCSG using its -// observed resource version. Nil replicas leaves capacity to external autoscaling. -// A successful write updates the supplied observation for the rest of this pass. -func (r *graphReconciler) scalePodCliqueScalingGroup(ctx context.Context, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas *int32) error { - if replicas == nil || pcsg.Spec.Replicas == *replicas { - return nil - } - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: pcsg.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: *replicas}, - } - if err := r.SubResource("scale").Update(ctx, pcsg, client.WithSubResourceBody(scale)); err != nil { - return fmt.Errorf("scale LPX PodCliqueScalingGroup %q: %w", pcsg.Name, err) - } - pcsg.Spec.Replicas = *replicas - return nil -} - -// reconcileRuntimeResources synchronizes native rendered resources and the -// serving endpoint. deployment and DGD are non-nil. -func (r *graphReconciler) reconcileRuntimeResources(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, resources []client.Object) error { +// reconcileRuntimeResources synchronizes additional resources for the LPX deployment. +func (r *graphReconciler) reconcileRuntimeResources(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, resources []client.Object) error { for _, resource := range resources { if _, _, err := commoncontroller.SyncResource(ctx, r, deployment, func(context.Context) (client.Object, bool, error) { return resource, false, nil @@ -350,59 +343,7 @@ func (r *graphReconciler) reconcileRuntimeResources(ctx context.Context, deploym return err } } - return r.reconcileEndpoint(ctx, deployment, dgd) -} - -func (r *graphReconciler) reconcileEndpoint(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment) error { - // One serving endpoint belongs to the engine, independently of mutable component names. - pcsName := dynamo.PCSNameForLPX(deployment) - serviceName := pcsName + "-serve" - backend := commoncontroller.GetDiscoveryBackend(r.config.Discovery.Backend, dgd.Annotations) - - observed := &corev1.Service{} - if err := r.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: serviceName}, observed); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - observed = nil - } - - // Switching discovery backends removes only this deployment's endpoint. - if backend != configv1alpha1.DiscoveryBackendKubernetes { - if observed == nil { - return nil - } - if err := commoncontroller.CheckControllerOwnership(observed, deployment, r.Scheme()); err != nil { - return err - } - return client.IgnoreNotFound(r.Delete(ctx, observed, client.Preconditions{UID: &observed.UID, ResourceVersion: &observed.ResourceVersion})) - } - - component := lpx.ServingComponent(dgd) - service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ - ServiceName: serviceName, Namespace: deployment.Namespace, - ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, - DynamoNamespace: dgd.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, - Labels: dynamo.GetDGDComponentResourceLabels(dgd, component.ComponentName, component), - Annotations: dynamo.GetDGDComponentResourceAnnotations(dgd, component.ComponentName, component), - }) - if err != nil { - return err - } - // Keep cleanup and pod selection scoped to this materialization. - service.Labels[deploymentUIDLabel] = string(deployment.UID) - service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue - service.Spec.Selector[grovecommon.LabelPartOfKey] = pcsName - - // Preserve API-allocated addresses and IP families across selector updates. - if observed != nil { - service.Spec.ClusterIP = observed.Spec.ClusterIP - service.Spec.ClusterIPs = observed.Spec.ClusterIPs - service.Spec.IPFamilies = observed.Spec.IPFamilies - service.Spec.IPFamilyPolicy = observed.Spec.IPFamilyPolicy - } - _, _, err = commoncontroller.SyncObservedResource(ctx, r, deployment, observed, service, commoncontroller.WithMetadataSync()) - return err + return nil } // reconcileReadiness combines scheduler receipts with Grove runtime readiness. @@ -412,15 +353,29 @@ func (r *graphReconciler) reconcileReadiness( deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, - pcsg *grovev1alpha1.PodCliqueScalingGroup, + pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, + plans map[string]*lpx.MaterializationPlan, requests map[string]*lpxv1alpha1.LPUPipelineRequest, ) (ctrl.Result, error) { - readiness, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, dgd, pcs, pcsg) - if err != nil { - return ctrl.Result{}, err - } + readiness := dynamo.GroveReadiness{Ready: true} + deployment.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) + + componentGroups := lpx.ComponentGroups(dgd) - deployment.Status.Components = readiness.ComponentStatuses + for _, groupName := range slices.Sorted(maps.Keys(plans)) { + plan := plans[groupName] + + observed, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, dgd, groupName, componentGroups[groupName], pcs, pcsgs[plan.LPXScalingGroup]) + if err != nil { + return ctrl.Result{}, err + } + + maps.Copy(deployment.Status.Components, observed.ComponentStatuses) + + if !observed.Ready && readiness.Ready { + readiness = observed + } + } if !setPipelineRequestReadyCondition(deployment, requests) { return ctrl.Result{}, nil @@ -441,19 +396,6 @@ func (r *graphReconciler) reconcileReadiness( return ctrl.Result{RequeueAfter: delay}, nil } -// deletePodCliqueSet deletes the already-validated, non-nil PCS. -func (r *graphReconciler) deletePodCliqueSet(ctx context.Context, pcs *grovev1alpha1.PodCliqueSet) error { - return client.IgnoreNotFound( - r.Delete( - ctx, - pcs, - // Foreground GC so that blocking LPRs and Grove dependents can be removed. - client.PropagationPolicy(metav1.DeletePropagationForeground), - client.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, - ), - ) -} - // deleteUnusedConfigMaps runs only after readiness so existing pods keep their // immutable configuration during replacement. Maps belong to the non-nil LPXGD, // not the PCS, and therefore survive PCS garbage collection. diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index c0d9f9af4f89..78fcdcf5d114 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -38,7 +38,6 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -142,7 +141,7 @@ func TestPipelineRequestDeadlineContinuesDuringRequestDeletion(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -191,15 +190,8 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { t.Log("Reject the status write and forbid cleanup until the failure becomes durable") statusErr := errors.New("status write unavailable") failStatus := true - groupMissing := true statusWrites, scaleWrites, deletes := 0, 0, 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && groupMissing { - return apierrors.NewNotFound(consts.PodCliqueScalingGroupGVR.GroupResource(), key.Name) - } - return delegated.Get(ctx, key, object, opts...) - }, SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { if subresource == "status" { statusWrites++ @@ -226,7 +218,7 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { require.NoError(t, r.Get(ctx, request.NamespacedName, child)) require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) - t.Log("Record failure even without a PCSG observation, but leave the LPR intact") + t.Log("Persist the failure while leaving capacity and requests intact for this pass") failStatus = false result, err = r.Reconcile(ctx, request) require.NoError(t, err) @@ -238,16 +230,9 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) failed := meta.FindStatusCondition(child.Status.Conditions, "Ready").DeepCopy() - t.Log("Persisted failure cannot authorize cleanup until the live group is observed") - result, err = r.Reconcile(ctx, request) - require.NoError(t, err) - require.Zero(t, result, "the PCSG watch resumes cleanup") - require.Zero(t, scaleWrites) - require.Zero(t, deletes) require.True(t, apiequality.Semantic.DeepEqual(pending, getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name))) - t.Log("Observing both the persisted failure and the group allows expired-replica cleanup") - groupMissing = false + t.Log("Observing the persisted failure allows expired-replica cleanup") result, err = r.Reconcile(ctx, request) require.NoError(t, err) require.Zero(t, result, "request and group events drive cleanup") @@ -266,12 +251,64 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { requirePipelineRequestNotFound(t, ctx, r.Client, pending.Namespace, pending.Name) } +func TestLPXDeadlineWaitsForScalingGroup(t *testing.T) { + t.Log("Publish an expired request and hide its scaling group from the cache") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, ctx, r.Client, objects...) + publishSelectedLPXForTest(t, ctx, r, child, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) + expired.Status = newTestPipelineRequest(child, pcs, expired.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(ctx, expired)) + expired = getTestPipelineRequest(t, ctx, r.Client, expired.Namespace, expired.Name) + base := r.Client + r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + groups.Items = nil + return nil + } + return delegated.List(ctx, list, opts...) + }, + }) + + t.Log("Wait for the group watch without evaluating deadlines or removing requests") + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(ctx, key) + require.NoError(t, err) + require.Zero(t, result) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + require.Equal(t, expired, getTestPipelineRequest(t, ctx, r.Client, expired.Namespace, expired.Name)) + + t.Log("Once the group is observed, persist scheduling failure before cleanup") + r.Client = base + result, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + require.Equal(t, expired, getTestPipelineRequest(t, ctx, r.Client, expired.Namespace, expired.Name)) + + t.Log("A later pass lowers capacity and removes the expired request") + _, err = r.Reconcile(ctx, key) + require.NoError(t, err) + group := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: desired.plan.LPXScalingGroup}, group)) + require.Zero(t, group.Spec.Replicas) + requirePipelineRequestNotFound(t, ctx, r.Client, expired.Namespace, expired.Name) +} + func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { - t.Log("Expire an interior engine while a higher ordinal is still Bound") + t.Log("Expire an interior workload while a higher ordinal is still Bound") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, desired) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -377,7 +414,7 @@ func TestLPXDownloadsBeforePublication(t *testing.T) { dgd.Spec.Components[1], dgd.Spec.Components[0], } } - component := lpx.ServingComponent(dgd) + component := dgd.GetComponentByName("lpx") if scenario.componentName != "" { component.ComponentName = scenario.componentName } @@ -486,18 +523,17 @@ func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { require.NotEqual(t, pending.UID, replacement.UID) } -func TestLPXEndpointLifecycle(t *testing.T) { +func TestLPXServiceReconciliation(t *testing.T) { const uppercaseComponentName = "LPX" t.Log("Configure an uppercase LPX component and independently named materialization using Kubernetes discovery") - dgd := loadTestDGD(t, lpx.PipelineSingle, "test-build") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Components[0].ComponentName = uppercaseComponentName dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) dgd.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} - dgd.Spec.Annotations = map[string]string{"example.com/model-discovery": "enabled"} - dgd.Spec.Labels = map[string]string{"example.com/policy": "enabled"} - child := newLPXTestDeployment(t, dgd) child.Name = "independent-materialization" - r := newLPXTestReconciler(t, nil, child, dgd) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) t.Log("Keep the ordinary model Service under the source DGD's ownership") modelService := &corev1.Service{ @@ -515,7 +551,9 @@ func TestLPXEndpointLifecycle(t *testing.T) { beforeModelService := modelService.DeepCopy() t.Log("Publish the LPX-owned endpoint with its serving-role selector") - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + _, resources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + require.NoError(t, r.reconcileRuntimeResources(t.Context(), child, resources)) service := &corev1.Service{} key := client.ObjectKey{Namespace: dgd.Namespace, Name: dynamo.PCSNameForLPX(child) + "-serve"} require.NoError(t, r.Get(t.Context(), key, service)) @@ -523,71 +561,33 @@ func TestLPXEndpointLifecycle(t *testing.T) { require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) require.Equal(t, dynamo.PCSNameForLPX(child), service.Spec.Selector[grovecommon.LabelPartOfKey]) - t.Log("Converge propagated endpoint metadata, including removed keys") - for _, value := range []string{"changed", ""} { - if value == "" { - dgd.Spec.Labels, dgd.Spec.Annotations = nil, nil - } else { - dgd.Spec.Labels["example.com/policy"] = value - dgd.Spec.Annotations["example.com/model-discovery"] = value - } - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) - updated := &corev1.Service{} - require.NoError(t, r.Get(t.Context(), key, updated)) - require.Equal(t, value, updated.Labels["example.com/policy"]) - require.Equal(t, value, updated.Annotations["example.com/model-discovery"]) - if value == "" { - require.NotContains(t, updated.Labels, "example.com/policy") - require.NotContains(t, updated.Annotations, "example.com/model-discovery") - } - require.Equal(t, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled], updated.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) - require.Equal(t, service.Spec, updated.Spec) - require.Equal(t, service.OwnerReferences, updated.OwnerReferences) - for _, key := range []string{commoncontroller.NvidiaAnnotationHashKey, commoncontroller.NvidiaAnnotationGenerationKey} { - require.NotEmpty(t, updated.Annotations[key]) - require.Equal(t, service.Annotations[key], updated.Annotations[key]) - } - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) - unchanged := &corev1.Service{} - require.NoError(t, r.Get(t.Context(), key, unchanged)) - require.Equal(t, updated.ResourceVersion, unchanged.ResourceVersion) - } - - t.Log("Preserve API-allocated Service fields when the endpoint selector changes") + t.Log("Update the endpoint selector when the component's Dynamo namespace changes") require.NoError(t, r.Get(t.Context(), key, service)) - service.Spec.ClusterIP = "10.96.0.10" - service.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} - service.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} - service.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyPreferDualStack) - require.NoError(t, r.Update(t.Context(), service)) - allocated := service.DeepCopy() + expected := service.DeepCopy() for _, global := range []bool{true, false} { dgd.Spec.Components[0].GlobalDynamoNamespace = global - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + _, resources, err = r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + require.NoError(t, r.reconcileRuntimeResources(t.Context(), child, resources)) require.NoError(t, r.Get(t.Context(), key, service)) - allocated.Spec.Selector[consts.KubeLabelDynamoNamespace] = dgd.GetDynamoNamespaceForComponent(&dgd.Spec.Components[0]) - require.Equal(t, allocated.Spec, service.Spec) - require.Equal(t, allocated.OwnerReferences, service.OwnerReferences) + expected.Spec.Selector[consts.KubeLabelDynamoNamespace] = dgd.GetDynamoNamespaceForComponent(&dgd.Spec.Components[0]) + require.Equal(t, expected.Spec, service.Spec) + require.Equal(t, expected.OwnerReferences, service.OwnerReferences) t.Log("Reconcile the unchanged endpoint without another write") version := service.ResourceVersion - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) + require.NoError(t, r.reconcileRuntimeResources(t.Context(), child, resources)) require.NoError(t, r.Get(t.Context(), key, service)) require.Equal(t, version, service.ResourceVersion) } - t.Log("Remove the LPX-owned discovery endpoint when switching back to non-Kubernetes discovery") - delete(dgd.Annotations, consts.KubeAnnotationDynamoDiscoveryBackend) - require.NoError(t, r.reconcileEndpoint(t.Context(), child, dgd)) - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), key, service))) - - t.Log("Discovery cleanup leaves the ordinary DGD model service unchanged") + t.Log("Service reconciliation leaves the ordinary DGD model Service unchanged") require.NoError(t, r.Get(t.Context(), modelKey, modelService)) require.Equal(t, beforeModelService, modelService) } -func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { - t.Log("Publish a speculative engine using its frozen child identity") +func TestLPXWorkloadOrderAndUnrelatedEditsPreservePublication(t *testing.T) { + t.Log("Publish a speculative workload using its frozen child identity") child, dgd, registry := newLPXSpecDecodeTestDGD(t) dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, @@ -606,7 +606,7 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) beforeChild := child.DeepCopy() - t.Log("Reorder engines, update the frontend and enable an ordinary checkpoint without changing LPX") + t.Log("Reorder workloads, update the frontend and enable an ordinary checkpoint without changing LPX") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) components := dgd.Spec.Components components[0], components[1] = components[1], components[0] @@ -624,7 +624,12 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, dgd, afterSelected) require.Equal(t, beforePCS, afterPCS) require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) - _, missing, changed := resolvePipelineRequests(child, beforeRequests.Items, afterSelected.workload, afterSelected.plan) + observed := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(beforeRequests.Items)) + for i := range beforeRequests.Items { + request := &beforeRequests.Items[i] + observed[request.Name] = request + } + _, missing, changed := resolvePipelineRequests(child, observed, afterSelected.workload, afterSelected.plan) require.False(t, changed) require.Empty(t, missing) afterRequests := &lpxv1alpha1.LPUPipelineRequestList{} @@ -635,13 +640,13 @@ func TestLPXEngineOrderAndUnrelatedEditsPreservePublication(t *testing.T) { require.Equal(t, beforeDGD, dgd) } -func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { +func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { t.Run("deadline", func(t *testing.T) { - t.Log("Observe a completed engine and a newer pending request in their shared PCS") + t.Log("Observe a completed workload and a newer pending request in their shared PCS") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -685,7 +690,7 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) require.Equal(t, int32(1), group.Spec.Replicas) - t.Log("After cleanup the later edit can retry without replacing the serving engine") + t.Log("After cleanup the later edit can retry without replacing the serving workload") retiredUID := pending.UID pending.Finalizers = nil require.NoError(t, r.Update(ctx, pending)) @@ -703,7 +708,7 @@ func TestLPXFailedScaleOutPreservesServingEngines(t *testing.T) { } func TestLPXInvalidReplacementPreservesExistingWorkload(t *testing.T) { - t.Log("Publish a speculative engine with two draft models") + t.Log("Publish a speculative workload with two draft models") ctx := t.Context() child, dgd, registry := newLPXSpecDecodeTestDGD(t) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) @@ -720,7 +725,7 @@ func TestLPXInvalidReplacementPreservesExistingWorkload(t *testing.T) { agent := draft.ComponentRole(v1beta1.ComponentRoleLPXAgent) mounts := agent.PodTemplate.Spec.Containers[0].VolumeMounts agent.PodTemplate.Spec.Containers[0].VolumeMounts = nil - replacement, err := lpx.ResolveSelectedWorkload(ctx, dgd, registry) + replacement, err := lpx.ResolveWorkload(ctx, dgd, singleGroupComponents(t, dgd), registry) require.NoError(t, err) require.NotEqual(t, selected.workload.Digest(), replacement.Digest()) updateTestDGD(t, r, child, dgd) @@ -751,7 +756,7 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(4)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(4)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -774,7 +779,7 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { } t.Log("Apply a four-to-three replica edit despite the retained interior expiry") - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(3)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(3)) updateTestDGD(t, r, child, dgd) request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} for range 3 { @@ -816,7 +821,7 @@ func TestLPXExternalScaleRejectsInvalidReplicaCount(t *testing.T) { t.Log("Observe externally managed capacity above the materialization limit") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(dgd).Replicas = nil + dgd.GetComponentByName("lpx").Replicas = nil r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) @@ -850,7 +855,7 @@ func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = nil + dgd.GetComponentByName("lpx").Replicas = nil r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) selected.plan.Replicas = 4 _, missing, changed := resolvePipelineRequests(child, nil, selected.workload, selected.plan) @@ -938,7 +943,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { {name: "inconsistent snapshot", message: "immutable LPX build snapshot"}, } { t.Run(scenario.name, func(t *testing.T) { - t.Log("Stage a PCS and discovery endpoint before the API supplies a PCS UID") + t.Log("Create a PCS, its scaling groups, and a discovery endpoint before publishing requests") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) r := newLPXTestReconciler(t, registry, child, dgd) @@ -950,6 +955,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) pcs.UID = "staged-pcs" require.NoError(t, r.Update(t.Context(), pcs)) + createLPXTestScalingGroups(t, r.Client, pcs) endpoint := &corev1.Service{} endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: pcsKey.Name + "-serve"} require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) @@ -964,7 +970,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { t.Log("Introduce a current terminal input failure, or a transient snapshot outage") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - component := lpx.ServingComponent(dgd) + component := dgd.GetComponentByName("lpx") switch scenario.name { case "render": component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].VolumeMounts = nil @@ -1058,7 +1064,7 @@ func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) publishSelectedLPXForTest(t, ctx, r, child, selected) @@ -1075,7 +1081,8 @@ func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { t.Log("Reconcile the PCS for the new generation before publishing the missing request") result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) - require.Zero(t, result, "the PCS update drives the next reconcile") + require.Positive(t, result.RequeueAfter, "the surviving request's creation-time deadline remains active") + require.LessOrEqual(t, result.RequeueAfter, 30*time.Second) requirePipelineRequestNotFound(t, ctx, r.Client, missing.Namespace, missing.Name) t.Log("Reconciliation recreates deterministic scheduler intent without inheriting a scheduling clock") @@ -1183,15 +1190,15 @@ func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { require.NotNil(t, synced) require.Equal(t, []string{"cond", "agt0", "agt1"}, synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) require.Len(t, synced.Spec.Template.Cliques, 3) - require.Equal(t, scaled.workload.Digest().String(), synced.Annotations[lpx.WorkloadDigestAnnotation]) + require.NotEqual(t, pcs.Annotations[lpx.WorkloadDigestAnnotation], synced.Annotations[lpx.WorkloadDigestAnnotation]) require.Less(t, len(synced.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames), originalCliqueCount) } func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { - t.Log("Publish one engine whose capacity is owned by the native Grove scaling group") + t.Log("Publish one workload whose capacity is owned by the native Grove scaling group") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(dgd).Replicas = nil + dgd.GetComponentByName("lpx").Replicas = nil r, initial := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, initial) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -1232,147 +1239,6 @@ func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { require.Equal(t, pcsUID, metav1.GetControllerOf(replacement).UID) } -func TestLPXReadableNamesSurviveServingComponentChanges(t *testing.T) { - t.Log("Publish a speculative engine with long deployment and component names") - _, dgd, registry := newLPXSpecDecodeTestDGD(t) - dgd.Name = "test-models-gpt-oss-20b-lp20-b300" - dgd.Spec.Components[0].ComponentName = "serving-component-with-name" - dgd.Spec.Components[1].ComponentName = "draft-component-with-name" - dgd.Spec.Components[1].Replicas = ptr.To(int32(1)) - dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - child := newLPXTestDeployment(t, dgd) - r := newLPXTestReconciler(t, registry, child, dgd) - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - _, err := r.Reconcile(t.Context(), request) - require.NoError(t, err) - root := dynamo.PCSNameForLPX(child) - require.True(t, strings.HasPrefix(root, dgd.Name+"-")) - require.Len(t, root, len(dgd.Name)+len("-ffff")) - pcs := &grovev1alpha1.PodCliqueSet{} - pcsKey := client.ObjectKey{Namespace: child.Namespace, Name: root} - require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) - pcs.UID = "stable-pcs" - require.NoError(t, r.Update(t.Context(), pcs)) - endpoint := &corev1.Service{} - endpointKey := client.ObjectKey{Namespace: child.Namespace, Name: root + "-serve"} - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - endpoint.UID = "stable-endpoint" - require.NoError(t, r.Update(t.Context(), endpoint)) - require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) - beforeGroup := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] - beforeOwners := endpoint.OwnerReferences - - t.Log("Observe Grove's group and publish the original requests") - selected := resolveLPXTestWorkload(t, r.modelRegistry, t.Context(), child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, selected) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) - group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - require.NoError(t, r.Create(t.Context(), group)) - _, err = r.Reconcile(t.Context(), request) - require.NoError(t, err) - - for _, change := range []string{"rename serving component", "move conductor"} { - t.Log(change) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) - require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - if change == "rename serving component" { - dgd.Spec.Components[0].ComponentName = "renamed-serving-component" - } else { - conductor := *dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) - dgd.Spec.Components[0].Roles = slices.DeleteFunc(dgd.Spec.Components[0].Roles, func(role v1beta1.ComponentRoleSpec) bool { - return role.Name == v1beta1.ComponentRoleLPXConductor - }) - dgd.Spec.Components[1].Roles = append(dgd.Spec.Components[1].Roles, conductor) - } - dgd.Generation++ - require.NoError(t, r.Update(t.Context(), dgd)) - child.Generation++ - child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") - require.NoError(t, err) - require.NoError(t, r.Update(t.Context(), child)) - - t.Log("Preserve the PCS on rename and retire it when target and draft assignments change") - _, err = r.Reconcile(t.Context(), request) - require.NoError(t, err) - if change == "move conductor" { - require.True(t, apierrors.IsNotFound(r.Get(t.Context(), pcsKey, &grovev1alpha1.PodCliqueSet{}))) - require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) - require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - require.Equal(t, "stable-endpoint", string(endpoint.UID)) - - t.Log("Simulate dependent request garbage collection, which the fake client does not perform") - requests := &lpxv1alpha1.LPUPipelineRequestList{} - require.NoError(t, r.List(t.Context(), requests)) - require.NotEmpty(t, requests.Items) - for index := range requests.Items { - require.Equal(t, "stable-pcs", string(metav1.GetControllerOf(&requests.Items[index]).UID)) - require.NoError(t, r.Delete(t.Context(), &requests.Items[index])) - } - - t.Log("Recreate the workload under the same readable names after retirement") - _, err = r.Reconcile(t.Context(), request) - require.NoError(t, err) - } - require.NoError(t, r.Get(t.Context(), pcsKey, pcs)) - require.NoError(t, r.Get(t.Context(), endpointKey, endpoint)) - if change == "rename serving component" { - require.Equal(t, "stable-pcs", string(pcs.UID)) - } else { - require.NotEqual(t, "stable-pcs", string(pcs.UID)) - } - require.Equal(t, "stable-endpoint", string(endpoint.UID)) - require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 1) - require.Equal(t, beforeGroup.Name, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Name) - require.Equal(t, beforeGroup.CliqueNames, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) - require.Equal(t, beforeOwners, endpoint.OwnerReferences) - require.Equal(t, dgd.Name, pcs.Labels[consts.KubeLabelDynamoGraphDeploymentName]) - require.Equal(t, lpx.ServingComponent(dgd).ComponentName, endpoint.Spec.Selector[consts.KubeLabelDynamoComponent]) - require.Equal(t, root, endpoint.Spec.Selector[grovecommon.LabelPartOfKey]) - require.Equal(t, consts.KubeLabelValueTrue, endpoint.Spec.Selector[dynamo.LPXServingLabel]) - allPCS, allEndpoints := &grovev1alpha1.PodCliqueSetList{}, &corev1.ServiceList{} - require.NoError(t, r.List(t.Context(), allPCS)) - require.NoError(t, r.List(t.Context(), allEndpoints)) - require.Len(t, allPCS.Items, 1) - require.Len(t, allEndpoints.Items, 1) - - t.Log("Select only the conductor and share one runtime config across the clique roster") - require.NotEmpty(t, pcs.Spec.Template.Cliques) - configHash := pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash] - require.NotEmpty(t, configHash) - cliqueNames := make([]string, 0, len(pcs.Spec.Template.Cliques)) - for _, clique := range pcs.Spec.Template.Cliques { - cliqueNames = append(cliqueNames, clique.Name) - require.Equal(t, configHash, clique.Annotations[consts.AnnotationExtraResourcesHash]) - for key, value := range clique.Labels { - require.Empty(t, validation.IsValidLabelValue(value), key) - } - if clique.Name == "cond" { - for key, value := range endpoint.Spec.Selector { - if key != grovecommon.LabelPartOfKey { - require.Equal(t, value, clique.Labels[key], key) - } - } - } else { - require.NotContains(t, clique.Labels, dynamo.LPXServingLabel) - } - } - require.ElementsMatch(t, []string{"cond", "agt0", "agt1"}, cliqueNames) - - t.Log("Keep runtime allocation aligned with the clique names") - config := &corev1.ConfigMap{} - configKey := client.ObjectKey{Namespace: child.Namespace, Name: lpx.LPUConfigMapName(root, configHash)} - require.NoError(t, r.Get(t.Context(), configKey, config)) - require.True(t, metav1.IsControlledBy(config, child)) - require.NotContains(t, config.Data, "datacenter.toml") - for _, clique := range pcs.Spec.Template.Cliques { - if clique.Name == "cond" { - require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_ALLOCATION", Value: "agt0:agt1"}) - } - } - } -} - func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { for _, editDGD := range []bool{true, false} { name := "child metadata edit" @@ -1382,7 +1248,7 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { t.Run(name, func(t *testing.T) { t.Log("Materialize Grove for two replicas before publishing scheduler intent") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, t.Context(), r.Client, objects...) @@ -1417,7 +1283,7 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { if creates == 1 { if editDGD { require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(1)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(1)) dgd.Generation++ return delegated.Update(ctx, dgd) } @@ -1462,7 +1328,7 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { _, err = r.Reconcile(t.Context(), request) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.Equal(t, *lpx.ServingComponent(dgd).Replicas, group.Spec.Replicas) + require.Equal(t, *dgd.GetComponentByName("lpx").Replicas, group.Spec.Replicas) require.NoError(t, r.List(t.Context(), requests)) require.Len(t, requests.Items, int(group.Spec.Replicas)) require.Equal(t, retained.UID, getTestPipelineRequest(t, t.Context(), r.Client, retained.Namespace, retained.Name).UID) @@ -1481,7 +1347,7 @@ func TestLPXReconcileWaitsForMatchingDGDRevision(t *testing.T) { child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) r := newLPXTestReconciler(t, registry, child, dgd) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) dgd.Generation++ if dgdAhead { require.NoError(t, r.Update(t.Context(), dgd)) @@ -1528,53 +1394,6 @@ func TestLPXReconcileWaitsForMatchingDGDRevision(t *testing.T) { } } -func TestScalePodCliqueScalingGroup(t *testing.T) { - conflict := apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), "group", errors.New("stale version")) - for _, tc := range []struct { - name string - replicas *int32 - err error - wantUpdates int - }{ - {name: "scale out", replicas: ptr.To(int32(12)), wantUpdates: 1}, - {name: "scale in", replicas: ptr.To(int32(2)), wantUpdates: 1}, - {name: "scale to zero", replicas: ptr.To(int32(0)), wantUpdates: 1}, - {name: "unchanged", replicas: ptr.To(int32(9))}, - {name: "omitted replicas"}, - {name: "conflict", replicas: ptr.To(int32(12)), err: conflict, wantUpdates: 1}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Log("Use an already-owned group observation without rendering a workload") - group := &grovev1alpha1.PodCliqueScalingGroup{ - ObjectMeta: metav1.ObjectMeta{Name: "group", Namespace: "default", ResourceVersion: "7"}, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 9}, - } - updates := 0 - r := &graphReconciler{Client: interceptor.NewClient(newLPXTestClient(t), interceptor.Funcs{ - SubResourceUpdate: func(_ context.Context, _ client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - require.Equal(t, "scale", subresource) - require.Same(t, group, object) - scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) - require.Equal(t, group.ResourceVersion, scale.ResourceVersion) - require.Equal(t, *tc.replicas, scale.Spec.Replicas) - updates++ - return tc.err - }, - })} - - t.Log("Only explicit changes write scale; successful writes update the in-pass observation") - err := r.scalePodCliqueScalingGroup(t.Context(), group, tc.replicas) - require.ErrorIs(t, err, tc.err) - require.Equal(t, tc.wantUpdates, updates) - want := int32(9) - if tc.replicas != nil && tc.err == nil { - want = *tc.replicas - } - require.Equal(t, want, group.Spec.Replicas) - }) - } -} - func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { t.Log("A request-list failure prevents deriving any active deadline") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -1603,7 +1422,7 @@ func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { t.Log("Observe an externally inflated group through a cache that stays stale after scale") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) @@ -1613,14 +1432,12 @@ func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { stale := group.DeepCopy() scaled, updates := false, 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if scaled && key == client.ObjectKeyFromObject(stale) { - if observed, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { - stale.DeepCopyInto(observed) - return nil - } + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if observed, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok && scaled { + observed.Items = []grovev1alpha1.PodCliqueScalingGroup{*stale.DeepCopy()} + return nil } - return delegated.Get(ctx, key, object, opts...) + return delegated.List(ctx, list, opts...) }, SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { if subresource != lpxTestScaleSubresource { @@ -1644,7 +1461,7 @@ func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - lpx.ServingComponent(dgd).Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -1656,7 +1473,7 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { liveDGD := &v1beta1.DynamoGraphDeployment{} require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), liveDGD)) - lpx.ServingComponent(liveDGD).Replicas = ptr.To(int32(1)) + liveDGD.GetComponentByName("lpx").Replicas = ptr.To(int32(1)) liveDGD.Generation++ require.NoError(t, r.Update(ctx, liveDGD)) liveChild := &v1alpha1.LPXGraphDeployment{} @@ -1670,11 +1487,12 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { t.Log("A missing PCSG observation blocks scale-in without deleting existing requests") base := r.Client r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { - return apierrors.NewNotFound(consts.PodCliqueScalingGroupGVR.GroupResource(), key.Name) + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + groups.Items = nil + return nil } - return delegated.Get(ctx, key, object, opts...) + return delegated.List(ctx, list, opts...) }, }) request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} @@ -1700,7 +1518,7 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { t.Log("Reverse the scale-down while the old request still protects its pods") require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), liveDGD)) - lpx.ServingComponent(liveDGD).Replicas = ptr.To(int32(2)) + liveDGD.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) liveDGD.Generation++ require.NoError(t, r.Update(ctx, liveDGD)) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(child), liveChild)) @@ -1743,7 +1561,7 @@ func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Log("Start without Grove resources and create the owned PCS at its immutable minimum") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - component := lpx.ServingComponent(dgd) + component := dgd.GetComponentByName("lpx") component.Replicas, component.MinAvailable = ptr.To(int32(3)), tc.minimum r := newLPXTestReconciler(t, registry, child, dgd) key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} @@ -1771,6 +1589,7 @@ func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, groupTemplate.Name), Namespace: pcs.Namespace, + Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: seed}, @@ -1913,7 +1732,7 @@ func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { } t.Log("Project logical draft counts and make both authored components await the complete pair") - readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, dgd, pcs, getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup)) + readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, dgd, dgd.GetComponentByName("lpx").ComponentName, singleGroupComponents(t, dgd), pcs, getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup)) require.NoError(t, err) require.Len(t, readiness.ComponentStatuses, 2) draft, target := readiness.ComponentStatuses["draft"], readiness.ComponentStatuses["lpx"] @@ -2098,7 +1917,7 @@ func TestLPXPublishedWorkloadCleanupAfterFailure(t *testing.T) { func TestLPXPodCliqueSetListOrder(t *testing.T) { t.Log("Create a PCS through the controller with ordered Agent init containers") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - agent := lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXAgent) + agent := dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXAgent) agent.PodTemplate.Spec.InitContainers = []corev1.Container{ {Name: "setup", Image: "busybox"}, {Name: "migrate", Image: "busybox"}, } @@ -2110,6 +1929,7 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { require.NoError(t, err) require.NotNil(t, pcs) originalUID := pcs.UID + createLPXTestScalingGroups(t, r.Client, pcs) t.Log("Reverse authored init order and verify the controller opts into order-sensitive synchronization") slices.Reverse(agent.PodTemplate.Spec.InitContainers) @@ -2153,7 +1973,7 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { case *grovev1alpha1.PodClique: live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas - require.Equal(t, desired.workload.LPXComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) + require.Equal(t, desired.workload.ServingComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) } } createLPXTestObjects(t, ctx, reconciler.Client, objects...) @@ -2168,18 +1988,19 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { return reader.Get(ctx, key, object, opts...) }}) observe := func() dynamo.GroveReadiness { - group, err := getPodCliqueScalingGroup(ctx, reconciler.Client, pcs) + groups, err := getPodCliqueScalingGroups(ctx, reconciler.Client, pcs) + group := groups[desired.plan.LPXScalingGroup] require.NoError(t, err) - ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, dgd, pcs, group) + ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, dgd, dgd.GetComponentByName("lpx").ComponentName, singleGroupComponents(t, dgd), pcs, group) require.NoError(t, err) require.Len(t, ready.ComponentStatuses, 1) require.Zero(t, rootReads, "reuse the synchronized PCS observation") return ready } - t.Log("Report one complete engine, never separate GPU or Agent component capacity") + t.Log("Report one complete workload, never separate GPU or Agent component capacity") ready := observe() require.True(t, ready.Ready) - status := ready.ComponentStatuses[desired.workload.LPXComponentName()] + status := ready.ComponentStatuses[desired.workload.ServingComponentName()] require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, status.ComponentKind) require.Equal(t, []string{desired.plan.LPXScalingGroup}, status.ComponentNames) require.Equal(t, int32(1), status.Replicas) @@ -2187,7 +2008,7 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { require.Equal(t, ptr.To(int32(1)), status.AvailableReplicas) require.Equal(t, ptr.To(int32(1)), status.ScheduledReplicas) if desired.plan.CyborgClique != "" { - t.Log("A stale GPU-role scale cannot satisfy complete-engine readiness") + t.Log("A stale GPU-role scale cannot satisfy complete-workload readiness") gpu := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique).DeepCopy() require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(gpu), gpu)) gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 1, 1, 1 @@ -2196,19 +2017,19 @@ func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { ready = observe() require.False(t, ready.Ready) require.Equal(t, v1beta1.DGDReadyReasonUpdating, ready.Classification) - t.Log("Partial GPU readiness contributes zero complete engine replicas") + t.Log("Partial GPU readiness contributes zero complete workload replicas") gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 2, 2, 2 gpu.Status.ScheduledReplicas = 2 require.NoError(t, reconciler.Update(ctx, gpu)) ready = observe() require.False(t, ready.Ready) require.Equal(t, v1beta1.DGDReadyReasonPodsNotReady, ready.Classification) - require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.LPXComponentName()].AvailableReplicas) + require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.ServingComponentName()].AvailableReplicas) gpu.Status.ReadyReplicas = 2 require.NoError(t, reconciler.Update(ctx, gpu)) require.True(t, observe().Ready) } - t.Log("Reject stale complete-engine capacity even when every existing role is ready") + t.Log("Reject stale complete-workload capacity even when every existing role is ready") group := &grovev1alpha1.PodCliqueScalingGroup{} require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.LPXScalingGroup}, group)) group.Spec.Replicas = 2 @@ -2313,7 +2134,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { // lpxTestWorkload is a fixture DSL, not reconciliation state. type lpxTestWorkload struct { - workload *lpx.SelectedWorkload + workload *lpx.Workload plan *lpx.MaterializationPlan requests []lpxv1alpha1.LPUPipelineRequest } @@ -2322,7 +2143,7 @@ func resolveLPXTestWorkload(t *testing.T, registry lpx.ModelRegistry, ctx contex deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, ) *lpxTestWorkload { t.Helper() - workload, err := lpx.ResolveSelectedWorkload(ctx, dgd, registry) + workload, err := lpx.ResolveWorkload(ctx, dgd, singleGroupComponents(t, dgd), registry) require.NoError(t, err) plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) require.NoError(t, err) @@ -2528,6 +2349,12 @@ func lpxMaterializedObjects( t.Helper() t.Log("Render LPX intent once and emulate the API-server and Grove observations") pcs := renderLPXTestPodCliqueSet(t, t.Context(), reconciler, deployment, dgd, desired) + return materializeLPXTestPCS(t, deployment, pcs, desired.plan) +} + +// materializeLPXTestPCS constructs native Grove observations for each independent workload. +func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, plans ...*lpx.MaterializationPlan) []client.Object { + t.Helper() pcs.TypeMeta = metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueSet"} pcs.UID, pcs.Generation = "pcs-uid", 1 pcs.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)} @@ -2537,57 +2364,86 @@ func lpxMaterializedObjects( } // Emulate Grove materialization after the controller applies the desired live scale. - groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].DeepCopy() - group := &grovev1alpha1.PodCliqueScalingGroup{ - TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, - ObjectMeta: metav1.ObjectMeta{ - Name: desired.plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: "lpu-group-uid", - Generation: 1, Labels: grovecommon.GetDefaultLabelsForPodCliqueSetManagedResources(pcs.Name), Annotations: groupTemplate.Annotations, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }, - Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ - Replicas: desired.plan.Replicas, MinAvailable: groupTemplate.MinAvailable, - CliqueNames: groupTemplate.CliqueNames, - }, - Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), - }, - } - group.Labels[grovecommon.LabelPartOfKey] = pcs.Name - group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" - objects := []client.Object{pcs, group} - - // Each Grove replica owns independent cliques. - for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { - parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} - for _, template := range pcs.Spec.Template.Cliques { - rendered := template.DeepCopy() - name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) - clique := &grovev1alpha1.PodClique{ - TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, - Labels: rendered.Labels, Annotations: rendered.Annotations, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, - }, - Spec: rendered.Spec, - Status: grovev1alpha1.PodCliqueStatus{ - ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), - CurrentPodTemplateHash: ptr.To(rendered.Name + "-pod-template-hash"), - }, - } - clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name - clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name - clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) - for index, dependency := range clique.Spec.StartsAfter { - clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) + objects := []client.Object{pcs} + for _, plan := range plans { + groupIndex := slices.IndexFunc(pcs.Spec.Template.PodCliqueScalingGroupConfigs, func(config grovev1alpha1.PodCliqueScalingGroupConfig) bool { + return config.Name == plan.ScalingGroupTemplate + }) + require.NotEqual(t, -1, groupIndex) + groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[groupIndex].DeepCopy() + group := &grovev1alpha1.PodCliqueScalingGroup{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, + ObjectMeta: metav1.ObjectMeta{ + Name: plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: types.UID(plan.LPXScalingGroup + "-uid"), + Generation: 1, Labels: grovecommon.GetDefaultLabelsForPodCliqueSetManagedResources(pcs.Name), Annotations: groupTemplate.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: plan.Replicas, MinAvailable: groupTemplate.MinAvailable, + CliqueNames: groupTemplate.CliqueNames, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + }, + } + group.Labels[grovecommon.LabelPartOfKey] = pcs.Name + group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" + objects = append(objects, group) + + // Each Grove replica owns independent cliques. + for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { + parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} + for _, template := range pcs.Spec.Template.Cliques { + if !slices.Contains(groupTemplate.CliqueNames, template.Name) { + continue + } + rendered := template.DeepCopy() + name := grovecommon.GeneratePodCliqueName(parent, rendered.Name) + clique := &grovev1alpha1.PodClique{ + TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodClique"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, + Labels: rendered.Labels, Annotations: rendered.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }, + Spec: rendered.Spec, + Status: grovev1alpha1.PodCliqueStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), + CurrentPodTemplateHash: ptr.To(rendered.Name + "-pod-template-hash"), + }, + } + clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name + clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) + for index, dependency := range clique.Spec.StartsAfter { + clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) + } + objects = append(objects, clique) } - objects = append(objects, clique) } } return objects } +// createLPXTestScalingGroups emulates Grove creating each group at its template seed. +func createLPXTestScalingGroups(t *testing.T, kubeClient client.Client, pcs *grovev1alpha1.PodCliqueSet) { + t.Helper() + for _, config := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name), + Namespace: pcs.Namespace, + Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: ptr.Deref(config.Replicas, 1), MinAvailable: config.MinAvailable, CliqueNames: config.CliqueNames, + }, + } + require.NoError(t, kubeClient.Create(t.Context(), group)) + } +} + func createLPXTestObjects(t *testing.T, ctx context.Context, kubeClient client.Client, objects ...client.Object) { t.Helper() for _, object := range objects { @@ -2615,75 +2471,6 @@ func observedLPXTestPodCliqueSet( return pcs } -func TestLPXQueueEditSynchronizesPCSMetadata(t *testing.T) { - t.Log("Keep PCS queue metadata and clique templates consistent across DGD queue edits") - child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Labels = map[string]string{consts.KubeLabelKaiSchedulerQueue: "queue-a"} - for _, component := range lpx.Components(dgd) { - for _, role := range component.Roles { - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "queue-a") - } - } - r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, selected) - createLPXTestObjects(t, t.Context(), r.Client, objects...) - publishSelectedLPXForTest(t, t.Context(), r, child, selected) - key := client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)) - req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - _, err := r.Reconcile(t.Context(), req) - require.NoError(t, err) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(t.Context(), key, pcs)) - metav1.SetMetaDataLabel(&pcs.ObjectMeta, "external.example/label", "preserved") - metav1.SetMetaDataAnnotation(&pcs.ObjectMeta, "external.example/annotation", "preserved") - require.NoError(t, r.Update(t.Context(), pcs)) - beforeUID := pcs.UID - - t.Log("Apply new queue metadata and templates in place while preserving untracked metadata") - dgd.Spec.Labels[consts.KubeLabelKaiSchedulerQueue] = "queue-b" - for _, component := range lpx.Components(dgd) { - for _, role := range component.Roles { - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "queue-b") - } - } - updateTestDGD(t, r, child, dgd) - _, err = r.Reconcile(t.Context(), req) - require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), key, pcs)) - require.Equal(t, beforeUID, pcs.UID) - require.Equal(t, "preserved", pcs.Labels["external.example/label"]) - require.Equal(t, "preserved", pcs.Annotations["external.example/annotation"]) - require.Equal(t, "queue-b", pcs.Labels[consts.KubeLabelKaiSchedulerQueue]) - for _, clique := range pcs.Spec.Template.Cliques { - require.Equal(t, "queue-b", clique.Labels[consts.KubeLabelKaiSchedulerQueue]) - } - - t.Log("Remove an omitted managed queue from both PCS metadata and clique templates") - delete(dgd.Spec.Labels, consts.KubeLabelKaiSchedulerQueue) - for _, component := range lpx.Components(dgd) { - for _, role := range component.Roles { - delete(role.PodTemplate.Labels, consts.KubeLabelKaiSchedulerQueue) - } - } - updateTestDGD(t, r, child, dgd) - _, err = r.Reconcile(t.Context(), req) - require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), key, pcs)) - require.NotContains(t, pcs.Labels, consts.KubeLabelKaiSchedulerQueue) - for _, clique := range pcs.Spec.Template.Cliques { - require.NotContains(t, clique.Labels, consts.KubeLabelKaiSchedulerQueue) - } - - t.Log("An unchanged reconciliation does not write the PCS again") - beforeVersion := pcs.ResourceVersion - _, err = r.Reconcile(t.Context(), req) - require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), key, pcs)) - require.Equal(t, beforeVersion, pcs.ResourceVersion) - require.Equal(t, "preserved", pcs.Labels["external.example/label"]) - require.Equal(t, "preserved", pcs.Annotations["external.example/annotation"]) -} - func TestLPXEditBetweenFailureAndCleanupPreservesRetry(t *testing.T) { t.Log("Persist a deadline failure before the user edits the input to retry") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) @@ -2842,3 +2629,319 @@ func committedLPXTestPod(pod *corev1.Pod) *lpxv1alpha1.Committed { }}, } } + +func TestLPXWaitsForAllScalingGroups(t *testing.T) { + for _, tc := range []struct { + name string + group string + deleting bool + }{ + {name: "missing explicit group", group: "lpx"}, + {name: "missing external group", group: "external"}, + {name: "deleting group", group: "external", deleting: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Create a PCS with explicit and externally managed workloads") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Components[0].Replicas = ptr.To(int32(1)) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName, second.Replicas = "external", nil + dgd.Spec.Components = append(dgd.Spec.Components, *second) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + pcs, _, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + createLPXTestObjects(t, t.Context(), r.Client, materializeLPXTestPCS(t, child, pcs, plans["lpx"], plans["external"])...) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + before := pcs.DeepCopy() + + t.Log("Observe one configured group as missing or deleting") + base := r.Client + r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := delegated.List(ctx, list, opts...); err != nil { + return err + } + if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + for i := range groups.Items { + if groups.Items[i].Name == plans[tc.group].LPXScalingGroup { + if tc.deleting { + groups.Items[i].DeletionTimestamp = ptr.To(metav1.Now()) + } else { + groups.Items = slices.Delete(groups.Items, i, i+1) + } + break + } + } + } + return nil + }, + }) + r.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: errors.New("workload resolution must wait for all groups")} + + t.Log("Return Pending before resolving workloads, updating the PCS, or publishing requests") + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Zero(t, result, "the PCSG watch resumes reconciliation") + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, ready) + require.Equal(t, v1alpha1.LPXReadyReasonPending, ready.Reason) + require.Equal(t, "Waiting for all LPX scaling groups", ready.Message) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.Equal(t, before, pcs) + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Empty(t, requests) + + t.Log("Observing every group allows PCS synchronization and request publication") + r.Client, r.modelRegistry = base, registry + for range 2 { + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + } + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 2) + }) + } +} + +func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { + t.Log("Author two independent workloads with explicit and external replica ownership") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Components[0].Replicas = ptr.To(int32(2)) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName, second.Replicas = "second-engine", nil + dgd.Spec.Components = append(dgd.Spec.Components, *second) + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + require.Len(t, workloads, 2) + pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + require.Len(t, resources, 4) + names := make(map[string]bool) + for _, resource := range resources { + require.NotContains(t, names, resource.GetName()) + names[resource.GetName()] = true + } + + t.Log("Reordering authored components preserves rendering and immutable identities") + reordered := dgd.DeepCopy() + slices.Reverse(reordered.Spec.Components) + reorderedWorkloads, reorderedPlans, err := r.resolveWorkloads(t.Context(), child, reordered) + require.NoError(t, err) + reorderedPCS, reorderedResources, err := r.renderPodCliqueSet(t.Context(), child, reordered, reorderedWorkloads, reorderedPlans) + require.NoError(t, err) + require.Equal(t, pcs, reorderedPCS) + require.Equal(t, resources, reorderedResources) + require.Equal(t, plans, reorderedPlans) + + t.Log("Observe two explicit replicas and three externally managed replicas in Grove") + firstPlan, secondPlan := plans["lpx"], plans["second-engine"] + secondPlan.Replicas = 3 + objects := materializeLPXTestPCS(t, child, pcs, firstPlan, secondPlan) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + } + } + createLPXTestObjects(t, t.Context(), r.Client, objects...) + + t.Log("Record request publication order across both workloads") + var publishedTargets []lpxv1alpha1.PodCliqueScalingGroupReference + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { + publishedTargets = append(publishedTargets, *request.Spec.MaterializationTarget.PodCliqueScalingGroupRef) + } + return delegated.Create(ctx, object, opts...) + }, + }) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + t.Log("Observe the synchronized PCS before publishing scheduler requests") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 5) + require.Equal(t, []lpxv1alpha1.PodCliqueScalingGroupReference{ + {Name: firstPlan.LPXScalingGroup, ReplicaIndex: 0}, + {Name: firstPlan.LPXScalingGroup, ReplicaIndex: 1}, + {Name: secondPlan.LPXScalingGroup, ReplicaIndex: 0}, + {Name: secondPlan.LPXScalingGroup, ReplicaIndex: 1}, + {Name: secondPlan.LPXScalingGroup, ReplicaIndex: 2}, + }, publishedTargets) + original := make(map[string]types.UID) + for i := range requests { + request := &requests[i] + original[request.Name] = request.UID + require.Equal(t, "default", request.Annotations[pipelineRequestModelAnnotation]) + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(t.Context(), request)) + } + + t.Log("Publish disjoint Services and aggregate both workloads' readiness") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + for groupName, plan := range plans { + status := child.Status.Components[workloads[groupName].ServingComponentName()] + require.True(t, status.Ready) + require.Equal(t, plan.Replicas, status.Replicas) + require.Equal(t, []string{plan.LPXScalingGroup}, status.ComponentNames) + service := &corev1.Service{} + require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: plan.ResourcePrefix + "-serve"}, service)) + require.Equal(t, workloads[groupName].ServingComponentName(), service.Spec.Selector[consts.KubeLabelDynamoComponent]) + require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) + } + + t.Log("A missing role in the second workload leaves the first workload's readiness intact") + clique := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: secondPlan.ConductorClique}, clique)) + clique.Status.ReadyReplicas = 0 + require.NoError(t, r.Update(t.Context(), clique)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.True(t, child.Status.Components["lpx"].Ready) + require.False(t, child.Status.Components["second-engine"].Ready) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + + t.Log("Scale down the explicit workload without replacing the PCS or its neighbor's requests") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + dgd.Spec.Components[0].Replicas = ptr.To(int32(1)) + require.NoError(t, r.Update(t.Context(), dgd)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + require.NoError(t, r.Update(t.Context(), child)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + groups, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) + require.NoError(t, err) + require.EqualValues(t, 1, groups[firstPlan.LPXScalingGroup].Spec.Replicas) + require.EqualValues(t, 3, groups[secondPlan.LPXScalingGroup].Spec.Replicas) + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 4) + for _, request := range requests { + require.Equal(t, original[request.Name], request.UID) + } + observedPCS, err := getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + require.Equal(t, pcs.UID, observedPCS.UID) + require.Equal(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, observedPCS.Spec.Template.PodCliqueScalingGroupConfigs) + + t.Log("External scale-out publishes only the new ordinal and preserves the explicit workload") + group := groups[secondPlan.LPXScalingGroup] + group.Spec.Replicas = 4 + require.NoError(t, r.Update(t.Context(), group)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 5) + for _, request := range requests { + if uid, exists := original[request.Name]; exists { + require.Equal(t, uid, request.UID) + } else { + require.Equal(t, secondPlan.LPXScalingGroup, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name) + require.EqualValues(t, 3, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex) + } + } +} + +func TestIndependentLPXDeadlineCleanup(t *testing.T) { + for _, tc := range []struct { + name string + firstExpired int64 + secondExternal bool + secondHealthy bool + wantFirst int32 + wantSecond int32 + wantRequests int + }{ + {name: "interior failure leaves the other engine's suffix removable", firstExpired: 0, wantFirst: 3, wantSecond: 1, wantRequests: 4}, + {name: "healthy neighbor survives failure cleanup", firstExpired: 2, secondHealthy: true, wantFirst: 2, wantSecond: 2, wantRequests: 4}, + {name: "independent expired suffixes", firstExpired: 2, wantFirst: 2, wantSecond: 1, wantRequests: 3}, + {name: "external engine retains capacity ownership", firstExpired: 2, secondExternal: true, wantFirst: 2, wantSecond: 2, wantRequests: 3}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Publish two workloads with different replica axes and active scheduling clocks") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Spec.Components[0].Replicas = ptr.To(int32(3)) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName, second.Replicas = "timeout-engine", ptr.To(int32(2)) + if tc.secondExternal { + second.Replicas = nil + } + dgd.Spec.Components = append(dgd.Spec.Components, *second) + dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(60))} + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + pcs, _, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + firstPlan, secondPlan := plans["lpx"], plans["timeout-engine"] + secondPlan.Replicas = 2 + createLPXTestObjects(t, t.Context(), r.Client, materializeLPXTestPCS(t, child, pcs, firstPlan, secondPlan)...) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + t.Log("Observe the synchronized PCS before publishing scheduler requests") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 5) + for i := range requests { + request := &requests[i] + target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef + phase := lpxv1alpha1.RequestPhaseBound + if target.Name == firstPlan.LPXScalingGroup && target.ReplicaIndex == tc.firstExpired || !tc.secondHealthy && target.Name == secondPlan.LPXScalingGroup && target.ReplicaIndex == 1 { + phase = lpxv1alpha1.RequestPhasePending + } + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now().Add(-2*time.Minute), phase).Status + require.NoError(t, r.Update(t.Context(), request)) + } + + t.Log("Persist scheduling failure before deleting either workload's requests") + result, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 5) + + t.Log("Apply each workload's suffix rule and capacity ownership independently") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + groups, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) + require.NoError(t, err) + require.Equal(t, tc.wantFirst, groups[firstPlan.LPXScalingGroup].Spec.Replicas) + require.Equal(t, tc.wantSecond, groups[secondPlan.LPXScalingGroup].Spec.Replicas) + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, tc.wantRequests) + + t.Log("A failed generation must not republish either workload's retired requests") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + again, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Equal(t, requests, again) + }) + } +} diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go index b212a992a7aa..601f395fc298 100644 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ b/deploy/operator/internal/controller/lpx/eviction.go @@ -10,7 +10,7 @@ import ( "fmt" "slices" - v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" @@ -27,8 +27,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" ) const ( @@ -85,7 +85,7 @@ func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context // A late runtime table observation must wake disrupted Agents using that exact table. var pods corev1.PodList if err := r.List(ctx, &pods, client.InNamespace(config.GetNamespace()), client.MatchingLabels{ - consts.KubeLabelDynamoComponentType: consts.ComponentTypeLPX, + commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, }); err != nil { log.FromContext(ctx).Error(err, "list LPU eviction triggers for runtime ConfigMap") return nil @@ -95,9 +95,8 @@ func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context var requests []ctrl.Request for i := range pods.Items { pod := &pods.Items[i] - root := pod.Labels[grovecommon.LabelPartOfKey] - hash := pod.Annotations[consts.AnnotationExtraResourcesHash] - if r.isTrigger(pod) && root != "" && hash != "" && lpx.LPUConfigMapName(root, hash) == config.GetName() { + name, err := dynamolpx.LPUAgentConfigMapName(pod) + if r.isTrigger(pod) && err == nil && name == config.GetName() { requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) } } @@ -114,11 +113,11 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: owner.Name}, clique); err != nil { return nil, err } - if clique.UID != owner.UID || !lpx.OwnsPodClique(ctx, r.Client, clique) { + if clique.UID != owner.UID || !dynamolpx.OwnsPodClique(ctx, r.Client, clique) { return nil, fmt.Errorf("cannot verify ownership of LPU eviction trigger pod %s/%s", trigger.Namespace, trigger.Name) } - mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[lpx.WorkloadModeAnnotation]) + mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[dynamolpx.WorkloadModeAnnotation]) if mode == "" { return nil, nil } @@ -130,13 +129,16 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor case lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly: case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: // The Pod's content-addressed runtime table survives native template updates. - root := trigger.Labels[grovecommon.LabelPartOfKey] - hash := trigger.Annotations[consts.AnnotationExtraResourcesHash] - if root == "" || hash == "" { + name, err := dynamolpx.LPUAgentConfigMapName(trigger) + if err != nil { + return nil, err + } + hash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] + if hash == "" { return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap identity", trigger.Namespace, trigger.Name) } var config corev1.ConfigMap - key := client.ObjectKey{Namespace: trigger.Namespace, Name: lpx.LPUConfigMapName(root, hash)} + key := client.ObjectKey{Namespace: trigger.Namespace, Name: name} if err := r.Get(ctx, key, &config); err != nil { return nil, fmt.Errorf("read LPU-GPU eviction runtime ConfigMap: %w", err) } @@ -146,14 +148,14 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor // Bind the table to the verified deployment and the Pod's full content hash. configOwner := metav1.GetControllerOf(&config) - if configOwner == nil || configOwner.APIVersion != v1alpha1.GroupVersion.String() || - configOwner.Kind != v1alpha1.LPXGraphDeploymentGVK.Kind || - configOwner.Name != clique.Annotations[lpx.DeploymentNameAnnotation] || - string(configOwner.UID) != clique.Annotations[lpx.DeploymentUIDAnnotation] || - lpx.LPUConfigMapHash(&config) != hash { + if configOwner == nil || configOwner.APIVersion != nvidiacomv1alpha1.GroupVersion.String() || + configOwner.Kind != nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind || + configOwner.Name != clique.Annotations[dynamolpx.DeploymentNameAnnotation] || + string(configOwner.UID) != clique.Annotations[dynamolpx.DeploymentUIDAnnotation] || + dynamolpx.LPUConfigMapHash(&config) != hash { return nil, fmt.Errorf("cannot authenticate LPU-GPU eviction runtime ConfigMap %s", key) } - triggerPartition, err := lpx.LPUAgentRuntimePartition(trigger, &config) + triggerPartition, err := dynamolpx.LPUAgentRuntimePartition(trigger, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction trigger pod %s/%s: %w", trigger.Namespace, trigger.Name, err) } @@ -161,7 +163,7 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor // Select the full runtime partition before authorizing any sibling deletion. selected := pods[:0] for _, pod := range pods { - partition, err := lpx.LPUAgentRuntimePartition(&pod, &config) + partition, err := dynamolpx.LPUAgentRuntimePartition(&pod, &config) if err != nil { return nil, fmt.Errorf("parse LPU-GPU eviction candidate pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -178,8 +180,8 @@ func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *cor func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { labels := trigger.GetLabels() - dgdName := labels[consts.KubeLabelDynamoGraphDeploymentName] - component := labels[consts.KubeLabelDynamoComponent] + dgdName := labels[commonconsts.KubeLabelDynamoGraphDeploymentName] + component := labels[commonconsts.KubeLabelDynamoComponent] pcsg := labels[grovecommon.LabelPodCliqueScalingGroup] pcsgReplica := labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] if dgdName == "" || component == "" || pcsg == "" || pcsgReplica == "" { @@ -188,8 +190,8 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev var pods corev1.PodList if err := r.List(ctx, &pods, client.InNamespace(trigger.Namespace), client.MatchingLabels{ - consts.KubeLabelDynamoGraphDeploymentName: dgdName, - consts.KubeLabelDynamoComponent: component, + commonconsts.KubeLabelDynamoGraphDeploymentName: dgdName, + commonconsts.KubeLabelDynamoComponent: component, grovecommon.LabelPodCliqueScalingGroup: pcsg, grovecommon.LabelPodCliqueScalingGroupReplicaIndex: pcsgReplica, }); err != nil { @@ -207,7 +209,7 @@ func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev owner.APIVersion != candidateOwner.APIVersion || owner.Kind != candidateOwner.Kind || owner.Name != candidateOwner.Name || owner.UID != candidateOwner.UID || pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || - pod.Annotations[consts.AnnotationExtraResourcesHash] != trigger.Annotations[consts.AnnotationExtraResourcesHash] || + pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] || model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model }), nil } @@ -243,13 +245,13 @@ func (r *lpuEvictionReconciler) deletePods(ctx context.Context, trigger *corev1. } func (r *lpuEvictionReconciler) isLPUAgentPod(pod *corev1.Pod) bool { - componentType := pod.Labels[consts.KubeLabelDynamoComponentType] - return componentType == consts.ComponentTypeLPX && + componentType := pod.Labels[commonconsts.KubeLabelDynamoComponentType] + return componentType == commonconsts.ComponentTypeLPX && pod.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent } func (r *lpuEvictionReconciler) isTrigger(pod *corev1.Pod) bool { - if pod.Spec.SchedulerName != lpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { + if pod.Spec.SchedulerName != dynamolpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { return false } return slices.ContainsFunc(pod.Status.Conditions, func(condition corev1.PodCondition) bool { diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index 7a461a853316..b1f65b914c16 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -10,7 +10,7 @@ import ( "sort" "testing" - v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -18,16 +18,18 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" - consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" ) @@ -42,15 +44,18 @@ const ( lpuEvictionTestConfigHash = "bf665efcf39a0c79fa97ea2c05664ca7235dd525aced22f47da0633d729d6171" ) -func newLPUEvictionReconciler(t *testing.T, objs ...client.Object) (*lpuEvictionReconciler, client.Client) { - t.Helper() +func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = nvidiacomv1alpha1.AddToScheme(scheme) + _ = grovev1alpha1.AddToScheme(scheme) - deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ + deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestLGD, Namespace: lpuEvictionTestNamespace, UID: "test-lgd-uid", }} pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestPCS, Namespace: lpuEvictionTestNamespace, UID: "test-pcs-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }} group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestPCSG, Namespace: lpuEvictionTestNamespace, UID: "test-pcsg-uid", @@ -59,18 +64,18 @@ func newLPUEvictionReconciler(t *testing.T, objs ...client.Object) (*lpuEviction clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ Name: "agents", Namespace: lpuEvictionTestNamespace, UID: "agents-uid", Annotations: map[string]string{ - lpx.DeploymentNameAnnotation: lpuEvictionTestLGD, - lpx.DeploymentUIDAnnotation: "test-lgd-uid", - lpx.WorkloadDigestAnnotation: "test-digest", - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + dynamolpx.DeploymentNameAnnotation: lpuEvictionTestLGD, + dynamolpx.DeploymentUIDAnnotation: "test-lgd-uid", + dynamolpx.WorkloadDigestAnnotation: "test-digest", + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, }} config := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, v1alpha1.LPXGraphDeploymentGVK)}, + Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }, Immutable: ptr.To(true), Data: map[string]string{ @@ -78,7 +83,14 @@ func newLPUEvictionReconciler(t *testing.T, objs ...client.Object) (*lpuEviction "partition_models": lpuEvictionTestModel + "\n" + lpuEvictionTestModel, }, } - c := newLPXTestClient(t, append([]client.Object{deployment, pcs, group, clique, config}, objs...)...) + cb := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&corev1.Pod{}). + WithObjects(deployment, pcs, group, clique, config) + for _, o := range objs { + cb = cb.WithObjects(o) + } + c := cb.Build() return &lpuEvictionReconciler{Client: c, Recorder: events.NewFakeRecorder(16)}, c } @@ -92,19 +104,26 @@ func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting OwnerReferences: []metav1.OwnerReference{{APIVersion: "grove.io/v1alpha1", Kind: "PodClique", Name: "agents", UID: "agents-uid", Controller: ptr.To(true)}}, Labels: map[string]string{ grovecommon.LabelPartOfKey: lpuEvictionTestPCS, - consts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, - consts.KubeLabelDynamoComponent: lpuEvictionTestComponent, - consts.KubeLabelDynamoComponentType: consts.ComponentTypeLPX, + commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, + commonconsts.KubeLabelDynamoComponent: lpuEvictionTestComponent, + commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, grovecommon.LabelPodCliqueScalingGroup: lpuEvictionTestPCSG, grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, }, Annotations: map[string]string{ - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - lpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), - consts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, + lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, + dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), + commonconsts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, }, }, - Spec: corev1.PodSpec{SchedulerName: lpx.SchedulerName}, + Spec: corev1.PodSpec{ + SchedulerName: dynamolpx.SchedulerName, + Volumes: []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{ + Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), + }}, + }}}, + }, Status: corev1.PodStatus{Phase: corev1.PodRunning}, } if podIndex != "" { @@ -168,28 +187,30 @@ func lpuEvictionPodNames(pods []corev1.Pod) []string { return names } -func TestLPUEvictionRejectsMissingPodIndex(t *testing.T) { - t.Log("Observe a hybrid Agent without its canonical Grove index") - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, _ := newLPUEvictionReconciler(t, trigger, sibling) +func TestLPUEviction_PodsForTrigger(t *testing.T) { + t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { + t.Log("Observe a hybrid Agent without its canonical Grove index") + trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + r, _ := newLPUEvictionReconciler(trigger, sibling) - t.Log("Reject the incomplete identity before selecting any deletion") - pods, err := r.podsForTrigger(context.Background(), trigger) + t.Log("Reject the incomplete identity before selecting any deletion") + pods, err := r.podsForTrigger(context.Background(), trigger) - require.Error(t, err) - assert.Nil(t, pods) + require.Error(t, err) + assert.Nil(t, pods) + }) } func TestLPUEvictionPreservesNativeReplacement(t *testing.T) { t.Log("Keep the original snapshot while a new native clique coexists") trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) replacement := newLPUEvictionPod("agent-1", "0", "1", false, "") replacement.OwnerReferences[0].UID = "replacement-clique" - r, _ := newLPUEvictionReconciler(t, trigger, replacement) + r, _ := newLPUEvictionReconciler(trigger, replacement) pods, err := r.podsForTrigger(t.Context(), trigger) require.NoError(t, err) require.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) @@ -203,7 +224,7 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { t.Run(string(mode), func(t *testing.T) { t.Log("OnDelete can retain old Pods and replace others under the same clique") trigger := newLPUEvictionPod("agent-old-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) sibling := newLPUEvictionPod("agent-old-1", "0", "1", false, "") replacement := newLPUEvictionPod("agent-new-1", "0", "1", false, "") trigger.Labels[grovecommon.LabelPodTemplateHash] = "old-image" @@ -213,7 +234,7 @@ func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { sibling.Spec.Containers = trigger.Spec.Containers replacement.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:new"}} require.Equal(t, trigger.OwnerReferences, replacement.OwnerReferences) - r, c := newLPUEvictionReconciler(t, trigger, sibling, replacement) + r, c := newLPUEvictionReconciler(trigger, sibling, replacement) t.Log("Evict the old revision's sibling without deleting the replacement") reconcileLPUEviction(t, r, trigger.Name) @@ -241,7 +262,7 @@ func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { agent := newLPUEvictionPod("agent-1", "0", "1", false, "") otherReplica := newLPUEvictionPod("agent-other-replica", "1", "0", false, "") - r, c := newLPUEvictionReconciler(t, + r, c := newLPUEvictionReconciler( trigger, conductor, agent, @@ -280,11 +301,11 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { t.Log("Observe an ownership snapshot with a stale LGD UID") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) clique := &grovev1alpha1.PodClique{} require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: "agents"}, clique)) - clique.Annotations[lpx.DeploymentUIDAnnotation] = "stale-lgd-uid" + clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "stale-lgd-uid" require.NoError(t, c.Update(t.Context(), clique)) t.Log("Return a retryable error without deleting the sibling") @@ -293,7 +314,7 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { assert.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) t.Log("Evict the sibling when the owner snapshot catches up") - clique.Annotations[lpx.DeploymentUIDAnnotation] = "test-lgd-uid" + clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "test-lgd-uid" require.NoError(t, c.Update(t.Context(), clique)) reconcileLPUEviction(t, r, trigger.Name) assert.Empty(t, remainingLPUEvictionPods(t, c)) @@ -302,7 +323,7 @@ func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { current := &corev1.Pod{} @@ -329,7 +350,7 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" noPartition := newLPUOnlyConductor() otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") t.Log("Consume the real shared-cache Pod projection without deprecated row annotations") @@ -337,7 +358,7 @@ func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { podcache.Project(pod) } - r, c := newLPUEvictionReconciler(t, + r, c := newLPUEvictionReconciler( trigger, samePartition, otherPartition, @@ -364,6 +385,7 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { name string candidateIndex string missingHash bool + missingVolume bool missingModel bool missingConfig bool mutableConfig bool @@ -375,6 +397,7 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { {name: "invalid candidate index", candidateIndex: "bad"}, {name: "out of range candidate", candidateIndex: "3"}, {name: "missing config identity", candidateIndex: "2", missingHash: true}, + {name: "missing runtime volume", candidateIndex: "2", missingVolume: true}, {name: "missing model", candidateIndex: "2", missingModel: true}, {name: "config not observed", candidateIndex: "2", missingConfig: true}, {name: "mutable config", candidateIndex: "2", mutableConfig: true}, @@ -386,20 +409,23 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Observe a valid sibling before another candidate whose identity cannot authorize deletion") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) + if test.missingVolume { + trigger.Spec.Volumes = nil + } if test.missingHash { - delete(trigger.Annotations, consts.AnnotationExtraResourcesHash) + delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) } if test.missingModel { delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) } sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") candidate := newLPUEvictionPod("agent-2", "0", test.candidateIndex, false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling, candidate) + r, c := newLPUEvictionReconciler(trigger, sibling, candidate) config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} require.NoError(t, c.Get(t.Context(), key, config)) - require.Equal(t, lpuEvictionTestConfigHash, lpx.LPUConfigMapHash(config)) + require.Equal(t, lpuEvictionTestConfigHash, dynamolpx.LPUConfigMapHash(config)) if test.missingConfig { require.NoError(t, c.Delete(t.Context(), config)) } @@ -435,11 +461,11 @@ func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { func TestLPUEvictionConfigMapObservationWakesTrigger(t *testing.T) { t.Log("Observe a hybrid disruption before its immutable runtime table reaches the cache") trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} + key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} require.NoError(t, c.Get(t.Context(), key, config)) require.NoError(t, c.Delete(t.Context(), config)) _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) @@ -464,13 +490,13 @@ func TestLPUEvictionLPUOnlyNeedsNoRuntimePartitionTable(t *testing.T) { t.Run(string(mode), func(t *testing.T) { t.Log("Observe an LPU-only disruption without hybrid runtime partition metadata") trigger := newLPUEvictionPod("agent-0", "0", "", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[lpx.WorkloadModeAnnotation] = string(mode) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) sibling := newLPUEvictionPod("agent-1", "0", "", false, "") - delete(trigger.Annotations, consts.AnnotationExtraResourcesHash) - delete(sibling.Annotations, consts.AnnotationExtraResourcesHash) - r, c := newLPUEvictionReconciler(t, trigger, sibling) + delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) + delete(sibling.Annotations, commonconsts.AnnotationExtraResourcesHash) + r, c := newLPUEvictionReconciler(trigger, sibling) config := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Namespace: lpuEvictionTestNamespace, Name: lpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), + Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), }} require.NoError(t, c.Delete(t.Context(), config)) @@ -485,7 +511,7 @@ func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t trigger := newLPUEvictionPod("agent-0", "0", "0", false, podDisruptionReasonTaintManagerDeletion) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) result := reconcileLPUEviction(t, r, "agent-0") assert.Equal(t, ctrl.Result{}, result) @@ -535,16 +561,16 @@ func TestLPUEviction_Noops(t *testing.T) { trigger.Status.Phase = tt.phase trigger.Status.Reason = tt.statusReason if tt.nonLPU { - trigger.Labels[consts.KubeLabelDynamoComponentType] = consts.ComponentTypeWorker + trigger.Labels[commonconsts.KubeLabelDynamoComponentType] = commonconsts.ComponentTypeWorker } if tt.conductor { trigger.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor } if tt.missingMode { - delete(trigger.Annotations, lpx.WorkloadModeAnnotation) + delete(trigger.Annotations, dynamolpx.WorkloadModeAnnotation) } sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) t.Log("Reconcile without deleting the surviving Agent") result := reconcileLPUEviction(t, r, trigger.Name) @@ -559,7 +585,7 @@ func TestLPUEviction_MissingGroveLabelsReturnsError(t *testing.T) { delete(trigger.Labels, grovecommon.LabelPodCliqueScalingGroup) sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(t, trigger, sibling) + r, c := newLPUEvictionReconciler(trigger, sibling) result, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: trigger.Name, Namespace: lpuEvictionTestNamespace}, @@ -568,3 +594,43 @@ func TestLPUEviction_MissingGroveLabelsReturnsError(t *testing.T) { assert.Equal(t, ctrl.Result{}, result) assert.Equal(t, []string{"agent-1"}, remainingLPUEvictionPods(t, c)) } + +func TestLPUEvictionUsesGroupScopedPodConfigMap(t *testing.T) { + for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid} { + t.Run(string(mode), func(t *testing.T) { + t.Log("Observe two workloads sharing a PCS and content hash but using distinct runtime tables") + trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonEvictionAPI) + trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) + sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") + otherPartition := newLPUEvictionPod("agent-2", "0", "2", false, "") + otherWorkload := newLPUEvictionPod("other-agent-1", "0", "1", false, "") + otherWorkload.Labels[grovecommon.LabelPodCliqueScalingGroup] = "other-workload" + groupConfig := dynamolpx.LPUConfigMapName(lpuEvictionTestPCS+"-decode", lpuEvictionTestConfigHash) + for _, pod := range []*corev1.Pod{trigger, sibling, otherPartition} { + pod.Spec.Volumes[0].ConfigMap.Name = groupConfig + } + otherWorkload.Spec.Volumes[0].ConfigMap.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS+"-other", lpuEvictionTestConfigHash) + for _, pod := range []*corev1.Pod{trigger, sibling, otherPartition, otherWorkload} { + podcache.Project(pod) + } + r, kube := newLPUEvictionReconciler(trigger, sibling, otherPartition, otherWorkload) + config := &corev1.ConfigMap{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKey{ + Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), + }, config)) + + t.Log("An old unscoped table must neither wake nor authorize eviction of this workload") + require.Empty(t, r.evictionRequestsForConfigMap(t.Context(), config)) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) + require.ErrorContains(t, err, groupConfig) + require.Equal(t, []string{sibling.Name, otherPartition.Name, otherWorkload.Name}, remainingLPUEvictionPods(t, kube)) + + t.Log("The exact group-scoped table wakes the trigger and evicts only its partition") + config.Name, config.ResourceVersion, config.UID = groupConfig, "", "" + require.NoError(t, kube.Create(t.Context(), config)) + require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(trigger)}}, r.evictionRequestsForConfigMap(t.Context(), config)) + reconcileLPUEviction(t, r, trigger.Name) + require.Equal(t, []string{otherPartition.Name, otherWorkload.Name}, remainingLPUEvictionPods(t, kube)) + }) + } +} diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index f44fd79b92d1..0371651f6b75 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -48,22 +48,21 @@ func (r *graphReconciler) getPipelineRequests(ctx context.Context, pcs *grovev1a // collects missing requests in replica/model order in the same pass. An immutable // mismatch returns nil collections and true: the caller must replace the PCS. // Pointer inputs are non-nil; deployment has a validated DGD controller owner. -// currentRequests contains owned observations and may be empty. +// currentRequests contains owned observations indexed by name and may be nil. // Inputs are not mutated; owner references are added only on publication. func resolvePipelineRequests( deployment *v1alpha1.LPXGraphDeployment, - currentRequests []lpxv1alpha1.LPUPipelineRequest, - workload *lpx.SelectedWorkload, + currentRequests map[string]*lpxv1alpha1.LPUPipelineRequest, + workload *lpx.Workload, plan *lpx.MaterializationPlan, ) (map[string]*lpxv1alpha1.LPUPipelineRequest, []*lpxv1alpha1.LPUPipelineRequest, bool) { // The reconcile boundary already validated the DGD owner; rendering uses that identity. dgdOwner := metav1.GetControllerOf(deployment) - // Index the observed requests once, preserving their scheduler-owned status. - currentRequestsByName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(currentRequests)) - for index := range currentRequests { - request := ¤tRequests[index] - currentRequestsByName[request.Name] = request + // Only multiple workloads need a group identity; sole workloads retain existing LPR names. + groupName := "" + if plan.ResourcePrefix != plan.PodCliqueSetName { + groupName = workload.ServingComponentName() } // Model projections are already ordered (default, or draft0..draft7 then target). @@ -75,7 +74,7 @@ func resolvePipelineRequests( replicaPlan := plan.ForReplica(replica) for index, projection := range projections { - digest := pipelineRequestIdentityDigest(deployment.Namespace, deployment.Name, deployment.UID, projection.Model(), replica) + digest := pipelineRequestIdentityDigest(deployment.Namespace, deployment.Name, deployment.UID, groupName, projection.Model(), replica) request := &lpxv1alpha1.LPUPipelineRequest{ ObjectMeta: metav1.ObjectMeta{ @@ -96,7 +95,7 @@ func resolvePipelineRequests( Spec: projection.RequestSpec(replicaPlan, replicaPlan.Agents[index].CliqueName), } - if currentRequest, exists := currentRequestsByName[request.Name]; exists { + if currentRequest, exists := currentRequests[request.Name]; exists { if !pipelineRequestMatches(currentRequest, request) { return nil, nil, true } @@ -112,12 +111,14 @@ func resolvePipelineRequests( return requests, missing, false } -// pipelineRequestIdentityDigest identifies one model at one engine ordinal. +// pipelineRequestIdentityDigest identifies a model and replica within an optional group. +// An empty group preserves the established single-workload identity. // InputRevision and generation are deliberately absent: there is no publication batch. func pipelineRequestIdentityDigest( namespace string, name string, uid types.UID, + group string, model string, replicaIndex int32, ) string { @@ -125,6 +126,9 @@ func pipelineRequestIdentityDigest( writeIdentityHashField(h, "namespace", namespace) writeIdentityHashField(h, "name", name) writeIdentityHashField(h, "uid", string(uid)) + if group != "" { + writeIdentityHashField(h, "group", group) + } writeIdentityHashField(h, "model", model) if replicaIndex > 0 { writeIdentityHashField(h, "group-replica", strconv.FormatInt(int64(replicaIndex), 10)) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 64e4bc80b579..92bbad5b4953 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -8,7 +8,9 @@ package lpx import ( "context" "fmt" + "maps" "math" + "slices" "time" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -30,9 +32,8 @@ func pipelineRequestDeadlineSeconds(dgd *v1beta1.DynamoGraphDeployment) *int64 { } // pipelineRequestDeadlines examines already-selected, owned requests without I/O. -// Missing schedulingStartedAt means the scheduler has not started a cycle yet. -// A surviving LPR can return to Pending; its scheduler-supplied start resets the clock. -// Creation time and managed fields cannot identify that new scheduling cycle. +// Creation time bounds the initial wait for the scheduler. On reused requests, +// the scheduler-supplied start takes precedence for later scheduling cycles. func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineRequest, seconds *int64) (expired []*lpxv1alpha1.LPUPipelineRequest, next time.Time) { if seconds == nil { return nil, time.Time{} @@ -42,10 +43,14 @@ func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineReques if !request.DeletionTimestamp.IsZero() || isPipelineRequestDeadlineExempt(request) { continue } - started, known := pipelineRequestSchedulingStartedAt(request) - if !known { + + // Desired requests awaiting publication have neither timestamp yet. + started := pipelineRequestSchedulingStartedAt(request) + if started.IsZero() { continue } + + // Track each request's expiration and the earliest remaining deadline. deadline := started.Add(time.Duration(*seconds) * time.Second) if !now.Before(deadline) { expired = append(expired, request) @@ -56,13 +61,13 @@ func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineReques return expired, next } -// pipelineRequestSchedulingStartedAt requires a nonnil request and reports whether the -// scheduler published a usable start for its current scheduling cycle. -func pipelineRequestSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) (time.Time, bool) { +// pipelineRequestSchedulingStartedAt returns the scheduler's nonzero cycle start, +// or creation time while waiting for the scheduler. request must be non-nil. +func pipelineRequestSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) time.Time { if request.Status == nil || request.Status.SchedulingStartedAt == nil || request.Status.SchedulingStartedAt.IsZero() { - return time.Time{}, false + return request.CreationTimestamp.Time } - return request.Status.SchedulingStartedAt.Time, true + return request.Status.SchedulingStartedAt.Time } // isPipelineRequestDeadlineExempt recognizes current receipts outside the scheduling timer. @@ -115,38 +120,49 @@ func requeueForPipelineRequestDeadline( return result } -// reconcilePipelineRequestDeadline records expiry before cleanup and blocks republication -// for the failed generation. Call only for expired requests or a current SchedulingFailed -// condition. deployment and requests are validated observations. A nil pcsg still -// permits recording failure, but cleanup must wait for an observed scaling group. -func (r *graphReconciler) reconcilePipelineRequestDeadline( +// reconcileSchedulingFailure records graph-wide failure before deadline cleanup. +// Each workload removes only its own expired suffix; omitted replicas retain +// external capacity ownership. All configured groups have been observed. +// deployment and requests are validated observations; replica counts are keyed by PCSG name. +func (r *graphReconciler) reconcileSchedulingFailure( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, - pcsg *grovev1alpha1.PodCliqueScalingGroup, + groups map[string]*grovev1alpha1.PodCliqueScalingGroup, + explicitReplicas map[string]*int32, requests []lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, - manageReplicas bool, ) (ctrl.Result, error) { - // Persist a failure covering every expired scheduling cycle before deleting requests. - if len(expired) > 0 { - if !schedulingFailureCoversPipelineRequests(deployment, expired) { - setSchedulingFailedCondition(deployment, true) - // Status-only LPXGD updates are filtered, so explicitly observe the persisted failure. - return ctrl.Result{RequeueAfter: time.Nanosecond}, nil - } - setSchedulingFailedCondition(deployment, false) - return r.reconcileExpiredPipelineRequests(ctx, pcsg, requests, expired, manageReplicas) + // Persist evidence covering every expired cycle before deleting any expired request. + if len(expired) > 0 && !schedulingFailureCoversPipelineRequests(deployment, expired) { + setSchedulingFailedCondition(deployment, true) + return ctrl.Result{RequeueAfter: time.Nanosecond}, nil } - - // A failed generation must not recreate requests removed by its deadline. setSchedulingFailedCondition(deployment, false) + + // Partition both collections by their actual target; ordinals belong to one workload. + expiredByGroup := make(map[string][]*lpxv1alpha1.LPUPipelineRequest) + for _, request := range expired { + name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name + expiredByGroup[name] = append(expiredByGroup[name], request) + } + requestsByGroup := make(map[string][]lpxv1alpha1.LPUPipelineRequest) + for _, request := range requests { + name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name + requestsByGroup[name] = append(requestsByGroup[name], request) + } + + // An interior failure in one workload does not prevent another workload's suffix cleanup. + for _, name := range slices.Sorted(maps.Keys(expiredByGroup)) { + if err := r.reconcileExpiredPipelineRequests(ctx, groups[name], requestsByGroup[name], expiredByGroup[name], explicitReplicas[name] != nil); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{}, nil } // reconcileExpiredPipelineRequests removes only complete expired trailing replicas. -// A nil pcsg defers cleanup: the live replica count is unknown, not zero. Non-nil -// pcsg and all requests are already validated against their PCS. No PodCliques -// are deleted for interior failures. +// pcsg must be non-nil; it and all requests are already validated against their PCS. +// No PodCliques are deleted for interior failures. // manageReplicas is false for omitted DGD replicas: external scaling retains capacity ownership. func (r *graphReconciler) reconcileExpiredPipelineRequests( ctx context.Context, @@ -154,26 +170,24 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( requests []lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, manageReplicas bool, -) (ctrl.Result, error) { - // Observe capacity before deciding which ordinals are safe to remove; the PCSG watch resumes cleanup. - if pcsg == nil { - return ctrl.Result{}, nil - } +) error { + // Use the observed capacity to decide which ordinals are safe to remove. replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, pcsg.Spec.Replicas) if err != nil || len(removed) == 0 { - return ctrl.Result{}, err + return err } // Persist the lower PCSG count before asynchronous LPR/pod cleanup starts. if manageReplicas { - if err := r.scaleDownPodCliqueScalingGroup(ctx, pcsg, &replicas); err != nil { - return ctrl.Result{}, err + if err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, replicas); err != nil { + return err } } - return ctrl.Result{}, r.deletePipelineRequests(ctx, removed) + + return r.deletePipelineRequests(ctx, removed) } -// expiredPipelineRequestSuffix returns complete engine replicas only when every +// expiredPipelineRequestSuffix returns complete workload replicas only when every // expired replica belongs to one contiguous trailing suffix. It orders the // expired requests last so transient sibling cleanup leaves durable expiry // evidence for the next reconciliation. An interior failure blocks all diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 1d4d95f68dc2..b62f589965a6 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -34,6 +34,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { for _, tc := range []struct { name string starts []*metav1.Time + created *metav1.Time seconds *int64 noStatus bool deleting bool @@ -43,9 +44,11 @@ func TestPipelineRequestDeadlines(t *testing.T) { }{ {name: "disabled deadline", starts: []*metav1.Time{oldStart}}, {name: "no requests", seconds: ptr.To(int64(30))}, - {name: "no scheduler receipt", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), noStatus: true}, - {name: "no scheduling start", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30))}, - {name: "zero scheduling start", starts: []*metav1.Time{{}}, seconds: ptr.To(int64(30))}, + {name: "no scheduler receipt expires from creation", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), noStatus: true, wantExpired: []string{"request-0"}}, + {name: "no scheduling start expires from creation", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}}, + {name: "zero scheduling start expires from creation", starts: []*metav1.Time{{}}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}}, + {name: "new request awaits scheduler with a deadline", starts: []*metav1.Time{nil}, created: ptr.To(metav1.NewTime(now)), seconds: ptr.To(int64(30)), noStatus: true, wantWake: now.Add(30 * time.Second)}, + {name: "unpublished request has no deadline", starts: []*metav1.Time{nil}, created: &metav1.Time{}, seconds: ptr.To(int64(30)), noStatus: true}, {name: "expired cycle", starts: []*metav1.Time{oldStart}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}}, {name: "deleting request", starts: []*metav1.Time{oldStart}, seconds: ptr.To(int64(30)), deleting: true}, {name: "independent starts", starts: []*metav1.Time{laterStart, newStart}, seconds: ptr.To(int64(30)), wantWake: newStart.Add(30 * time.Second)}, @@ -53,7 +56,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { {name: "surviving request starts again", starts: []*metav1.Time{newStart}, seconds: ptr.To(int64(30)), lastPlanRevision: 1, wantWake: newStart.Add(30 * time.Second)}, } { t.Run(tc.name, func(t *testing.T) { - t.Log("Observe scheduling cycles independently of object creation time and prior plans") + t.Log("Use creation time until the scheduler supplies a scheduling-cycle start") requests := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(tc.starts)) for index, start := range tc.starts { request := &lpxv1alpha1.LPUPipelineRequest{ @@ -63,6 +66,9 @@ func TestPipelineRequestDeadlines(t *testing.T) { SchedulingStartedAt: start, LastPlanRevision: tc.lastPlanRevision, }, } + if tc.created != nil { + request.CreationTimestamp = *tc.created + } if tc.noStatus { request.Status = nil } @@ -184,7 +190,7 @@ func TestReconcileExpiredPipelineRequests(t *testing.T) { {name: "externally managed capacity", replicas: 1, expired: []int{0}, wantReplicas: 1, removed: []int{0}}, } { t.Run(tc.name, func(t *testing.T) { - t.Log("Observe independent engine requests with only the selected ordinals expired") + t.Log("Observe independent workload requests with only the selected ordinals expired") dgd := loadTestDGD(t, lpx.PipelineSingle, "build-v2") deployment := newLPXTestDeployment(t, dgd) pcs := newTestPodCliqueSet(deployment) @@ -225,9 +231,8 @@ func TestReconcileExpiredPipelineRequests(t *testing.T) { return delegated.Delete(ctx, object, opts...) }, }) - result, err := r.reconcileExpiredPipelineRequests(t.Context(), group, requests, expired, tc.manageReplicas) + err := r.reconcileExpiredPipelineRequests(t.Context(), group, requests, expired, tc.manageReplicas) require.NoError(t, err) - require.Zero(t, result, "LPR and PCSG watches drive further cleanup") require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) require.Equal(t, tc.wantReplicas, group.Spec.Replicas) for i := range requests { @@ -323,7 +328,7 @@ func newTestPodCliqueSet(deployment *v1alpha1.LPXGraphDeployment) *grovev1alpha1 } func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { - t.Log("Publish multiple model requests for one engine and expire one of them") + t.Log("Publish multiple model requests for one workload and expire one of them") ctx := t.Context() child, dgd, registry := newLPXSpecDecodeTestDGD(t) r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) @@ -333,7 +338,8 @@ func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { pcs := findLPXTestPodCliqueSet(t, objects) requests, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) - group, err := getPodCliqueScalingGroup(ctx, r.Client, pcs) + groups, err := getPodCliqueScalingGroups(ctx, r.Client, pcs) + group := groups[desired.plan.LPXScalingGroup] require.NoError(t, err) require.Greater(t, len(requests), 1) expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) @@ -357,50 +363,19 @@ func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { return delegated.Delete(ctx, object, opts...) }, }) - _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) require.ErrorIs(t, err, deleteErr) require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - t.Log("Retry from the retained expiry proof and remove the complete engine request set") + t.Log("Retry from the retained expiry proof and remove the complete workload request set") r.Client = base requests, err = r.getPipelineRequests(ctx, pcs) require.NoError(t, err) expired = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, expired.Name) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) require.NoError(t, err) for index := range desired.requests { requirePipelineRequestNotFound(t, ctx, r.Client, child.Namespace, desired.requests[index].Name) } } - -func TestExpiredPipelineRequestsWaitForScalingGroup(t *testing.T) { - t.Log("Keep an expired request while its live PCS has not materialized a scaling group") - ctx := t.Context() - child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, desired) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, desired) - pcs := findLPXTestPodCliqueSet(t, objects) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) - expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) - require.NoError(t, r.Delete(ctx, group)) - requests, err := r.getPipelineRequests(ctx, pcs) - require.NoError(t, err) - _, err = r.reconcileExpiredPipelineRequests(ctx, nil, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) - - t.Log("Once Grove creates the group, issue scale-down before deleting the LPR") - group.ResourceVersion = "" - group.UID = "replacement-group" - group.DeletionTimestamp = nil - group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - require.NoError(t, r.Create(ctx, group)) - _, err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Zero(t, group.Spec.Replicas) - require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{}))) -} diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 554a57562e85..1fd9b8b81308 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -112,7 +112,11 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { func TestPipelineRequestIdentityDigest(t *testing.T) { t.Log("Every identity field distinguishes requests; the same tuple always has the same digest") - base := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "default", 0) + base := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "", "default", 0) + firstGroup := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "first", "default", 0) + secondGroup := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "second", "default", 0) + require.NotEqual(t, base, firstGroup) + require.NotEqual(t, firstGroup, secondGroup) for _, test := range []struct { name string namespace string @@ -130,17 +134,17 @@ func TestPipelineRequestIdentityDigest(t *testing.T) { {"replica", "ns", "dgd", "uid-a", "default", 1, false}, } { t.Run(test.name, func(t *testing.T) { - digest := pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, test.model, test.replica) + digest := pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, "", test.model, test.replica) require.NotEmpty(t, digest) require.Equal(t, test.same, base == digest) - require.Equal(t, digest, pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, test.model, test.replica)) + require.Equal(t, digest, pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, "", test.model, test.replica)) }) } } func TestPipelineRequestName(t *testing.T) { t.Log("Keep request names readable and within the DNS label limit") - digest := pipelineRequestIdentityDigest("ns", "dgd", "uid", "default", 0) + digest := pipelineRequestIdentityDigest("ns", "dgd", "uid", "", "default", 0) for _, test := range []struct{ name, prefix string }{ {"dgd", "dgd"}, {"a-very-long-but-readable-dynamo-graph-deployment-name", "a-very-long-but-readabl"}, @@ -172,7 +176,7 @@ func TestResolvePipelineRequestsPreservesImmutableIntent(t *testing.T) { current := desired.requests[0].DeepCopy() current.Annotations[test.annotation] = test.value require.Equal(t, !test.changed, pipelineRequestMatches(current, &desired.requests[0])) - requests, missing, changed := resolvePipelineRequests(deployment, []lpxv1alpha1.LPUPipelineRequest{*current}, desired.workload, desired.plan) + requests, missing, changed := resolvePipelineRequests(deployment, map[string]*lpxv1alpha1.LPUPipelineRequest{current.Name: current}, desired.workload, desired.plan) require.Equal(t, test.changed, changed) require.Empty(t, missing) if !changed { @@ -227,9 +231,6 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( require.NotContains(t, clique.Annotations, lpxv1alpha1.PodPartitionIDAnnotation) require.NotContains(t, clique.Annotations, lpxv1alpha1.PodRankInPartitionAnnotation) } - pcs := findLPXTestPodCliqueSet(t, objects) - require.Equal(t, desired.workload.Digest().String(), pcs.Annotations[lpx.WorkloadDigestAnnotation]) - } func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { @@ -248,7 +249,7 @@ func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { {name: "mismatch discards earlier missing requests", replicas: 2, published: []int{5}, intentChanged: true}, } { t.Run(tc.name, func(t *testing.T) { - t.Log("Resolve two engine replicas with draft0, draft1 and target models") + t.Log("Resolve two workload replicas with draft0, draft1 and target models") deployment, dgd, registry := newLPXSpecDecodeTestDGD(t) desired := resolveLPXTestWorkload(t, registry, t.Context(), deployment, dgd) desired.plan.Replicas = 2 @@ -261,7 +262,7 @@ func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { } t.Log("Keep observed receipts and collect only absent desired names without sorting") - var observed []lpxv1alpha1.LPUPipelineRequest + observed := make(map[string]*lpxv1alpha1.LPUPipelineRequest) for _, index := range tc.published { request := rendered[index].DeepCopy() request.UID = types.UID(request.Name) @@ -269,7 +270,7 @@ func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { if tc.intentChanged { request.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation] = "another-snapshot" } - observed = append(observed, *request) + observed[request.Name] = request } desired.plan.Replicas = tc.replicas requests, missing, changed := resolvePipelineRequests(deployment, observed, desired.workload, desired.plan) @@ -285,8 +286,7 @@ func TestResolvePipelineRequestsCollectsMissingInOrder(t *testing.T) { require.Equal(t, rendered[renderedIndex], missing[index]) require.Same(t, requests[missing[index].Name], missing[index]) } - for index := range observed { - request := &observed[index] + for _, request := range observed { if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex < int64(tc.replicas) { require.Same(t, request, requests[request.Name]) } diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 688f60d528fe..2679b92d9f02 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -5,15 +5,19 @@ package lpx import ( "context" + "crypto/sha256" "encoding/json" "fmt" "maps" + "slices" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" consts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,67 +28,139 @@ const ( deploymentUIDLabel = deploymentUIDAnnotation ) -// renderPodCliqueSet renders the admitted DGD using the resolved build and plan. -// The DGD, child, workload and plan must be non-nil; the plan must use the -// child's PodCliqueSet identity. The renderer -// does not mutate its inputs. secretsRetriever may be nil when no secrets are used. -func (r *graphReconciler) renderPodCliqueSet( +// resolveWorkloads resolves every component group and finalizes its resource names. +// deployment and dgd must be non-nil; dgd must have passed admission. +// Returned maps share conductor component keys. Inputs are not mutated. +func (r *graphReconciler) resolveWorkloads( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, - workload *lpx.SelectedWorkload, - plan *lpx.MaterializationPlan, -) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { - // Apply shared Dynamo defaults before materializing the independently owned LPX roles. - pcs, input, err := dynamo.RenderLPXBasePodCliqueSet( - ctx, - dgd, - r.config, - r.runtimeConfig, - r.dockerSecretRetriever, - workload, - plan, +) (map[string]*lpx.Workload, map[string]*lpx.MaterializationPlan, error) { + var ( + workloads = make(map[string]*lpx.Workload) + plans = make(map[string]*lpx.MaterializationPlan) + groups = lpx.ComponentGroups(dgd) ) - if err != nil { - return nil, nil, err + + for _, groupName := range slices.Sorted(maps.Keys(groups)) { + workload, err := lpx.ResolveWorkload(ctx, dgd, groups[groupName], r.modelRegistry) + if err != nil { + return nil, nil, err + } + workloads[groupName] = workload + + plan, err := workload.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(deployment)) + if err != nil { + return nil, nil, err + } + + // Only independent workloads need additional names within the shared PCS. + if len(groups) > 1 { + plan, err = plan.WithGroup(groupName) + if err != nil { + return nil, nil, err + } + } + + // Compare finalized identities only against previously resolved groups. + for previousGroup, previous := range plans { + if previous.ScalingGroupTemplate == plan.ScalingGroupTemplate || previous.ResourcePrefix == plan.ResourcePrefix { + return nil, nil, fmt.Errorf("LPX components %q and %q resolve to conflicting resource names; rename one component", previousGroup, groupName) + } + } + plans[groupName] = plan } - resources, err := lpx.RenderSelectedNodeLocal(pcs, workload, plan, *input) + + return workloads, plans, nil +} + +// renderPodCliqueSet composes resolved workloads into one Grove envelope with +// runtime ConfigMaps and, for Kubernetes discovery, serving Services. +// Inputs must be non-nil and workloads must contain every component group. +// Plans must have finalized names and use the same keys as workloads. +// Inputs remain read-only. +func (r *graphReconciler) renderPodCliqueSet( + ctx context.Context, + deployment *v1alpha1.LPXGraphDeployment, + dgd *v1beta1.DynamoGraphDeployment, + workloads map[string]*lpx.Workload, + plans map[string]*lpx.MaterializationPlan, +) (*grovev1alpha1.PodCliqueSet, []client.Object, error) { + // Shared defaults and queue resolution belong to the single PCS envelope. + pcs, err := dynamo.RenderLPXPodCliqueSet(ctx, dgd, r.config, r.runtimeConfig, dynamo.PCSNameForLPX(deployment)) if err != nil { return nil, nil, err } - // Stamp stable ownership and identify the engine's serving endpoint. - stampDeploymentIdentity(deployment, pcs, resources) - serving := plan.ConductorTemplate - if serving == "" { - serving = plan.CyborgTemplate - } + var ( + resources []client.Object + digest = sha256.New() + ) - // The child retains the last delivered LPX restart while other components restart. - restartToken := deployment.Annotations[dynamo.LPXRestartAnnotation] - for _, clique := range pcs.Spec.Template.Cliques { - if restartToken != "" { - clique.Annotations[consts.RestartAnnotation] = restartToken + for _, groupName := range slices.Sorted(maps.Keys(plans)) { + workload, plan := workloads[groupName], plans[groupName] + + // Render this workload's roles using the full graph for shared defaults. + rendered, err := dynamo.RenderLPXWorkloadTemplates(dgd, r.config, r.runtimeConfig, r.dockerSecretRetriever, workload, plan) + if err != nil { + return nil, nil, err } - delete(clique.Labels, dynamo.LPXServingLabel) - if clique.Name == serving { - clique.Labels[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue - } else { - // Auxiliary roles must be private in both Service and Pod discovery modes. - delete(clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) - delete(clique.Labels, consts.KubeLabelDynamoDiscoveryBackend) - delete(clique.Labels, consts.KubeLabelDynamoBaseModelHash) + + // Discovery selects only this workload's conductor; all auxiliary roles stay private. + serving := plan.ConductorTemplate + if serving == "" { + serving = plan.CyborgTemplate + } + for _, clique := range rendered.Cliques { + if token := deployment.Annotations[dynamo.LPXRestartAnnotation]; token != "" { + clique.Annotations[consts.RestartAnnotation] = token + } + delete(clique.Labels, dynamo.LPXServingLabel) + if clique.Name == serving { + clique.Labels[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue + } else { + delete(clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) + delete(clique.Labels, consts.KubeLabelDynamoDiscoveryBackend) + delete(clique.Labels, consts.KubeLabelDynamoBaseModelHash) + } + } + + // Component order fixes both rendering and the graph's immutable workload digest. + pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, rendered.Cliques...) + pcs.Spec.Template.PodCliqueScalingGroupConfigs = append(pcs.Spec.Template.PodCliqueScalingGroupConfigs, rendered.ScalingGroup) + resources = append(resources, rendered.Resources...) + + // Kubernetes discovery exposes only this workload's serving role within its PCS. + if commoncontroller.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, dgd.Annotations) { + component := dgd.GetComponentByName(workload.ServingComponentName()) + service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ + ServiceName: plan.ResourcePrefix + "-serve", Namespace: deployment.Namespace, + ComponentType: string(component.ComponentType), ComponentName: component.ComponentName, + DynamoNamespace: dgd.GetDynamoNamespaceForComponent(component), IsK8sDiscovery: true, + Labels: dynamo.GetDGDComponentResourceLabels(dgd, component.ComponentName, component), + Annotations: dynamo.GetDGDComponentResourceAnnotations(dgd, component.ComponentName, component), + }) + if err != nil { + return nil, nil, err + } + service.Spec.Selector[dynamo.LPXServingLabel] = consts.KubeLabelValueTrue + service.Spec.Selector[grovecommon.LabelPartOfKey] = pcs.Name + resources = append(resources, service) } + + writeIdentityHashField(digest, groupName, workload.Digest().String()) } - // Enforce the size budget after ownership, scheduling and discovery metadata are final. + pcs.Annotations[lpx.WorkloadDigestAnnotation] = fmt.Sprintf("sha256:%x", digest.Sum(nil)) + stampDeploymentIdentity(deployment, pcs, resources) + + // Enforce the aggregate size budget after identity and discovery metadata are final. serialized, err := json.Marshal(pcs) if err != nil { return nil, nil, fmt.Errorf("serializing selected LPX PodCliqueSet: %w", err) } if len(serialized) > lpx.MaxRenderedPodCliqueSetBytes { - return nil, nil, fmt.Errorf("rendered LPX PodCliqueSet is %d bytes; maximum is %d", - len(serialized), lpx.MaxRenderedPodCliqueSetBytes) + return nil, nil, fmt.Errorf("rendered LPX PodCliqueSet is %d bytes; maximum is %d", len(serialized), lpx.MaxRenderedPodCliqueSetBytes) } return pcs, resources, nil } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 0841a6f2ad33..5682f779a494 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -26,6 +26,8 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" "gotest.tools/v3/golden" @@ -67,6 +69,8 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { "from_dgd_yaml/node-local-v3-hx-specdecode", "from_dgd_yaml/node-local-v3-hx-hybrid", "from_dgd_yaml/single_v2", + "from_dgd_yaml/lpu-cyborg-specdec", + "from_dgd_yaml/lpu-gpu-specdecode", } for _, name := range tests { @@ -83,13 +87,12 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { if dynamoDeployment.Generation == 0 { dynamoDeployment.Generation = 1 } - selected, err := lpx.ResolveSelectedWorkload(t.Context(), &dynamoDeployment, testdataModelRegistry) + t.Log("Resolve and render every selected workload into the shared PCS") + r := &graphReconciler{config: controllerConfig, runtimeConfig: runtimeConfig, modelRegistry: testdataModelRegistry} + child := newLPXRenderDeployment(t, &dynamoDeployment) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, &dynamoDeployment) require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, &dynamoDeployment))) - require.NoError(t, err) - t.Log("Render all LPX roles with the separate LPX PCS identity") - r := &graphReconciler{config: controllerConfig, runtimeConfig: runtimeConfig} - got, extraResources, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, &dynamoDeployment), &dynamoDeployment, selected, plan) + got, extraResources, err := r.renderPodCliqueSet(t.Context(), child, &dynamoDeployment, workloads, plans) require.NoError(t, err) for _, clique := range got.Spec.Template.Cliques { require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) @@ -112,7 +115,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { require.NotEqual(t, normal.Name, got.Name) for _, clique := range normal.Spec.Template.Cliques { require.NotEqual(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) - require.NotEqual(t, selected.LPXComponentName(), clique.Labels[consts.KubeLabelDynamoComponent]) + require.False(t, dynamoDeployment.GetComponentByName(clique.Labels[consts.KubeLabelDynamoComponent]).IsLPX()) } podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} if len(normal.Spec.Template.Cliques) > 0 { @@ -142,25 +145,100 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { out = append(out, string(b)) } - t.Log("Replace computed content hashes with stable golden placeholders; mismatched references remain visible") - replacements := []string{registryRoot, "/testdata"} - for _, resource := range extraResources { - config, ok := resource.(*corev1.ConfigMap) - if !ok { - continue - } - hash := lpx.LPUConfigMapHash(config) - role := strings.TrimPrefix(strings.TrimSuffix(config.Name, "-"+hash[:16]), got.Name+"-") - replacements = append(replacements, - hash, "<"+role+"-config-hash>", - hash[:16], "<"+role+"-config-hash-prefix>", - ) - } goldenPath, err := filepath.Abs("../../dynamo/lpx/testdata/" + name + ".yaml") require.NoError(t, err) const header = "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0\n\n" - golden.Assert(t, header+strings.NewReplacer(replacements...).Replace(strings.Join(out, "---\n")), goldenPath) + golden.Assert(t, header+strings.Join(out, "---\n"), goldenPath) + }) + } +} + +func TestLPXRenderingIncludesDiscoveryServices(t *testing.T) { + for _, tc := range []struct { + name string + pipeline lpx.Pipeline + backend configv1alpha1.DiscoveryBackend + override configv1alpha1.DiscoveryBackend + wantServices bool + }{ + {name: "discovery disabled", pipeline: lpx.PipelineSingle}, + {name: "operator enables Kubernetes discovery", pipeline: lpx.PipelineSingle, backend: configv1alpha1.DiscoveryBackendKubernetes, wantServices: true}, + {name: "DGD enables hybrid discovery", pipeline: lpx.PipelineLPX, override: configv1alpha1.DiscoveryBackendKubernetes, wantServices: true}, + {name: "DGD disables Kubernetes discovery", pipeline: lpx.PipelineSingle, backend: configv1alpha1.DiscoveryBackendKubernetes, override: configv1alpha1.DiscoveryBackendEtcd}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Resolve two workloads with authored metadata and the selected discovery backend") + child, dgd, registry := newLPXTestDGD(t, tc.pipeline) + dgd.Spec.Components[0].ModelRef = &v1beta1.ModelReference{Name: "test/model"} + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName = "second" + dgd.Spec.Components = append(dgd.Spec.Components, *second) + dgd.Spec.Labels = map[string]string{"example.com/team": "inference"} + dgd.Spec.Annotations = map[string]string{"example.com/description": "serving"} + if tc.override != "" { + dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(tc.override) + } + r := newLPXTestReconciler(t, registry, child, dgd) + r.config.Discovery.Backend = tc.backend + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + before := dgd.DeepCopy() + + t.Log("Render ConfigMaps and optional Services without writing resources or mutating the DGD") + pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + require.Equal(t, before, dgd) + serviceCount, configMapCount := 0, 0 + for _, resource := range resources { + switch resource.(type) { + case *corev1.Service: + serviceCount++ + case *corev1.ConfigMap: + configMapCount++ + } + } + require.Positive(t, configMapCount) + if !tc.wantServices { + require.Zero(t, serviceCount) + return + } + require.Equal(t, len(plans), serviceCount) + + t.Log("Only serving roles advertise model discovery") + for _, clique := range pcs.Spec.Template.Cliques { + if clique.Labels[dynamo.LPXServingLabel] == consts.KubeLabelValueTrue { + require.NotEqual(t, lpxv1alpha1.PodRoleAgent, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) + require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[consts.KubeLabelDynamoBaseModelHash]) + } else { + require.NotContains(t, clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) + require.NotContains(t, clique.Labels, consts.KubeLabelDynamoBaseModelHash) + } + } + + t.Log("Each Service carries discovery metadata and selects its own serving component and PCS") + for groupName, plan := range plans { + service := getResource[*corev1.Service](t, resources, plan.ResourcePrefix+"-serve") + component := dgd.GetComponentByName(groupName) + require.Equal(t, child.Namespace, service.Namespace) + require.Equal(t, "inference", service.Labels["example.com/team"]) + require.Equal(t, map[string]string{ + "example.com/description": "serving", + lpx.DeploymentNameAnnotation: child.Name, + deploymentUIDAnnotation: string(child.UID), + lpx.DGDUIDAnnotation: string(dgd.UID), + }, service.Annotations) + require.Equal(t, string(child.UID), service.Labels[deploymentUIDLabel]) + require.Equal(t, consts.KubeLabelValueTrue, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) + require.Equal(t, consts.DiscoveryBackendKubernetes, service.Labels[consts.KubeLabelDynamoDiscoveryBackend]) + require.Equal(t, map[string]string{ + consts.KubeLabelDynamoComponent: groupName, + consts.KubeLabelDynamoComponentType: string(component.ComponentType), + consts.KubeLabelDynamoNamespace: dgd.GetDynamoNamespaceForComponent(component), + dynamo.LPXServingLabel: consts.KubeLabelValueTrue, + grovecommon.LabelPartOfKey: pcs.Name, + }, service.Spec.Selector) + } }) } } @@ -172,21 +250,20 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { dgd := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, dgd)) dgd.Annotations = map[string]string{"kai.scheduler/padding": ""} - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, newTestDataModelRegistry(t, t.TempDir())) - require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) + selected, err := lpx.ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) config := &configv1alpha1.OperatorConfiguration{} runtimeConfig := &commoncontroller.RuntimeConfig{} r := &graphReconciler{config: config, runtimeConfig: runtimeConfig} - pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + plan := mustPlanSelectedLPX(t, dgd, selected) + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.NoError(t, err) serialized, err := json.Marshal(pcs) require.NoError(t, err) t.Log("Accept exactly one MiB including final identity, discovery and scheduler metadata") dgd.Annotations["kai.scheduler/padding"] = strings.Repeat("x", lpx.MaxRenderedPodCliqueSetBytes-len(serialized)) - pcs, _, err = r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + pcs, _, err = r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.NoError(t, err) serialized, err = json.Marshal(pcs) require.NoError(t, err) @@ -197,19 +274,19 @@ func TestLPXRenderingChecksFinalPodCliqueSetSize(t *testing.T) { t.Log("Reject one additional final-metadata byte as a selected-render failure before publication") dgd.Annotations["kai.scheduler/padding"] += "x" - pcs, resources, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + pcs, resources, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.ErrorContains(t, err, "rendered LPX PodCliqueSet is 1048577 bytes; maximum is 1048576") require.Nil(t, pcs) require.Nil(t, resources) } func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { - t.Log("Load an authored hybrid engine and override its independent leader template") + t.Log("Load an authored hybrid workload and override its independent leader template") payload, err := os.ReadFile(filepath.Join("../../dynamo/lpx/testdata", "from_dgd_yaml", "node-local-v2-hybrid.input.yaml")) require.NoError(t, err) dgd := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, dgd)) - leader := lpx.ServingComponent(dgd).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate + leader := dgd.GetComponentByName("lpu").ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate leader.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ {Name: "config", MountPath: "/custom-config", ReadOnly: true}, {Name: "infiniband", MountPath: "/custom-infiniband", ReadOnly: true}, @@ -234,11 +311,11 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { leader.Spec.Volumes = append(leader.Spec.Volumes, authoredConfig) t.Log("Render the complete LPX workload with ordinary pod spec overrides") - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, newTestDataModelRegistry(t, t.TempDir())) + selected, err := lpx.ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), newTestDataModelRegistry(t, t.TempDir())) require.NoError(t, err) plan := mustPlanSelectedLPX(t, dgd, selected) r := &graphReconciler{config: &configv1alpha1.OperatorConfiguration{}, runtimeConfig: &commoncontroller.RuntimeConfig{}} - pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.NoError(t, err) cliqueIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { return clique.Name == plan.CyborgTemplate @@ -271,290 +348,149 @@ func TestLPXRenderingPreservesCyborgOverrides(t *testing.T) { } func TestLPXRenderingPreservesInputs(t *testing.T) { - const restartToken = "2026-09-08T00:00:00Z" - t.Log("Create immutable native builds while keeping each rendering's mutable inputs separate") + const mutatedValue = "rendered-change" + t.Log("Create compiler builds for each supported runtime shape") registry := newTestDataModelRegistry(t, t.TempDir()) - - for _, test := range []struct { - name string - wantFamily lpxv1alpha1.TargetFamily - wantMode lpxv1alpha1.WorkloadMode - }{ - {"node-local-v2-lpu-only", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, - {"node-local-v2-hybrid", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2StrictHybrid}, - {"node-local-v2-specdecode", lpxv1alpha1.TargetFamilyXt8888, lpxv1alpha1.WorkloadModeV2LPUOnly}, - {"node-local-v3-hx-lpu-only", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, - {"node-local-v3-hx-hybrid", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxStrictHybrid}, - {"node-local-v3-hx-specdecode", lpxv1alpha1.TargetFamilyHx16x8x2x3, lpxv1alpha1.WorkloadModeV3HxLPUOnly}, + for _, name := range []string{ + "node-local-v2-lpu-only", "node-local-v2-hybrid", "node-local-v2-specdecode", + "node-local-v3-hx-lpu-only", "node-local-v3-hx-hybrid", "node-local-v3-hx-specdecode", } { - t.Run(test.name, func(t *testing.T) { - t.Log("Resolve the authored engine and its exact child without changing either input") - fixture := test.name - payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + fixture + ".input.yaml") + t.Run(name, func(t *testing.T) { + t.Log("Resolve authored inputs with shared and role-specific metadata") + payload, err := os.ReadFile("../../dynamo/lpx/testdata/from_dgd_yaml/" + name + ".input.yaml") require.NoError(t, err) dgd := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, dgd)) - dgd.UID, dgd.Generation = "source-dgd-uid", 7 - dgd.Labels = map[string]string{ - "priorityClassName": "inference", "kai.scheduler/preemptibility": "NonPreemptible", - "project": "test", "unrelated": "ignored", - consts.KubeLabelKaiSchedulerQueue: "lpx-queue", - } - metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, "kai.scheduler/topology", "source-topology") - metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, "unrelated", "ignored") - metav1.SetMetaDataAnnotation(&dgd.ObjectMeta, consts.KubeAnnotationEnableMetrics, "false") dgd.Spec.Labels = map[string]string{"shared-label": "source"} - dgd.Spec.Annotations = map[string]string{ - "shared-annotation": "source", "kai.scheduler/topology": "explicit-pcs-topology", - "selected-metadata": "from-dgd", - lpx.DGDUIDAnnotation: "stale", - lpx.DGDGenerationAnnotation: "stale", - } - dgd.Spec.Env = []corev1.EnvVar{{Name: "LPX_DGD_ENV", Value: test.name}} + dgd.Spec.Annotations = map[string]string{"shared-annotation": "source"} for _, component := range lpx.Components(dgd) { - component.GlobalDynamoNamespace = true - component.RuntimeVersionOverride = "1.5.0" for _, role := range component.Roles { - if role.PodTemplate != nil { - metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, "selected-metadata", "from-component") - metav1.SetMetaDataAnnotation(&role.PodTemplate.ObjectMeta, consts.RestartAnnotation, "2026-09-07T00:00:00Z") - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, "shared-label", "from-component") - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelKaiSchedulerQueue, "lpx-queue") - metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, consts.KubeLabelDynamoNamespace, "authored-namespace") - } + metav1.SetMetaDataLabel(&role.PodTemplate.ObjectMeta, "shared-label", "role") } } - component := lpx.ServingComponent(dgd) - component.ModelRef = &v1beta1.ModelReference{Name: "test/model"} - component.MinAvailable = ptr.To(int32(1)) - hybrid := strings.HasSuffix(fixture, "-hybrid") - singleXT := test.name == "node-local-v2-lpu-only" - if hybrid { - component.Replicas = ptr.To(int32(3)) - component.MinAvailable = ptr.To(int32(3)) - component.ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - } - if singleXT { - t.Log("Keep XT's authored readonly mount at the canonical config path") - main := &component.ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0] - configMount := slices.IndexFunc(main.VolumeMounts, func(mount corev1.VolumeMount) bool { return mount.MountPath == "/configs" }) - require.GreaterOrEqual(t, configMount, 0) - main.VolumeMounts[configMount].ReadOnly = true - } - - t.Log("Freeze the authored role metadata and replica units before rendering") child := newLPXRenderDeployment(t, dgd) - child.Annotations[dynamo.LPXRestartAnnotation] = restartToken - child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, restartToken) - require.NoError(t, err) - dgd.Generation++ before, childBefore := dgd.DeepCopy(), child.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) - require.NoError(t, err) - plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) - require.NoError(t, err) - config := &configv1alpha1.OperatorConfiguration{ - Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, - } - - t.Log("Project every model into the expected scheduler family, mode and physical Agent shape") - require.Equal(t, lpx.BuildFamily(test.wantFamily), selected.BuildFamily()) - projections := selected.ModelProjections() - require.Len(t, plan.Agents, len(projections)) - projectionIndices := make(map[string]int, len(projections)) - for index, projection := range projections { - projectionIndices[plan.Agents[index].TemplateName] = index - request := projection.RequestSpec(plan, "agents") - require.Equal(t, lpxv1alpha1.ExecutionBackendNodeLocal, request.ExecutionBackend) - require.NotNil(t, request.NodeLocal) - require.Equal(t, test.wantFamily, request.TargetFamily) - require.Equal(t, test.wantMode, request.WorkloadMode) - require.NotEmpty(t, request.Partitions) - require.Len(t, request.NodeLocal.PartitionMappings, len(request.Partitions)) - require.Equal(t, hybrid, request.CyborgPodCliqueRef != nil) - if test.wantFamily == lpxv1alpha1.TargetFamilyHx16x8x2x3 { - require.Len(t, request.Partitions, 1) - require.Equal(t, int64(1), request.Partitions[0].CompilerPartitionID) - require.Equal(t, []int64{16, 1, 1, 1}, *request.Partitions[0].Extent) - require.Empty(t, request.PropSyncConnectors) - require.Equal(t, 1, plan.Agents[index].Replicas) - } + r := &graphReconciler{ + config: &configv1alpha1.OperatorConfiguration{ + Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, + }, + runtimeConfig: &commoncontroller.RuntimeConfig{}, modelRegistry: registry, } + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + plan := plans["lpu"] + planBefore := plan.ForReplica(plan.ReplicaIndex) - t.Log("Render twice and require identical resources with no input or shared-metadata mutation") - r := &graphReconciler{config: config, runtimeConfig: &commoncontroller.RuntimeConfig{}} - first, firstResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, selected, plan) + t.Log("Render twice and require identical independently owned output") + first, firstResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) require.NoError(t, err) - second, secondResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, selected, plan) + second, secondResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) require.NoError(t, err) require.Equal(t, first, second) require.Equal(t, firstResources, secondResources) - require.Equal(t, "inference", first.Labels["priorityClassName"]) - require.Equal(t, "NonPreemptible", first.Labels["kai.scheduler/preemptibility"]) - require.Equal(t, "lpx-queue", first.Labels[consts.KubeLabelKaiSchedulerQueue]) - require.Equal(t, "test", first.Labels["project"]) - require.Equal(t, "explicit-pcs-topology", first.Annotations["kai.scheduler/topology"]) - require.NotContains(t, first.Annotations, consts.RestartAnnotation) - require.NotContains(t, first.Labels, "unrelated") - require.NotContains(t, first.Annotations, "unrelated") - require.Equal(t, selected.Digest().String(), first.Annotations[lpx.WorkloadDigestAnnotation]) - - t.Log("Stamp frozen source identity on every final resource despite a newer ordinary DGD generation") - require.NotEqual(t, dgd.UID, child.UID) - annotationMaps := []map[string]string{first.Annotations} + secondBefore := second.DeepCopy() + resourcesBefore := make([]client.Object, len(secondResources)) + for i, resource := range secondResources { + resourcesBefore[i] = resource.DeepCopyObject().(client.Object) + } + + t.Log("Mutate rendered Pods, metadata and resources without changing other output or inputs") + first.Labels["shared-label"] = mutatedValue + first.Annotations["shared-annotation"] = mutatedValue for _, clique := range first.Spec.Template.Cliques { - require.Equal(t, restartToken, clique.Annotations[consts.RestartAnnotation]) - annotationMaps = append(annotationMaps, clique.Annotations) + clique.Spec.PodSpec.Containers[0].Image = "rendered:changed" + clique.Labels["shared-label"] = mutatedValue + clique.Annotations["shared-annotation"] = mutatedValue } for _, group := range first.Spec.Template.PodCliqueScalingGroupConfigs { - require.NotContains(t, group.Annotations, consts.RestartAnnotation) - annotationMaps = append(annotationMaps, group.Annotations) + group.Annotations[lpx.WorkloadDigestAnnotation] = mutatedValue } for _, resource := range firstResources { - require.NotContains(t, resource.GetAnnotations(), consts.RestartAnnotation) - annotationMaps = append(annotationMaps, resource.GetAnnotations()) - } - for _, annotations := range annotationMaps { - require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) - require.NotContains(t, annotations, lpx.DGDGenerationAnnotation) - require.Equal(t, string(child.UID), annotations[dynamo.LPXDeploymentUIDAnnotation]) - require.NotContains(t, annotations, "lpx.nvidia.com/deployment-generation") - require.NotContains(t, annotations, "lpx.nvidia.com/input-revision") - } - - t.Log("Component and role order preserve the exact child, names and resources") - equivalent := dgd.DeepCopy() - slices.Reverse(equivalent.Spec.Components) - for index := range equivalent.Spec.Components { - slices.Reverse(equivalent.Spec.Components[index].Roles) - } - equivalentBefore := equivalent.DeepCopy() - revision, err := dynamo.LPXInputRevision(equivalent, restartToken) - require.NoError(t, err) - require.Equal(t, child.Spec.InputRevision, revision) - equivalentSelected, err := lpx.ResolveSelectedWorkload(t.Context(), equivalent, registry) - require.NoError(t, err) - equivalentPlan, err := equivalentSelected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(child)) - require.NoError(t, err) - require.Equal(t, selected.Digest(), equivalentSelected.Digest()) - require.Equal(t, plan, equivalentPlan) - equivalentPCS, equivalentResources, err := r.renderPodCliqueSet(t.Context(), child, equivalent, equivalentSelected, equivalentPlan) - require.NoError(t, err) - require.Equal(t, first, equivalentPCS) - require.Equal(t, firstResources, equivalentResources) - require.Equal(t, equivalentBefore, equivalent) - - t.Log("Metadata edits and scale 1, 2, 10, 12, 10 preserve every role template and immutable config version") - replicaCounts := []int32{1, 2, 10, 12, 10} - if selected.Pipeline() == lpx.PipelineSpecDecode { - replicaCounts = []int32{1} // Shared-draft topology supports one target engine. - } - for _, replicas := range replicaCounts { - scaled := dgd.DeepCopy() - metav1.SetMetaDataAnnotation(&scaled.ObjectMeta, "unrelated", "changed") - lpx.ServingComponent(scaled).Replicas = ptr.To(replicas) - scaledChild := child.DeepCopy() - scaledChild.Generation++ - scaledChild.Spec.InputRevision, err = dynamo.LPXInputRevision(scaled, restartToken) - require.NoError(t, err) - workload, err := lpx.ResolveSelectedWorkload(t.Context(), scaled, registry) - require.NoError(t, err) - scaledPlan, err := workload.PlanNodeLocalMaterialization(plan.PodCliqueSetName) - require.NoError(t, err) - pcs, resources, err := r.renderPodCliqueSet(t.Context(), scaledChild, scaled, workload, scaledPlan) - require.NoError(t, err) - require.Equal(t, firstResources, resources) - require.Equal(t, first.Spec.Template.Cliques, pcs.Spec.Template.Cliques) - require.Equal(t, first.Spec.Template.PodCliqueScalingGroupConfigs, pcs.Spec.Template.PodCliqueScalingGroupConfigs) - } - - t.Log("Connect every model projection to its rendered Agent while preserving shared runtime metadata and replica units") - require.Len(t, first.Spec.Template.PodCliqueScalingGroupConfigs, 1) - group := first.Spec.Template.PodCliqueScalingGroupConfigs[0] - require.Equal(t, ptr.To(ptr.Deref(component.MinAvailable, 1)), group.Replicas) - require.Equal(t, component.MinAvailable, group.MinAvailable) - var conductor, gpuClique *grovev1alpha1.PodCliqueTemplateSpec - serving := 0 - for index, clique := range first.Spec.Template.Cliques { - require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName, "role %s must use the LPX backend", clique.Name) - require.Equal(t, "lpx-queue", clique.Labels[consts.KubeLabelKaiSchedulerQueue]) - if clique.Labels[dynamo.LPXServingLabel] == consts.KubeLabelValueTrue { - serving++ - require.NotEqual(t, lpxv1alpha1.PodRoleAgent, clique.Annotations[lpxv1alpha1.PodRoleAnnotation]) - require.Equal(t, dynamo.HashModelName("test/model"), clique.Labels[consts.KubeLabelDynamoBaseModelHash]) - } else { - require.NotContains(t, clique.Labels, consts.KubeLabelDynamoDiscoveryEnabled) - require.NotContains(t, clique.Labels, consts.KubeLabelDynamoBaseModelHash) - } - require.Equal(t, ptr.To(clique.Spec.Replicas), clique.Spec.MinAvailable) - require.NotEmpty(t, clique.Spec.PodSpec.Containers) - require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_DGD_ENV", Value: test.name}) - require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: consts.DynamoNamespaceEnvVar, Value: consts.GlobalDynamoNamespace}) - require.Equal(t, consts.GlobalDynamoNamespace, clique.Labels[consts.KubeLabelDynamoNamespace]) - require.Equal(t, "from-component", clique.Labels["shared-label"]) - require.Equal(t, "false", clique.Annotations[consts.KubeAnnotationEnableMetrics]) - require.Equal(t, "kubernetes", clique.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend]) - if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] != lpxv1alpha1.PodRoleCyborgWorker || test.wantFamily == lpxv1alpha1.TargetFamilyXt8888 { - t.Log("Bind the generated immutable ConfigMap alongside authored volumes and mounts") - configIndex := slices.IndexFunc(clique.Spec.PodSpec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "config" }) - require.GreaterOrEqual(t, configIndex, 0) - configVolume := clique.Spec.PodSpec.Volumes[configIndex] - require.NotNil(t, configVolume.ConfigMap) - generatedConfig := getResource[*corev1.ConfigMap](t, firstResources, configVolume.ConfigMap.Name) - require.Equal(t, ptr.To(true), generatedConfig.Immutable) + resource.GetAnnotations()[deploymentUIDAnnotation] = mutatedValue + switch resource := resource.(type) { + case *corev1.ConfigMap: + resource.Data["rendered-only"] = mutatedValue + case *corev1.Service: + resource.Spec.Selector[dynamo.LPXServingLabel] = mutatedValue } - switch clique.Annotations[lpxv1alpha1.PodRoleAnnotation] { - case lpxv1alpha1.PodRoleAgent: - index, found := projectionIndices[clique.Name] - require.True(t, found, "unexpected or duplicate Agent %q", clique.Name) - require.Equal(t, projections[index].Model(), clique.Annotations[lpxv1alpha1.PodModelAnnotation]) - require.Equal(t, projections[index].Digest().String(), clique.Annotations[lpx.WorkloadDigestAnnotation]) - require.Equal(t, int32(plan.Agents[index].Replicas), clique.Spec.Replicas) - if singleXT { - require.Contains(t, clique.Spec.PodSpec.Containers[0].VolumeMounts, - corev1.VolumeMount{Name: "config", MountPath: "/configs", ReadOnly: true}) - } - delete(projectionIndices, clique.Name) - case lpxv1alpha1.PodRoleConductor: - require.Nil(t, conductor) - conductor = clique - case lpxv1alpha1.PodRoleCyborgWorker: - require.Nil(t, gpuClique) - gpuClique = clique - } - - t.Log("Mutate this rendered role without changing the second rendering or authored input") - clique.Spec.PodSpec.Containers[0].Image = "rendered:changed" - clique.Labels["shared-label"] = "rendered-role" - clique.Annotations["shared-annotation"] = "rendered-role" - require.NotEqual(t, clique.Spec.PodSpec.Containers[0].Image, second.Spec.Template.Cliques[index].Spec.PodSpec.Containers[0].Image) - require.Equal(t, "from-component", second.Spec.Template.Cliques[index].Labels["shared-label"]) - require.Equal(t, "source", second.Spec.Template.Cliques[index].Annotations["shared-annotation"]) - } - require.Empty(t, projectionIndices) - require.Equal(t, 1, serving) - if hybrid { - require.Nil(t, conductor) - require.NotNil(t, gpuClique) - require.Equal(t, int32(2), gpuClique.Spec.Replicas) - require.Equal(t, selected.Digest().String(), gpuClique.Annotations[lpx.WorkloadDigestAnnotation]) - require.Contains(t, gpuClique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "true"}) - } else { - require.NotNil(t, conductor) - require.Equal(t, "from-component", conductor.Annotations["selected-metadata"]) - require.Equal(t, selected.Digest().String(), conductor.Annotations[lpx.WorkloadDigestAnnotation]) - require.Nil(t, gpuClique) } - first.Labels["shared-label"] = "rendered" - first.Annotations["shared-annotation"] = "rendered" - require.Equal(t, "source", second.Labels["shared-label"]) - require.Equal(t, "source", second.Annotations["shared-annotation"]) + require.Equal(t, secondBefore, second) + require.Equal(t, resourcesBefore, secondResources) require.Equal(t, before, dgd) require.Equal(t, childBefore, child) + require.Equal(t, planBefore, plan) }) } } +func TestLPXRenderingMetadata(t *testing.T) { + const restartToken = "2026-09-08T00:00:00Z" + t.Log("Author scheduler metadata and stale identity annotations before freezing the child") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + dgd.Labels = map[string]string{"project": "inference", "unrelated": "ignored"} + dgd.Annotations["kai.scheduler/topology"] = "source-topology" + dgd.Spec.Annotations = map[string]string{ + "kai.scheduler/topology": "explicit-topology", + lpx.DGDUIDAnnotation: "stale", lpx.DGDGenerationAnnotation: "stale", + } + child.Annotations[dynamo.LPXRestartAnnotation] = restartToken + dgd.Generation++ + r := newLPXTestReconciler(t, registry, child, dgd) + r.config.Discovery.Backend = configv1alpha1.DiscoveryBackendKubernetes + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + + t.Log("Inherit scheduler metadata while explicit PCS metadata takes precedence") + require.Equal(t, "inference", pcs.Labels["project"]) + require.NotContains(t, pcs.Labels, "unrelated") + require.Equal(t, "explicit-topology", pcs.Annotations["kai.scheduler/topology"]) + require.NotContains(t, pcs.Annotations, consts.RestartAnnotation) + + t.Log("Stamp the frozen source identity everywhere and the restart token only on Pods") + annotationMaps := []map[string]string{pcs.Annotations} + for _, clique := range pcs.Spec.Template.Cliques { + require.Equal(t, restartToken, clique.Annotations[consts.RestartAnnotation]) + annotationMaps = append(annotationMaps, clique.Annotations) + } + for _, group := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { + require.NotContains(t, group.Annotations, consts.RestartAnnotation) + annotationMaps = append(annotationMaps, group.Annotations) + } + for _, resource := range resources { + require.NotContains(t, resource.GetAnnotations(), consts.RestartAnnotation) + require.Equal(t, string(child.UID), resource.GetLabels()[deploymentUIDLabel]) + annotationMaps = append(annotationMaps, resource.GetAnnotations()) + } + for _, annotations := range annotationMaps { + require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) + require.Equal(t, string(child.UID), annotations[deploymentUIDAnnotation]) + require.NotContains(t, annotations, lpx.DGDGenerationAnnotation) + require.NotContains(t, annotations, "lpx.nvidia.com/deployment-generation") + require.NotContains(t, annotations, "lpx.nvidia.com/input-revision") + } +} + +func TestLPXReplicaChangesPreserveRenderedTemplates(t *testing.T) { + t.Log("Render a hybrid workload at its initial capacity") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + before, beforeResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + + t.Log("Changing only workload capacity preserves templates, configuration and the graph digest") + dgd.Spec.Components[0].Replicas = ptr.To(int32(12)) + workloads, plans, err = r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + after, afterResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + require.Equal(t, before, after) + require.Equal(t, beforeResources, afterResources) +} + func TestLPXSpecDecodeConductorTemplate(t *testing.T) { t.Log("Give the speculative components distinct authored Agent and conductor commands") registry := newTestDataModelRegistry(t, t.TempDir()) @@ -568,7 +504,7 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { agent.Spec.Containers[0].Command = []string{"/bin/quasar-entrypoint"} agent.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "AGENT_ONLY", Value: "kept"}} } - target := lpx.ServingComponent(dgd) + target := dgd.GetComponentByName("lpu") template := target.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate template.Spec.Containers[0].Image = "independent-conductor-runtime" template.Spec.Containers[0].Command = []string{"/bin/nova"} @@ -581,14 +517,14 @@ func TestLPXSpecDecodeConductorTemplate(t *testing.T) { }}}, }} before := dgd.DeepCopy() - selected, err := lpx.ResolveSelectedWorkload(t.Context(), dgd, registry) + selected, err := lpx.ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), registry) require.NoError(t, err) plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) t.Log("Render one shared conductor without changing either component's Agent template") r := &graphReconciler{config: &configv1alpha1.OperatorConfiguration{}, runtimeConfig: &commoncontroller.RuntimeConfig{}} - pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, selected, plan) + pcs, _, err := r.renderPodCliqueSet(t.Context(), newLPXRenderDeployment(t, dgd), dgd, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.NoError(t, err) t.Log("Keep Nova separate from Quasar and preserve each role's metadata and storage") @@ -654,12 +590,12 @@ func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { template := component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate template.Spec.Containers[0].Env = []corev1.EnvVar{env} - selected, err := lpx.ResolveSelectedWorkload(t.Context(), &deployment, registry) + selected, err := lpx.ResolveWorkload(t.Context(), &deployment, singleGroupComponents(t, &deployment), registry) require.NoError(t, err) child := newLPXRenderDeployment(t, &deployment) plan := mustPlanSelectedLPX(t, &deployment, selected) r := &graphReconciler{config: controllerConfig, runtimeConfig: &commoncontroller.RuntimeConfig{}} - pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, &deployment, selected, plan) + pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, &deployment, map[string]*lpx.Workload{selected.ServingComponentName(): selected}, map[string]*lpx.MaterializationPlan{selected.ServingComponentName(): plan}) require.NoError(t, err) t.Log("Keep authored environment values without changing shared partition files") @@ -690,7 +626,7 @@ func getResource[T any](t *testing.T, resources []client.Object, name string) T return resources[i].(T) } -func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.SelectedWorkload) *lpx.MaterializationPlan { +func mustPlanSelectedLPX(t *testing.T, dgd *v1beta1.DynamoGraphDeployment, selected *lpx.Workload) *lpx.MaterializationPlan { t.Helper() plan, err := selected.PlanNodeLocalMaterialization(dynamo.PCSNameForLPX(newLPXRenderDeployment(t, dgd))) require.NoError(t, err) @@ -1019,7 +955,7 @@ func renderLPXTestPodCliqueSet( deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, desired *lpxTestWorkload, ) *grovev1alpha1.PodCliqueSet { t.Helper() - rendered, _, err := reconciler.renderPodCliqueSet(ctx, deployment, dgd, desired.workload, desired.plan) + rendered, _, err := reconciler.renderPodCliqueSet(ctx, deployment, dgd, map[string]*lpx.Workload{desired.workload.ServingComponentName(): desired.workload}, map[string]*lpx.MaterializationPlan{desired.workload.ServingComponentName(): desired.plan}) require.NoError(t, err) return rendered } @@ -1106,3 +1042,32 @@ func writeLPXTestBuild( require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(buildDir, "manifest.v2.capnp.bin"), payload, 0o600)) } + +// singleGroupComponents reads the one admitted workload in a test fixture. +func singleGroupComponents(t *testing.T, dgd *v1beta1.DynamoGraphDeployment) []string { + t.Helper() + groups := lpx.ComponentGroups(dgd) + require.Len(t, groups, 1) + for _, components := range groups { + return components + } + return nil +} + +func TestHybridWorkloadUsesLPXSchedulerWithKaiEnabled(t *testing.T) { + t.Log("Resolve a hybrid workload with KAI integration enabled") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, KaiScheduler: true}} + + t.Log("Use the LPX scheduler for every role and leave queue selection to the PCS") + rendered, err := dynamo.RenderLPXWorkloadTemplates(dgd, r.config, runtimeConfig, nil, workloads["lpx"], plans["lpx"]) + require.NoError(t, err) + require.NotEmpty(t, rendered.Cliques) + for _, clique := range rendered.Cliques { + require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.NotContains(t, clique.Labels, consts.KubeLabelKaiSchedulerQueue, "clique %s", clique.Name) + } +} diff --git a/deploy/operator/internal/controller/lpx/status.go b/deploy/operator/internal/controller/lpx/status.go index c7685d31435b..d1e4ddfb252c 100644 --- a/deploy/operator/internal/controller/lpx/status.go +++ b/deploy/operator/internal/controller/lpx/status.go @@ -28,13 +28,7 @@ const ( // updateStatus persists the updated status on the deployment. func (r *graphReconciler) updateStatus(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, previous *v1alpha1.LPXGraphDeploymentStatus) error { - // Every component inherits the parent ready condition. - ready := meta.IsStatusConditionTrue(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) - for name, component := range deployment.Status.Components { - component.Ready = ready - deployment.Status.Components[name] = component - } - + // Persist the complete observation without overwriting per-workload readiness. if apiequality.Semantic.DeepEqual(previous, &deployment.Status) { return nil } @@ -169,8 +163,8 @@ func schedulingFailureCoversPipelineRequests( return false } for _, request := range requests { - startedAt, known := pipelineRequestSchedulingStartedAt(request) - if !known || !startedAt.Before(failed.LastTransitionTime.Time) { + startedAt := pipelineRequestSchedulingStartedAt(request) + if startedAt.IsZero() || !startedAt.Before(failed.LastTransitionTime.Time) { return false } } @@ -201,9 +195,9 @@ func setSchedulingFailedCondition(deployment *v1alpha1.LPXGraphDeployment, renew }) } -// clearPreviousSchedulingFailure clears an old failure only for a later generation. +// acknowledgeSchedulingRetry marks an older failure superseded by a later generation. // deployment is non-nil; callers must not use this after observing a current expiry. -func clearPreviousSchedulingFailure(deployment *v1alpha1.LPXGraphDeployment) { +func acknowledgeSchedulingRetry(deployment *v1alpha1.LPXGraphDeployment) { prior := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) if prior == nil || deployment.Generation <= prior.ObservedGeneration { return @@ -211,7 +205,7 @@ func clearPreviousSchedulingFailure(deployment *v1alpha1.LPXGraphDeployment) { meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ Type: schedulingFailedCondition, Status: metav1.ConditionFalse, ObservedGeneration: deployment.Generation, Reason: "LPXSchedulingRetryAuthorized", - Message: "The expired request was removed and current input may be published", + Message: "The current input revision supersedes the previous scheduling failure", }) } diff --git a/deploy/operator/internal/controller/lpx/status_test.go b/deploy/operator/internal/controller/lpx/status_test.go index ab8aadb50f68..9d10f4becb16 100644 --- a/deploy/operator/internal/controller/lpx/status_test.go +++ b/deploy/operator/internal/controller/lpx/status_test.go @@ -138,13 +138,13 @@ func TestLPXStatusSkipsUnchangedObservations(t *testing.T) { require.Zero(t, writes) require.Equal(t, previous, &child.Status) - t.Log("A real transition is persisted once and updates component readiness") + t.Log("A graph transition is persisted without overwriting independently observed component readiness") setReadyCondition(child, different, "new observation") require.Zero(t, writes) require.NoError(t, r.updateStatus(t.Context(), child, previous)) require.Equal(t, 1, writes) require.True(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").LastTransitionTime.After(transition.Time)) - require.Equal(t, different == v1beta1.DGDStateSuccessful, child.Status.Components["engine"].Ready) + require.Equal(t, state == v1beta1.DGDStateSuccessful, child.Status.Components["engine"].Ready) }) } } @@ -243,6 +243,8 @@ func TestSchedulingFailureCoversPipelineRequests(t *testing.T) { noCondition bool noFailureTime bool start *metav1.Time + created *metav1.Time + noStatus bool wantCurrent bool wantCovered bool }{ @@ -255,7 +257,11 @@ func TestSchedulingFailureCoversPipelineRequests(t *testing.T) { {name: "older cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt.Add(-time.Minute))), wantCurrent: true, wantCovered: true}, {name: "same timestamp", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt)), wantCurrent: true}, {name: "newer cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: ptr.To(metav1.NewTime(failedAt.Add(time.Second))), wantCurrent: true}, - {name: "unknown cycle", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, wantCurrent: true}, + {name: "no scheduler receipt uses creation", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, noStatus: true, wantCurrent: true, wantCovered: true}, + {name: "no scheduling start uses creation", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, wantCurrent: true, wantCovered: true}, + {name: "zero scheduling start uses creation", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, start: &metav1.Time{}, wantCurrent: true, wantCovered: true}, + {name: "creation after failure is not covered", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, created: ptr.To(metav1.NewTime(failedAt.Add(time.Second))), noStatus: true, wantCurrent: true}, + {name: "unpublished request is not covered", conditionStatus: metav1.ConditionTrue, reason: pipelineRequestDeadlineExceededReason, generation: 2, created: &metav1.Time{}, noStatus: true, wantCurrent: true}, } { t.Run(tc.name, func(t *testing.T) { t.Log("A durable failure authorizes cleanup of older cycles independently of retry generation") @@ -270,7 +276,16 @@ func TestSchedulingFailureCoversPipelineRequests(t *testing.T) { } deployment.Status.Conditions = []metav1.Condition{condition} } - request := &lpxv1alpha1.LPUPipelineRequest{Status: &lpxv1alpha1.LPUPipelineRequestStatus{SchedulingStartedAt: tc.start}} + request := &lpxv1alpha1.LPUPipelineRequest{ + ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(failedAt.Add(-time.Hour))}, + Status: &lpxv1alpha1.LPUPipelineRequestStatus{SchedulingStartedAt: tc.start}, + } + if tc.created != nil { + request.CreationTimestamp = *tc.created + } + if tc.noStatus { + request.Status = nil + } require.Equal(t, tc.wantCurrent, isSchedulingFailedConditionCurrent(deployment)) require.Equal(t, tc.wantCovered, schedulingFailureCoversPipelineRequests(deployment, []*lpxv1alpha1.LPUPipelineRequest{request})) }) @@ -334,7 +349,7 @@ func TestClearPreviousSchedulingFailure(t *testing.T) { }} } before := deployment.Status.DeepCopy() - clearPreviousSchedulingFailure(deployment) + acknowledgeSchedulingRetry(deployment) if !tc.wantCleared { require.Equal(t, *before, deployment.Status) return diff --git a/deploy/operator/internal/controller/lpx/util.go b/deploy/operator/internal/controller/lpx/util.go index 1e95cd848065..ae55bf9ecfb7 100644 --- a/deploy/operator/internal/controller/lpx/util.go +++ b/deploy/operator/internal/controller/lpx/util.go @@ -12,6 +12,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + autoscalingv1 "k8s.io/api/autoscaling/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -78,33 +79,82 @@ func getPodCliqueSet(ctx context.Context, reader client.Reader, deployment *v1al return pcs, nil } -// getPodCliqueScalingGroup observes the single engine group under PCS ordinal zero. -// reader and pcs must be non-nil; reader must use the cache, and pcs must already -// be owned by this LPXGD. A non-nil result is controlled by that PCS and not -// deleting; nil means the group is absent from the cache or is deleting. -// A group controlled by another PCS is an ownership error. -// Nil does not mean zero replicas, and a zero-replica group is returned normally. -func getPodCliqueScalingGroup(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet) (*grovev1alpha1.PodCliqueScalingGroup, error) { - // Engine replicas belong to one PCSG, not to additional top-level PCS ordinals. +// deletePodCliqueSet deletes the already-validated, non-nil PCS. +func deletePodCliqueSet(ctx context.Context, cl client.Client, pcs *grovev1alpha1.PodCliqueSet) error { + return client.IgnoreNotFound( + cl.Delete( + ctx, + pcs, + // Foreground GC so that blocking LPRs and Grove dependents can be removed. + client.PropagationPolicy(metav1.DeletePropagationForeground), + client.Preconditions{UID: &pcs.UID, ResourceVersion: &pcs.ResourceVersion}, + ), + ) +} + +// getPodCliqueScalingGroups observes all workload groups under PCS ordinal zero. +// reader and pcs must be non-nil; reader uses the cache and pcs is owned by this LPXGD. +// Missing or deleting groups are omitted; a foreign group at an expected name is an +// ownership error, never a cache miss. +func getPodCliqueScalingGroups(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet) (map[string]*grovev1alpha1.PodCliqueScalingGroup, error) { configs := pcs.Spec.Template.PodCliqueScalingGroupConfigs - if len(configs) != 1 { - return nil, fmt.Errorf("LPX PodCliqueSet %q requires exactly one scaling-group template", pcs.Name) + if len(configs) == 0 { + return nil, fmt.Errorf("LPX PodCliqueSet %q requires at least one scaling-group template", pcs.Name) } - name := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, configs[0].Name) - // Establish the group's identity and lifetime once for all downstream operations. - pcsg := &grovev1alpha1.PodCliqueScalingGroup{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: pcs.Namespace, Name: name}, pcsg); err != nil { - return nil, client.IgnoreNotFound(err) + // Select only the current templates under PCS ordinal zero. + expected := make(map[string]struct{}, len(configs)) + for _, config := range configs { + name := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name) + expected[name] = struct{}{} } - if !metav1.IsControlledBy(pcsg, pcs) { - return nil, fmt.Errorf("PodCliqueScalingGroup %q is not controlled by PodCliqueSet %q", pcsg.Name, pcs.Name) + observed := &grovev1alpha1.PodCliqueScalingGroupList{} + if err := reader.List(ctx, observed, client.InNamespace(pcs.Namespace), client.MatchingLabels{grovecommon.LabelPartOfKey: pcs.Name}); err != nil { + return nil, err } - if !pcsg.DeletionTimestamp.IsZero() { - return nil, nil + // Establish each expected group's identity and lifetime once for downstream operations. + groups := make(map[string]*grovev1alpha1.PodCliqueScalingGroup, len(configs)) + for i := range observed.Items { + group := &observed.Items[i] + if _, selected := expected[group.Name]; !selected { + continue + } + if !metav1.IsControlledBy(group, pcs) { + return nil, fmt.Errorf("PodCliqueScalingGroup %q is not controlled by PodCliqueSet %q", group.Name, pcs.Name) + } + if group.DeletionTimestamp.IsZero() { + groups[group.Name] = group + } } + return groups, nil +} + +// scaleDownPodCliqueScalingGroup lowers capacity before asynchronous request cleanup. +// pcsg must be non-nil, owned and not deleting. Call only for explicitly managed +// capacity. Scale-out must wait until terminating request names disappear. +func scaleDownPodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) error { + if replicas >= pcsg.Spec.Replicas { + return nil + } + return scalePodCliqueScalingGroup(ctx, cl, pcsg, replicas) +} - return pcsg, nil +// scalePodCliqueScalingGroup updates the non-nil, already-owned PCSG using its +// observed resource version. Call only for explicitly managed capacity. +// A successful write updates the supplied observation for the rest of this pass. +func scalePodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) error { + if pcsg.Spec.Replicas == replicas { + return nil + } + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ResourceVersion: pcsg.ResourceVersion}, + Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, + } + if err := cl.SubResource("scale").Update(ctx, pcsg, client.WithSubResourceBody(scale)); err != nil { + return fmt.Errorf("scale LPX PodCliqueScalingGroup %q: %w", pcsg.Name, err) + } + pcsg.Spec.Replicas = replicas + return nil } diff --git a/deploy/operator/internal/controller/lpx/util_test.go b/deploy/operator/internal/controller/lpx/util_test.go index ac111d4678fb..7eddbec2fbe6 100644 --- a/deploy/operator/internal/controller/lpx/util_test.go +++ b/deploy/operator/internal/controller/lpx/util_test.go @@ -10,9 +10,12 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" + autoscalingv1 "k8s.io/api/autoscaling/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,13 +24,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) -func TestGetPodCliqueScalingGroup(t *testing.T) { +func TestGetPodCliqueScalingGroups(t *testing.T) { for _, tc := range []struct { name string templates int missing bool previousPCS bool deleting bool + secondGroup bool + listError bool replicas int32 wantGroup bool wantError string @@ -38,19 +43,25 @@ func TestGetPodCliqueScalingGroup(t *testing.T) { {name: "previous PCS incarnation", templates: 1, previousPCS: true, wantError: `PodCliqueScalingGroup "pcs-0-engines" is not controlled by PodCliqueSet "pcs"`}, {name: "deleting previous PCS incarnation", templates: 1, previousPCS: true, deleting: true, wantError: `PodCliqueScalingGroup "pcs-0-engines" is not controlled by PodCliqueSet "pcs"`}, {name: "deleting group", templates: 1, deleting: true}, - {name: "missing template", wantError: "requires exactly one scaling-group template"}, - {name: "multiple templates", templates: 2, wantError: "requires exactly one scaling-group template"}, + {name: "missing template", wantError: "requires at least one scaling-group template"}, + {name: "multiple templates with one observed group", templates: 2, wantGroup: true}, + {name: "multiple observed groups", templates: 2, secondGroup: true, wantGroup: true}, + {name: "list failure", templates: 2, listError: true, wantError: "group observation unavailable"}, } { t.Run(tc.name, func(t *testing.T) { - t.Log("Observe the one group under PCS ordinal zero and establish ownership once") + t.Log("Observe the expected groups under PCS ordinal zero and establish ownership once") pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: "pcs", Namespace: "default", UID: "pcs-uid"}} pcs.Spec.Template.PodCliqueScalingGroupConfigs = make([]grovev1alpha1.PodCliqueScalingGroupConfig, tc.templates) for index := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { pcs.Spec.Template.PodCliqueScalingGroupConfigs[index].Name = "engines" + if index > 0 { + pcs.Spec.Template.PodCliqueScalingGroupConfigs[index].Name = "pending" + } } group := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "pcs-0-engines", Namespace: pcs.Namespace, + Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: tc.replicas}, @@ -67,17 +78,58 @@ func TestGetPodCliqueScalingGroup(t *testing.T) { if !tc.missing { objects = append(objects, group) } - reader := newLPXTestClient(t, objects...) + if tc.secondGroup { + second := group.DeepCopy() + second.Name, second.Spec.Replicas = "pcs-0-pending", 4 + objects = append(objects, second) + } + + t.Log("Ignore unrelated names, higher PCS ordinals, and matching names in other namespaces") + for _, key := range []client.ObjectKey{ + {Namespace: pcs.Namespace, Name: "unrelated"}, + {Namespace: pcs.Namespace, Name: "pcs-1-engines"}, + {Namespace: "other", Name: group.Name}, + } { + objects = append(objects, &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, Namespace: key.Namespace, + }}) + } + lists := 0 + listError := errors.New("group observation unavailable") + reader := interceptor.NewClient(newLPXTestClient(t, objects...), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Fatal("group observation must not issue individual GETs") + return nil + }, + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if tc.listError { + return listError + } + return delegated.List(ctx, list, opts...) + }, + }) t.Log("Only a current non-deleting group supplies capacity, including an explicit zero") - observed, err := getPodCliqueScalingGroup(t.Context(), reader, pcs) + groups, err := getPodCliqueScalingGroups(t.Context(), reader, pcs) + if tc.templates == 0 { + require.Zero(t, lists) + } else { + require.Equal(t, 1, lists) + } + observed := groups[group.Name] if tc.wantError != "" { require.ErrorContains(t, err, tc.wantError) require.Nil(t, observed) + if tc.listError { + require.ErrorIs(t, err, listError) + } return } require.NoError(t, err) + wantGroups := 0 if tc.wantGroup { + wantGroups++ require.NotNil(t, observed) require.Equal(t, group.Name, observed.Name) require.Equal(t, tc.replicas, observed.Spec.Replicas) @@ -85,56 +137,51 @@ func TestGetPodCliqueScalingGroup(t *testing.T) { } else { require.Nil(t, observed) } + if tc.secondGroup { + wantGroups++ + require.NotNil(t, groups["pcs-0-pending"]) + require.EqualValues(t, 4, groups["pcs-0-pending"].Spec.Replicas) + } + require.Len(t, groups, wantGroups) }) } } -func TestLPXCacheMissesPreservePublishedWorkload(t *testing.T) { - for _, missing := range []string{"source", "group"} { - t.Run(missing, func(t *testing.T) { - t.Log("Hide one dependency from the cache while its objects still exist") - child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, desired) - createLPXTestObjects(t, t.Context(), r.Client, objects...) - publishSelectedLPXForTest(t, t.Context(), r, child, desired) - before := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, desired.requests[0].Name) - base := r.Client.(client.WithWatch) - writes := 0 - r.Client = interceptor.NewClient(base, interceptor.Funcs{ - Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - kind := "" - switch object.(type) { - case *v1beta1.DynamoGraphDeployment: - kind = "source" - case *grovev1alpha1.PodCliqueScalingGroup: - kind = "group" - } - if kind == missing { - return apierrors.NewNotFound(grovev1alpha1.SchemeGroupVersion.WithResource(kind).GroupResource(), key.Name) - } - return delegated.Get(ctx, key, object, opts...) - }, - Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { - writes++ - return errors.New("unexpected create during cache miss") - }, - Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { - writes++ - return errors.New("unexpected deletion during cache miss") - }, - }) - - t.Log("Wait without treating a missing cache observation as deletion authority") - for range 2 { - result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.Zero(t, result, "dependency watches resume reconciliation") +func TestLPXMissingSourcePreservesPublishedWorkload(t *testing.T) { + t.Log("Hide the source DGD from the cache while its published workload still exists") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, desired) + before := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, desired.requests[0].Name) + base := r.Client.(client.WithWatch) + writes := 0 + r.Client = interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*v1beta1.DynamoGraphDeployment); ok { + return apierrors.NewNotFound(v1beta1.GroupVersion.WithResource("dynamographdeployments").GroupResource(), key.Name) } - require.Zero(t, writes) - require.Equal(t, before, getTestPipelineRequest(t, t.Context(), base, before.Namespace, before.Name)) - }) + return delegated.Get(ctx, key, object, opts...) + }, + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + writes++ + return errors.New("unexpected create during cache miss") + }, + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + writes++ + return errors.New("unexpected deletion during cache miss") + }, + }) + + t.Log("Wait without treating a missing cache observation as deletion authority") + for range 2 { + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result, "dependency watches resume reconciliation") } + require.Zero(t, writes) + require.Equal(t, before, getTestPipelineRequest(t, t.Context(), base, before.Namespace, before.Name)) } func TestLPXForeignRequestNameIsNeverAdopted(t *testing.T) { @@ -196,11 +243,11 @@ func TestLPXDeletionUsesSnapshotPreconditions(t *testing.T) { pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ Name: "pcs", Namespace: "default", UID: "pcs-uid", ResourceVersion: "1", }} - r := &graphReconciler{Client: newLPXTestClient(t, pcs)} + cl := newLPXTestClient(t, pcs) changed := pcs.DeepCopy() changed.Labels = map[string]string{"example.com/concurrent": "update"} - require.NoError(t, r.Update(t.Context(), changed)) - r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + require.NoError(t, cl.Update(t.Context(), changed)) + cl = interceptor.NewClient(cl, interceptor.Funcs{ Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { options := (&client.DeleteOptions{}).ApplyOptions(opts) require.Equal(t, metav1.DeletePropagationForeground, *options.PropagationPolicy) @@ -211,9 +258,55 @@ func TestLPXDeletionUsesSnapshotPreconditions(t *testing.T) { }) t.Log("Foreground deletion cannot remove a resource changed since observation") - err := r.deletePodCliqueSet(t.Context(), pcs) + err := deletePodCliqueSet(t.Context(), cl, pcs) require.True(t, apierrors.IsConflict(err), "expected a resource-version conflict: %v", err) stored := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), stored)) + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(pcs), stored)) require.Equal(t, changed, stored) } + +func TestScalePodCliqueScalingGroup(t *testing.T) { + conflict := apierrors.NewConflict(consts.PodCliqueScalingGroupGVR.GroupResource(), "group", errors.New("stale version")) + for _, tc := range []struct { + name string + replicas int32 + err error + wantUpdates int + }{ + {name: "scale out", replicas: 12, wantUpdates: 1}, + {name: "scale in", replicas: 2, wantUpdates: 1}, + {name: "scale to zero", replicas: 0, wantUpdates: 1}, + {name: "unchanged", replicas: 9}, + {name: "conflict", replicas: 12, err: conflict, wantUpdates: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Use an already-owned group observation without rendering a workload") + group := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "group", Namespace: "default", ResourceVersion: "7"}, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 9}, + } + updates := 0 + cl := interceptor.NewClient(newLPXTestClient(t), interceptor.Funcs{ + SubResourceUpdate: func(_ context.Context, _ client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "scale", subresource) + require.Same(t, group, object) + scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) + require.Equal(t, group.ResourceVersion, scale.ResourceVersion) + require.Equal(t, tc.replicas, scale.Spec.Replicas) + updates++ + return tc.err + }, + }) + + t.Log("Only explicit changes write scale; successful writes update the in-pass observation") + err := scalePodCliqueScalingGroup(t.Context(), cl, group, tc.replicas) + require.ErrorIs(t, err, tc.err) + require.Equal(t, tc.wantUpdates, updates) + want := int32(9) + if tc.err == nil { + want = tc.replicas + } + require.Equal(t, want, group.Spec.Replicas) + }) + } +} diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 2fb2025b1a1c..dabfb73ed4f0 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -25,61 +25,76 @@ import ( "k8s.io/utils/ptr" ) -// RenderLPXBasePodCliqueSet applies shared Dynamo defaults to the Grove envelope -// and independently owned LPX role templates. The LPX controller completes the -// compiled workload and its identity. All pointer inputs except secretsRetriever -// must be non-nil; preflight supplies the validated workload and its materialization plan. -func RenderLPXBasePodCliqueSet( +// RenderLPXPodCliqueSet constructs the shared LPX Grove envelope without workload +// templates. Its labels include the resolved scheduler queue. +// Pointer inputs must be non-nil and are not mutated. +func RenderLPXPodCliqueSet( ctx context.Context, dynamoDeployment *v1beta1.DynamoGraphDeployment, operatorConfig *configv1alpha1.OperatorConfiguration, runtimeConfig *controller_common.RuntimeConfig, - secretsRetriever SecretsRetriever, - selectedWorkload *dynamolpx.SelectedWorkload, - plan *dynamolpx.MaterializationPlan, -) (*grovev1alpha1.PodCliqueSet, *dynamolpx.RenderInput, error) { - // Reuse the Grove envelope, but keep LPX compilation out of normal rendering. + pcsName string, +) (*grovev1alpha1.PodCliqueSet, error) { + // Reuse the ordinary Grove defaults once for the complete LPX graph. pcs, err := newGrovePodCliqueSet(dynamoDeployment, operatorConfig, runtimeConfig) if err != nil { - return nil, nil, err + return nil, err } - pcs.Name = plan.PodCliqueSetName + + pcs.Name = pcsName + pcs.Spec.Template.StartupType = ptr.To(grovev1alpha1.CliqueStartupTypeExplicit) + queue, err := resolveGroveSchedulerQueue(ctx, dynamoDeployment.Annotations, runtimeConfig) if err != nil { - return nil, nil, err + return nil, err } // Publish the validated queue for the LPX backend's KAI fallback. if queue != "" { - if pcs.Labels == nil { - pcs.Labels = make(map[string]string) - } pcs.Labels[commonconsts.KubeLabelKaiSchedulerQueue] = queue } - // Preserve the complete source graph as the independent role-default context. - component := dynamolpx.ServingComponent(dynamoDeployment) + // Explicit PCS/spec metadata takes precedence over inherited scheduler metadata. + labels, annotations := lpxSchedulingMetadata(dynamoDeployment.Labels), lpxSchedulingMetadata(dynamoDeployment.Annotations) + maps.Copy(labels, pcs.Labels) + maps.Copy(annotations, pcs.Annotations) + pcs.Labels, pcs.Annotations = labels, annotations + return pcs, nil +} - input, gpuCliques, err := renderLPXComponents(cliqueParams{ +// RenderLPXWorkloadTemplates renders one workload's roles, scaling group and resources. +// Pointer inputs must be non-nil, except secretsRetriever when no secrets are used. +// Inputs are not mutated. +func RenderLPXWorkloadTemplates( + source *v1beta1.DynamoGraphDeployment, + operatorConfig *configv1alpha1.OperatorConfiguration, + runtimeConfig *controller_common.RuntimeConfig, + secretsRetriever SecretsRetriever, + workload *dynamolpx.Workload, + plan *dynamolpx.MaterializationPlan, +) (*dynamolpx.WorkloadTemplates, error) { + // Apply Dynamo defaults independently to the selected workload's authored roles. + component := source.GetComponentByName(workload.ServingComponentName()) + input, err := renderLPXComponents(cliqueParams{ component: component, componentName: component.ComponentName, - dynamoDeployment: dynamoDeployment, operatorConfig: operatorConfig, runtimeConfig: runtimeConfig, - secretsRetriever: secretsRetriever, - discoveryBackend: controller_common.GetDiscoveryBackend(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), - discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, dynamoDeployment.Annotations), - validatedQueueName: queue, - }, selectedWorkload, plan) + dynamoDeployment: source, operatorConfig: operatorConfig, runtimeConfig: runtimeConfig, + secretsRetriever: secretsRetriever, + discoveryBackend: controller_common.GetDiscoveryBackend(operatorConfig.Discovery.Backend, source.Annotations), + discoveryContext: NewDiscoveryContext(operatorConfig.Discovery.Backend, source.Annotations), + }, workload, plan) if err != nil { - return nil, nil, err + return nil, err + } + rendered, err := dynamolpx.RenderNodeLocal(workload, plan, *input) + if err != nil { + return nil, err } - pcs.Spec.Template.Cliques = gpuCliques - // Preserve scheduler-visible source metadata below the new owner boundary. - // Explicit PCS/spec metadata retains precedence over inherited DGD metadata. - labels, annotations := lpxSchedulingMetadata(dynamoDeployment.Labels), lpxSchedulingMetadata(dynamoDeployment.Annotations) - maps.Copy(labels, pcs.Labels) - maps.Copy(annotations, pcs.Annotations) - pcs.Labels, pcs.Annotations = labels, annotations - return pcs, input, nil + // Scope the rendered resources to the graph's Kubernetes namespace. + for _, resource := range rendered.Resources { + resource.SetNamespace(source.Namespace) + } + return rendered, nil } const ( @@ -239,7 +254,7 @@ func PCSNameForLPX(deployment *v1alpha1.LPXGraphDeployment) string { // authored role. The full source DGD supplies discovery and shared defaults; // LPX component's roles are returned to the same PCS renderer. // Preflight supplies the non-nil validated workload and materialization plan. -func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, plan *dynamolpx.MaterializationPlan) (*dynamolpx.RenderInput, []*grovev1alpha1.PodCliqueTemplateSpec, error) { +func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dynamolpx.MaterializationPlan) (*dynamolpx.RenderInput, error) { // Pass runtime inputs; deployment identity is stamped only on final resources. input := &dynamolpx.RenderInput{ MinAvailable: p.component.MinAvailable, @@ -252,8 +267,9 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p alphaComponents = alpha.Spec.Services } - gpuCliques := make([]*grovev1alpha1.PodCliqueTemplateSpec, 0, 1) - for _, component := range dynamolpx.Components(p.dynamoDeployment) { + // Render only members of this workload; other graph components supply shared context. + for _, name := range workload.ComponentNames() { + component := p.dynamoDeployment.GetComponentByName(name) alphaComponent := alphaComponents[component.ComponentName] agent := component.ComponentRole(v1beta1.ComponentRoleLPXAgent) lpuRole := lpxRoleComponent(component, agent.PodTemplate, p.dynamoDeployment, p.discoveryBackend) @@ -261,7 +277,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p lpuTemplate, err := renderSelectedLPXRole(lpuRole, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, lpuDefaults) if err != nil { - return nil, nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) + return nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) } lpuTemplate.Labels[dynamolpx.StageLabel] = component.ComponentName input.Stages[component.ComponentName] = *lpuTemplate @@ -276,7 +292,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p input.Conductor, err = renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, lpuDefaults) if err != nil { - return nil, nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) + return nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) } input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName continue @@ -290,9 +306,9 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p role.MinAvailable = nil defaults := &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} if workload.BuildFamily() == dynamolpx.BuildFamilyXT { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(p.dynamoDeployment.Namespace, plan) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(plan) if err != nil { - return nil, nil, err + return nil, err } } gpu := p @@ -302,19 +318,23 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.SelectedWorkload, p gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, defaults) if err != nil { - return nil, nil, fmt.Errorf("rendering %s.conductor: failed to generate podSpec for role %s: %w", component.ComponentName, gpu.r.Name, err) + return nil, fmt.Errorf("rendering %s.conductor: failed to generate podSpec for role %s: %w", component.ComponentName, gpu.r.Name, err) } + + // Select LPX before applying Grove defaults; the PCS already owns the KAI queue. + gpuTemplate.Spec.SchedulerName = dynamolpx.SchedulerName clique, err := buildCliqueFromTemplate(gpu, *gpuTemplate) if err != nil { - return nil, nil, fmt.Errorf("rendering %s.conductor: %w", component.ComponentName, err) + return nil, fmt.Errorf("rendering %s.conductor: %w", component.ComponentName, err) } - // Every GPU worker is required by the same compiled engine replica. + + // Every GPU worker is required by the same compiled workload replica. clique.Spec.MinAvailable = ptr.To(clique.Spec.Replicas) clique.Labels[commonconsts.KubeLabelDynamoComponentType] = string(v1beta1.ComponentTypeLPX) clique.Labels[dynamolpx.StageLabel] = component.ComponentName - gpuCliques = append(gpuCliques, clique) + input.Cyborg = clique } - return input, gpuCliques, nil + return input, nil } func lpxRoleComponent(source *v1beta1.DynamoComponentDeploymentSharedSpec, template *corev1.PodTemplateSpec, dgd *v1beta1.DynamoGraphDeployment, backend configv1alpha1.DiscoveryBackend) *v1beta1.DynamoComponentDeploymentSharedSpec { diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index c2023e3fa554..3caf1554b1f4 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -253,47 +253,49 @@ func TestLPXInputRevision(t *testing.T) { wantChange bool mutate func(*v1beta1.DynamoGraphDeployment) }{ - {"component/name", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).ComponentName = "new-engine" }}, - {"component/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).Replicas = ptr.To(int32(3)) }}, - {"component/min-available", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).MinAvailable = ptr.To(int32(1)) }}, - {"component/namespace", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).GlobalDynamoNamespace = true }}, - {"component/runtime-version", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).RuntimeVersionOverride = "1.5.0" }}, + {"component/name", true, func(d *v1beta1.DynamoGraphDeployment) { d.GetComponentByName("decode").ComponentName = "new-engine" }}, + {"component/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { d.GetComponentByName("decode").Replicas = ptr.To(int32(3)) }}, + {"component/min-available", true, func(d *v1beta1.DynamoGraphDeployment) { d.GetComponentByName("decode").MinAvailable = ptr.To(int32(1)) }}, + {"component/namespace", true, func(d *v1beta1.DynamoGraphDeployment) { d.GetComponentByName("decode").GlobalDynamoNamespace = true }}, + {"component/runtime-version", true, func(d *v1beta1.DynamoGraphDeployment) { + d.GetComponentByName("decode").RuntimeVersionOverride = "1.5.0" + }}, {"component/shared-memory", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).SharedMemorySize = ptr.To(resource.MustParse("16Gi")) + d.GetComponentByName("decode").SharedMemorySize = ptr.To(resource.MustParse("16Gi")) }}, {"component/model", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ModelRef = &v1beta1.ModelReference{Name: "model", Revision: "2"} + d.GetComponentByName("decode").ModelRef = &v1beta1.ModelReference{Name: "model", Revision: "2"} }}, {"component/compilation-cache", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).CompilationCache = &v1beta1.CompilationCacheConfig{PVCName: "cache"} + d.GetComponentByName("decode").CompilationCache = &v1beta1.CompilationCacheConfig{PVCName: "cache"} }}, {"component/topology", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} + d.GetComponentByName("decode").TopologyConstraint = &v1beta1.TopologyConstraint{PackDomain: "rack"} }}, - {"component/build", true, func(d *v1beta1.DynamoGraphDeployment) { lpx.ServingComponent(d).LPX.BuildID = "next-build" }}, + {"component/build", true, func(d *v1beta1.DynamoGraphDeployment) { d.GetComponentByName("decode").LPX.BuildID = "next-build" }}, {"agent/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(int32(4)) + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(int32(4)) }}, {"conductor/replicas", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) }}, {"agent/image", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "agent:next"}} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "agent:next"}} }}, {"conductor/image", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "conductor:next"}} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.Containers = []corev1.Container{{Name: "main", Image: "conductor:next"}} }}, {"agent/placement", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeSelector = map[string]string{"lpu": "new"} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.NodeSelector = map[string]string{"lpu": "new"} }}, {"conductor/placement", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.NodeSelector = map[string]string{"gpu": "new"} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Spec.NodeSelector = map[string]string{"gpu": "new"} }}, {"agent/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Labels = map[string]string{"role": "new"} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Labels = map[string]string{"role": "new"} }}, {"conductor/metadata", true, func(d *v1beta1.DynamoGraphDeployment) { - lpx.ServingComponent(d).ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Annotations = map[string]string{"role": "new"} + d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Annotations = map[string]string{"role": "new"} }}, {"scheduling/deadline", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To(int64(60)) @@ -369,7 +371,7 @@ func TestLPXInputRevision(t *testing.T) { t.Log("Both members independently contribute to the shared LPX revision") pair := source.DeepCopy() - target := lpx.ServingComponent(pair) + target := pair.GetComponentByName("decode") target.Replicas = ptr.To(int32(1)) draft := target.DeepCopy() draft.ComponentName = "small-model" @@ -397,7 +399,7 @@ func TestLPXInputRevisionTracksIndirectRenderMetadata(t *testing.T) { require.NoError(t, err) source := &v1beta1.DynamoGraphDeployment{} require.NoError(t, yaml.Unmarshal(payload, source)) - component := lpx.ServingComponent(source) + component := source.GetComponentByName("lpu") role := lpxRoleComponent(component, component.ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate, source, "") role.ComponentType = v1beta1.ComponentTypeDecode metadata := generatePodMetadata(role, source, getDGDAlphaComponent(source, component.ComponentName), component.ComponentName, DiscoveryContext{}) @@ -422,7 +424,7 @@ func TestLPXInputRevisionTracksIndirectRenderMetadata(t *testing.T) { } require.NoError(t, alpha.ConvertTo(changed)) } - require.Equal(t, component, lpx.ServingComponent(changed)) + require.Equal(t, component, changed.GetComponentByName("lpu")) nextMetadata := generatePodMetadata(role, changed, getDGDAlphaComponent(changed, component.ComponentName), component.ComponentName, DiscoveryContext{}) if change == "alpha-annotation" { require.NotEqual(t, metadata.Annotations, nextMetadata.Annotations) diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index de7abac052f4..9636c0fc1116 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -6,6 +6,7 @@ package dynamo import ( "context" "fmt" + "slices" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" @@ -20,16 +21,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -// EvaluateLPXGroveReadiness observes one complete engine, including every role -// in every replica. Ordinary DGD components are deliberately not evaluated here. -// source is non-nil. pcs and group may be nil while materializing; otherwise the -// caller has verified ownership, deletion state, and the single-group template. -func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, group *grovev1alpha1.PodCliqueScalingGroup) (GroveReadiness, error) { - component := lpx.ServingComponent(source) +// EvaluateLPXGroveReadiness observes every role and replica of one component group. +// source is non-nil and admitted; groupName and componentNames identify one entry +// from ComponentGroups. pcs and group may be nil while materializing; otherwise +// the caller has verified ownership and deletion state. reader may be nil until +// both pcs and group are available. Inputs remain read-only. +func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, groupName string, componentNames []string, pcs *grovev1alpha1.PodCliqueSet, pcsg *grovev1alpha1.PodCliqueScalingGroup) (GroveReadiness, error) { + component := source.GetComponentByName(groupName) status := v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, RuntimeNamespace: source.GetDynamoNamespaceForComponent(component)} // Draft instances are counted from their own complete Agent cliques. statuses := make(map[string]v1beta1.ComponentReplicaStatus) - for _, member := range lpx.Components(source) { + for _, name := range componentNames { + member := source.GetComponentByName(name) if member.ComponentName != component.ComponentName { statuses[member.ComponentName] = v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindPodClique, RuntimeNamespace: source.GetDynamoNamespaceForComponent(member), @@ -59,20 +62,18 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source if hash == nil { return pending("Waiting for Grove to accept the LPX PodCliqueSet revision"), nil } - config := pcs.Spec.Template.PodCliqueScalingGroupConfigs[0] - groupName := grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name) - status.ComponentNames = []string{groupName} - if group == nil { + if pcsg == nil { return pending("Waiting for the LPX scaling group"), nil } - status.Replicas, status.UpdatedReplicas = group.Status.Replicas, group.Status.UpdatedReplicas - status.AvailableReplicas = ptr.To(group.Status.AvailableReplicas) - if group.Status.ObservedGeneration == nil || *group.Status.ObservedGeneration != group.Generation { + status.ComponentNames = []string{pcsg.Name} + status.Replicas, status.UpdatedReplicas = pcsg.Status.Replicas, pcsg.Status.UpdatedReplicas + status.AvailableReplicas = ptr.To(pcsg.Status.AvailableReplicas) + if pcsg.Status.ObservedGeneration == nil || *pcsg.Status.ObservedGeneration != pcsg.Generation { return pending("Waiting for the exact observed LPX scaling group"), nil } - status.ScheduledReplicas = ptr.To(group.Status.ScheduledReplicas) - replicas := ptr.Deref(component.Replicas, group.Spec.Replicas) - if group.Spec.Replicas != replicas || group.Status.CurrentPodCliqueSetGenerationHash == nil || *group.Status.CurrentPodCliqueSetGenerationHash != *hash { + status.ScheduledReplicas = ptr.To(pcsg.Status.ScheduledReplicas) + replicas := ptr.Deref(component.Replicas, pcsg.Spec.Replicas) + if pcsg.Spec.Replicas != replicas || pcsg.Status.CurrentPodCliqueSetGenerationHash == nil || *pcsg.Status.CurrentPodCliqueSetGenerationHash != *hash { return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity"), nil } // Observe every member before returning so partial draft readiness remains visible. @@ -85,9 +86,12 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source for replica := int32(0); replica < replicas; replica++ { replicaReady := true for _, template := range pcs.Spec.Template.Cliques { - name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: groupName, Replica: int(replica)}, template.Name) + if !slices.Contains(pcsg.Spec.CliqueNames, template.Name) { + continue + } + name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(replica)}, template.Name) memberName := template.Labels[lpx.StageLabel] - readiness, err := observeLPXRole(ctx, reader, group, name, template.Spec.Replicas) + readiness, err := observeLPXRole(ctx, reader, pcsg, name, template.Spec.Replicas) if err != nil { return GroveReadiness{}, err } @@ -114,7 +118,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source if unreadyMessage != "" { return result(false, unreadyClassification, unreadyMessage), nil } - ready, message, classification := pcsgStatusReady(group, replicas) + ready, message, classification := pcsgStatusReady(pcsg, replicas) if ready { classification = v1beta1.DGDReadyReasonAllResourcesReady } diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go index 525d2150c961..4be72e576b35 100644 --- a/deploy/operator/internal/dynamo/grove_lpx_test.go +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -7,8 +7,10 @@ import ( "testing" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T) { @@ -33,9 +35,21 @@ func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{draft, serving}}, } - readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, nil, nil) - require.NoError(t, err) - require.NotEqual(t, source.GetDynamoNamespaceForComponent(&draft), source.GetDynamoNamespaceForComponent(&serving)) - require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) - require.Equal(t, source.GetDynamoNamespaceForComponent(&serving), readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) + t.Log("Preserve component namespaces while the PCS or its scaling group is still missing") + for _, pcs := range []*grovev1alpha1.PodCliqueSet{ + nil, + { + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Status: grovev1alpha1.PodCliqueSetStatus{ + ObservedGeneration: ptr.To(int64(1)), CurrentGenerationHash: ptr.To("accepted"), + }, + }, + } { + readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, "serving", []string{"draft", "serving"}, pcs, nil) + require.NoError(t, err) + require.False(t, readiness.Ready) + require.NotEqual(t, source.GetDynamoNamespaceForComponent(&draft), source.GetDynamoNamespaceForComponent(&serving)) + require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) + require.Equal(t, source.GetDynamoNamespaceForComponent(&serving), readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) + } } diff --git a/deploy/operator/internal/dynamo/lpx/components.go b/deploy/operator/internal/dynamo/lpx/components.go new file mode 100644 index 000000000000..fd1a65947238 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/components.go @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "slices" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// Components returns LPX components in authored order without mutating dgd. +// dgd must be non-nil. +// Returned component pointers remain read-only. +func Components(dgd *dynamov1beta1.DynamoGraphDeployment) []*dynamov1beta1.DynamoComponentDeploymentSharedSpec { + components := make([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, 0, len(dgd.Spec.Components)) + for index := range dgd.Spec.Components { + component := &dgd.Spec.Components[index] + if component.IsLPX() { + components = append(components, component) + } + } + return components +} + +// ComponentGroups returns workload membership keyed by conductor component name. +// Each conductor owns a group. Admission permits an agent-only component only +// when it shares the graph's sole conductor. Member names are sorted. +// dgd must be non-nil and have passed admission; it is not mutated. +func ComponentGroups(dgd *dynamov1beta1.DynamoGraphDeployment) map[string][]string { + components := Components(dgd) + groups := make(map[string][]string) + + for _, component := range components { + groupName := component.ComponentName + + // Admission makes the shared conductor unambiguous for agent-only members. + if component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) == nil { + for _, targetComponent := range components { + if targetComponent.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { + groupName = targetComponent.ComponentName + break + } + } + } + + groups[groupName] = append(groups[groupName], component.ComponentName) + } + + // Keep membership deterministic without changing the authored component order. + for _, members := range groups { + slices.Sort(members) + } + + return groups +} diff --git a/deploy/operator/internal/dynamo/lpx/components_test.go b/deploy/operator/internal/dynamo/lpx/components_test.go new file mode 100644 index 000000000000..79cc5aae1701 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/components_test.go @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +import ( + "fmt" + "slices" + "testing" + + dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { + t.Log("Select components by their declared type, independently of build or templates") + canonicalLPX := []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentType: dynamov1beta1.ComponentTypeLPX}} + + t.Log("Define default-selection scenarios") + tests := []struct { + name string + annotations map[string]string + components []dynamov1beta1.DynamoComponentDeploymentSharedSpec + want bool + }{ + {name: "canonical LPX", components: canonicalLPX, want: true}, + { + name: "component provider", + annotations: map[string]string{ + commonconsts.KubeAnnotationWorkloadProvider: commonconsts.WorkloadProviderComponent, + }, + components: canonicalLPX, + want: true, + }, + { + name: "Grove opt-out", + annotations: map[string]string{ + commonconsts.KubeAnnotationEnableGrove: "FALSE", + }, + components: canonicalLPX, + want: true, + }, + {name: "non-LPX", components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: dynamov1beta1.ComponentRoleLPXAgent}}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Log("Construct the DGD selection input") + dgd := &dynamov1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Annotations: test.annotations}, + Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ + Components: test.components, + Scheduling: &dynamov1beta1.SchedulingSpec{}, + }, + } + + t.Log("Resolve and verify scheduler selection") + selected := dgd.HasLPXComponent() + require.Equal(t, test.want, selected) + }) + } +} + +func TestComponentGroups(t *testing.T) { + for _, test := range []struct { + name string + conductors []bool + want map[string][]string + }{ + {name: "no LPX components", want: map[string][]string{}}, + {name: "single workload", conductors: []bool{true}, want: map[string][]string{"component-0": {"component-0"}}}, + {name: "two independent workloads", conductors: []bool{true, true}, want: map[string][]string{"component-0": {"component-0"}, "component-1": {"component-1"}}}, + {name: "three independent workloads", conductors: []bool{true, true, true}, want: map[string][]string{"component-0": {"component-0"}, "component-1": {"component-1"}, "component-2": {"component-2"}}}, + {name: "shared conductor first", conductors: []bool{true, false}, want: map[string][]string{"component-0": {"component-0", "component-1"}}}, + {name: "shared conductor last", conductors: []bool{false, true}, want: map[string][]string{"component-1": {"component-0", "component-1"}}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Author LPX members beside an ordinary frontend") + dgd := newSelectedTestDGD(t, "graph", dynamov1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", ComponentType: dynamov1beta1.ComponentTypeFrontend}) + for i, conductor := range test.conductors { + component := testLPXComponent(fmt.Sprintf("component-%d", i), "build", dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}) + if conductor { + component.Roles = append(component.Roles, dynamov1beta1.ComponentRoleSpec{Name: dynamov1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}) + } + dgd.Spec.Components = append(dgd.Spec.Components, component) + } + + t.Log("Group every LPX member by its conductor without mutating the graph") + before := dgd.DeepCopy() + require.Equal(t, test.want, ComponentGroups(dgd)) + require.Equal(t, before, dgd) + + t.Log("Reordering authored components preserves group identity and membership") + slices.Reverse(dgd.Spec.Components) + require.Equal(t, test.want, ComponentGroups(dgd)) + }) + } +} + +// singleGroupComponents reads the one admitted workload in a test fixture. +func singleGroupComponents(t *testing.T, dgd *dynamov1beta1.DynamoGraphDeployment) []string { + t.Helper() + groups := ComponentGroups(dgd) + require.Len(t, groups, 1) + for _, components := range groups { + return components + } + return nil +} diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go index 7bc8c6247570..edfef0eb649e 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render_test.go @@ -20,20 +20,22 @@ import ( func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { t.Parallel() - t.Log("Project a hybrid engine with the longest PCS name and maximum scheduling replica count") + t.Log("Project a hybrid workload at the combined name limit and maximum scheduling replica count") fixture := newV3CompilerFixture() fixture.compilationMode = manifestcapnp.CompilationMode_lpx projection := projectRenderFixture(t, PipelineLPX, acquireTestSnapshot(t, writeCompilerFixture(t, fixture))) projection.stage = testRenderComponentName projection.configuredBuild.IOFPGACount = 1 projection.configuredBuild.IOFanoutFactor = 1 - workload := &SelectedWorkload{ + workload := &Workload{ modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 2496, } - pcsName := strings.Repeat("a", MaxPodCliqueSetNameLength) + pcsName := strings.Repeat("a", 28) // target + target-cond consume the remaining Grove budget. plan, err := workload.PlanNodeLocalMaterialization(pcsName) require.NoError(t, err) + plan, err = plan.WithGroup("target") + require.NoError(t, err) t.Log("Validate actual GPU widths rather than assuming the largest int32 pod index") for _, test := range []struct { @@ -50,9 +52,11 @@ func TestRenderHybridBoundsActualGPUHostnames(t *testing.T) { pcs := renderTestPCS(true) pcs.Name = pcsName cyborg := namedClique(t, pcs, "cond") + cyborg.Name, cyborg.Spec.RoleName = plan.CyborgTemplate, plan.CyborgTemplate cyborg.Spec.Replicas = test.replicas cyborg.Spec.MinAvailable = ptr.To(test.replicas) - _, err := RenderSelectedNodeLocal(pcs, workload, plan, RenderInput{ + _, err := RenderNodeLocal(workload, plan, RenderInput{ + Cyborg: cyborg, Stages: map[string]corev1.PodTemplateSpec{testRenderComponentName: {Spec: renderTestPodSpec()}}, }) if test.wantError { diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index b5193e453b8e..93d8b3b2f829 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -14,15 +14,12 @@ import ( // RenderCyborgConfigMap renders the XT hybrid Agent endpoints. // The workload and plan must be non-nil, validated, and describe the selected hybrid model. -// The workload and plan are not mutated. -func (w *SelectedWorkload) RenderCyborgConfigMap( - namespace string, - plan *MaterializationPlan, -) (*corev1.ConfigMap, error) { +// The workload and plan are not mutated. The caller assigns the returned ConfigMap's namespace. +func (w *Workload) RenderCyborgConfigMap(plan *MaterializationPlan) (*corev1.ConfigMap, error) { build := &w.modelProjections[0].configuredBuild - // Cyborg supplies the PCS prefix; startup supplies this engine's Grove index. - serverPrefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" + // Cyborg supplies the PCS prefix; startup supplies this workload's Grove index. + serverPrefix := plan.ScalingGroupTemplate + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(build.Partitions)) offset := 0 for index, partition := range build.Partitions { @@ -30,7 +27,7 @@ func (w *SelectedWorkload) RenderCyborgConfigMap( offset += partition.effectiveNodeCount() } - return renderRuntimeConfigMap(namespace, plan.PodCliqueSetName+"-decode", map[string]string{ + return renderRuntimeConfigMap(plan.ResourcePrefix+"-decode", map[string]string{ "lpu_servers": strings.Join(servers, "\n"), }) } diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 7f9f8567c85c..2e62076c0472 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -31,7 +31,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { require.NoError(t, err) projection := projectionBatch[0] projection.stage = testRenderComponentName - workload := &SelectedWorkload{ + workload := &Workload{ modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1, } @@ -39,16 +39,16 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Render the generated Agent endpoints") plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - initial, err := workload.RenderCyborgConfigMap("test-namespace", plan) + initial, err := workload.RenderCyborgConfigMap(plan) require.NoError(t, err) - t.Log("Verify every engine replica addresses only its own Agents") + t.Log("Verify every workload replica addresses only its own Agents") for _, replicas := range []int32{1, 2, 10, 12} { t.Run(strconv.Itoa(int(replicas)), func(t *testing.T) { workload.scalingGroupReplicas = replicas scaledPlan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - configMap, err := workload.RenderCyborgConfigMap("test-namespace", scaledPlan) + configMap, err := workload.RenderCyborgConfigMap(scaledPlan) require.NoError(t, err) require.Equal(t, initial, configMap) require.True(t, *configMap.Immutable) @@ -56,7 +56,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { require.Equal(t, prefix+"agt-0\n"+prefix+"agt-2", configMap.Data["lpu_servers"]) require.Len(t, configMap.Data, 1) - t.Log("Resolve Cyborg server addresses to the last engine replica's actual Agent hostnames") + t.Log("Resolve Cyborg server addresses to the last workload replica's actual Agent hostnames") lastReplica := scaledPlan.ForReplica(replicas - 1) servers := strings.Split(strings.ReplaceAll(configMap.Data["lpu_servers"], "${GROVE_PCSG_INDEX}", strconv.Itoa(int(replicas-1))), "\n") for index, offset := range []int{0, 2} { @@ -100,12 +100,12 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { projection.stage = testRenderComponentName t.Log("Render only projected endpoints without compressing their physical Agent offsets") - workload := &SelectedWorkload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1} + workload := &Workload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: 1} plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) require.Len(t, projection.RequestSpec(plan, "agents").Partitions, test.partitions) require.Equal(t, 2*test.partitions, projection.agentReplicas) - configMap, err := workload.RenderCyborgConfigMap("test", plan) + configMap, err := workload.RenderCyborgConfigMap(plan) require.NoError(t, err) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(test.wantOffsets)) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 30462226ad0f..8087d58e6ffb 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -18,24 +18,32 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) const lpuConfigVolumeName = "config" // LPUConfigMapName names the immutable runtime table using its Pod-template content hash. -// The root is the PCS identity carried by Grove's part-of Pod label. +// The root is the workload resource prefix, including its group when independent workloads share a PCS. func LPUConfigMapName(root, configHash string) string { return fmt.Sprintf("%s-lpu-%.16s", root, configHash) } -func renderRuntimeConfigMap(namespace, namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { +// LPUAgentConfigMapName returns the runtime table referenced by a non-nil Agent Pod. +// The cached Pod projection must preserve ConfigMap volume names. Reading the Pod's +// reference keeps old runtime revisions addressable across workload template updates. +func LPUAgentConfigMapName(pod *corev1.Pod) (string, error) { + for _, volume := range pod.Spec.Volumes { + if volume.Name == lpuConfigVolumeName && volume.ConfigMap != nil && volume.ConfigMap.Name != "" { + return volume.ConfigMap.Name, nil + } + } + return "", fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap volume", pod.Namespace, pod.Name) +} + +func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - }, Immutable: ptr.To(true), Data: data, } diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 5c4bf7084232..1c8d12b63da3 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -20,12 +20,12 @@ func TestRenderRuntimeConfigMapSizeLimit(t *testing.T) { "text": strings.Repeat("é", corev1.MaxSecretSize/2-1), "suffix": "é", } - _, err := renderRuntimeConfigMap("test-namespace", "test-lpu", data) + _, err := renderRuntimeConfigMap("test-lpu", data) require.NoError(t, err) t.Log("Reject one extra byte with a named error and no usable ConfigMap") data["suffix"] += "x" - configMap, err := renderRuntimeConfigMap("test-namespace", "test-lpu", data) + configMap, err := renderRuntimeConfigMap("test-lpu", data) require.Nil(t, configMap) require.ErrorContains(t, err, `rendered LPX ConfigMap "test-lpu-`) require.ErrorContains(t, err, "data is 1048577 bytes; maximum is 1048576") @@ -36,9 +36,9 @@ func TestLPXRuntimeConfigNamesMatchPodIdentity(t *testing.T) { t.Run(root, func(t *testing.T) { t.Log("Render immutable runtime tables for PCS and Pod-label identity bounds") data := map[string]string{"runtime": "config"} - lpu, err := renderRuntimeConfigMap("test-namespace", root+"-lpu", data) + lpu, err := renderRuntimeConfigMap(root+"-lpu", data) require.NoError(t, err) - decode, err := renderRuntimeConfigMap("test-namespace", root+"-decode", data) + decode, err := renderRuntimeConfigMap(root+"-decode", data) require.NoError(t, err) t.Log("Resolve the same LPU table from Pod identity and preserve each role suffix") diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 3646274e0431..6caa04bcf91d 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -6,6 +6,7 @@ package lpx import ( + "crypto/sha256" "fmt" "slices" "strings" @@ -18,11 +19,12 @@ import ( const ( lpxScalingGroupTemplateName = "lpx" conductorTemplateName = "cond" - maxEngineReplicas = 2496 + maxWorkloadReplicas = 2496 + minGroupNameLength = 6 ) -// MaxPodCliqueSetNameLength reserves Grove's combined name budget for the fixed -// scaling group and longest role name, independently of the selected components. +// MaxPodCliqueSetNameLength preserves the established PCS identity budget. +// WithGroup separately validates the extra names needed by multiple workloads. const MaxPodCliqueSetNameLength = commonconsts.MaxCombinedGroveResourceNameLength - len(lpxScalingGroupTemplateName) - len(conductorTemplateName) @@ -41,6 +43,10 @@ type ExpectedAgent struct { // MaterializationPlan is the deterministic identity projection shared by the // renderer and the controller's API-object observer. type MaterializationPlan struct { + // ScalingGroupTemplate is unique within the shared PCS. + ScalingGroupTemplate string + // ResourcePrefix scopes runtime ConfigMaps and discovery Services to this workload. + ResourcePrefix string // PodCliqueSetName is the actual PCS used for every generated child address. PodCliqueSetName string // ConductorClique is the materialized conductor PodClique name, if present. @@ -63,7 +69,7 @@ type MaterializationPlan struct { // PlanNodeLocalMaterialization derives the exact identities shared by graph // rendering and lifecycle observation without mutating the workload. -func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*MaterializationPlan, error) { +func (w *Workload) PlanNodeLocalMaterialization(pcsName string) (*MaterializationPlan, error) { // Leave room for the fixed scaling group and every LPX role in Grove's name budget. if strings.TrimSpace(pcsName) == "" { return nil, fmt.Errorf("PodCliqueSet name is required") @@ -88,10 +94,12 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi // Construct the plan with Grove's canonical replica-zero scaling-group identity. plan := &MaterializationPlan{ - PodCliqueSetName: pcsName, - ConductorTemplate: conductorTemplate, - CyborgTemplate: cyborgTemplate, - Agents: agents, + PodCliqueSetName: pcsName, + ScalingGroupTemplate: lpxScalingGroupTemplateName, + ResourcePrefix: pcsName, + ConductorTemplate: conductorTemplate, + CyborgTemplate: cyborgTemplate, + Agents: agents, LPXScalingGroup: grovecommon.GeneratePodCliqueScalingGroupName( grovecommon.ResourceNameReplica{Name: pcsName, Replica: 0}, lpxScalingGroupTemplateName, ), @@ -102,10 +110,10 @@ func (w *SelectedWorkload) PlanNodeLocalMaterialization(pcsName string) (*Materi return plan, plan.ValidateReplicaCount() } -// ValidateReplicaCount bounds per-engine allocations and checks Pod hostnames. +// ValidateReplicaCount bounds per-workload allocations and checks Pod hostnames. func (p *MaterializationPlan) ValidateReplicaCount() error { - if p.Replicas < 0 || p.Replicas > maxEngineReplicas { - return fmt.Errorf("LPX replica count must be between 0 and %d", maxEngineReplicas) + if p.Replicas < 0 || p.Replicas > maxWorkloadReplicas { + return fmt.Errorf("LPX replica count must be between 0 and %d", maxWorkloadReplicas) } if p.ConductorTemplate != "" { if err := p.validatePodHostname("conductor", p.ConductorTemplate, 0); err != nil { @@ -170,3 +178,64 @@ func (p *MaterializationPlan) validatePodHostname(role, templateName string, pod func materializedPodHostname(cliqueName string, podIndex int) string { return fmt.Sprintf("%s-%d", cliqueName, podIndex) } + +// WithGroup scopes a workload's resource names within a multi-workload PCS. +// Sole workloads retain the unscoped plan's established names. +// The receiver is a validated non-nil plan and groupName is an admitted, +// non-empty conductor component name from ComponentGroups. +// The receiver is not mutated; names that cannot fit Grove's budget are rejected. +func (p *MaterializationPlan) WithGroup(groupName string) (*MaterializationPlan, error) { + // Reserve the group prefix in both the scaling group and its longest role. + roleLength := max(len(p.ConductorTemplate), len(p.CyborgTemplate)) + for _, agent := range p.Agents { + roleLength = max(roleLength, len(agent.TemplateName)) + } + groupNameLength := (commonconsts.MaxCombinedGroveResourceNameLength - len(p.PodCliqueSetName) - roleLength - 1) / 2 + name, err := boundedGroupName(groupName, groupNameLength) + if err != nil { + return nil, fmt.Errorf("naming group %q in PCS %q: %w", groupName, p.PodCliqueSetName, err) + } + + // ConfigMaps and Services retain more of the group name than Grove allows. + const maxResourcePrefixLength = validation.DNS1123LabelMaxLength - len("-serve") + resourceName, err := boundedGroupName(groupName, maxResourcePrefixLength-len(p.PodCliqueSetName)-1) + if err != nil { + return nil, err + } + out := p.ForReplica(0) + out.ResourcePrefix = p.PodCliqueSetName + "-" + resourceName + out.ScalingGroupTemplate = name + out.LPXScalingGroup = grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: p.PodCliqueSetName, Replica: 0}, name) + + // Every role remains unique within the shared PCS and readable in Grove objects. + if out.ConductorTemplate != "" { + out.ConductorTemplate = name + "-" + out.ConductorTemplate + } + if out.CyborgTemplate != "" { + out.CyborgTemplate = name + "-" + out.CyborgTemplate + } + for i := range out.Agents { + out.Agents[i].TemplateName = name + "-" + out.Agents[i].TemplateName + } + out = out.ForReplica(0) + return out, out.ValidateReplicaCount() +} + +// boundedGroupName preserves admitted component names when they fit. Hashes retain +// the original identity when lowercasing or shortening; at least one readable +// character and four hash characters must fit alongside the separator. +func boundedGroupName(component string, maxLength int) (string, error) { + name := strings.ToLower(component) + if name == component && len(name) <= maxLength { + return name, nil + } + if maxLength < minGroupNameLength { + return "", fmt.Errorf("component name needs more than %d characters; shorten the deployment name", maxLength) + } + + // Prefer eight hash characters, reducing only for Grove's tighter name budget. + digest := sha256.Sum256([]byte(component)) + suffix := fmt.Sprintf("%x", digest[:4])[:min(8, maxLength-2)] + prefixLength := min(len(name), maxLength-len(suffix)-1) + return strings.TrimRight(name[:prefixLength], "-") + "-" + suffix, nil +} diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index 4b5dc16e9325..44742fbb4c6c 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -48,7 +48,7 @@ func TestPlanMaterializationBounds(t *testing.T) { } projections[index] = &projection } - workload := &SelectedWorkload{ + workload := &Workload{ modelProjections: projections, scalingGroupReplicas: 2496, } @@ -63,8 +63,8 @@ func TestPlanMaterializationBounds(t *testing.T) { if lastReplica.ConductorClique != "" { require.Equal(t, "cond", plan.ConductorTemplate) hostnames = append(hostnames, materializedPodHostname(lastReplica.ConductorClique, 0)) - require.Equal(t, commonconsts.MaxCombinedGroveResourceNameLength, - len(pcsName)+len(lpxScalingGroupTemplateName)+len(plan.ConductorTemplate)) + require.LessOrEqual(t, len(pcsName)+len(lpxScalingGroupTemplateName)+len(plan.ConductorTemplate), + commonconsts.MaxCombinedGroveResourceNameLength) } for _, agent := range lastReplica.Agents { require.LessOrEqual(t, len(pcsName)+len(lpxScalingGroupTemplateName)+len(agent.TemplateName), commonconsts.MaxCombinedGroveResourceNameLength) @@ -90,7 +90,7 @@ func TestPlanMaterializationBounds(t *testing.T) { _, err = workload.PlanNodeLocalMaterialization(pcsName + "a") require.ErrorContains(t, err, "exceeds the LPX maximum of 38 characters") - t.Log("Bound engine replicas before allocating per-engine request state") + t.Log("Bound workload replicas before allocating per-workload request state") for _, replicas := range []int32{-1, 0, 1, 2496, 2497, math.MaxInt32} { workload.scalingGroupReplicas = replicas _, err := workload.PlanNodeLocalMaterialization(pcsName) @@ -99,3 +99,81 @@ func TestPlanMaterializationBounds(t *testing.T) { }) } } + +func TestWorkloadNames(t *testing.T) { + t.Log("Project a real workload at the maximum scaling-group replica count") + projection := projectRenderFixture(t, PipelineSingle, acquireTestSnapshot(t, writeV2CompilerFixture(t))) + workload := &Workload{modelProjections: []*ModelProjection{projection}, scalingGroupReplicas: maxWorkloadReplicas} + for _, pcsName := range []string{"model", strings.Repeat("p", 26)} { + t.Run(pcsName, func(t *testing.T) { + t.Log("Keep shared-runtime resource names and the PCS identity") + plan, err := workload.PlanNodeLocalMaterialization(pcsName) + require.NoError(t, err) + before := plan.ForReplica(0) + + t.Log("Use readable component names while bounding and distinguishing shortened names") + names := make(map[string]bool) + groups := make(map[string]bool) + for _, component := range []string{ + "draft", "target", "Draft", "DRAFT", + strings.Repeat("long-", 12) + "one", strings.Repeat("long-", 12) + "two", + } { + scoped, err := plan.WithGroup(component) + require.NoError(t, err) + require.Equal(t, pcsName, scoped.PodCliqueSetName) + require.True(t, strings.HasPrefix(scoped.ResourcePrefix, pcsName+"-")) + require.False(t, names[scoped.ResourcePrefix], "duplicate prefix for %s", component) + names[scoped.ResourcePrefix] = true + require.False(t, groups[scoped.ScalingGroupTemplate], "duplicate group for %s", component) + groups[scoped.ScalingGroupTemplate] = true + require.Empty(t, validation.IsDNS1035Label(scoped.ResourcePrefix+"-serve")) + if component == "draft" || component == "target" { + require.Equal(t, pcsName+"-"+component, scoped.ResourcePrefix) + require.Equal(t, component, scoped.ScalingGroupTemplate) + require.Equal(t, component+"-cond", scoped.ConductorTemplate) + require.Equal(t, component+"-agt", scoped.Agents[0].TemplateName) + } else { + require.Regexp(t, "-(draft|long).*-[a-f0-9]{8}$", scoped.ResourcePrefix) + require.Regexp(t, "^[a-z].*-[a-f0-9]{4,8}$", scoped.ScalingGroupTemplate) + } + + t.Log("Keep Grove's combined names and the last replica's Pod hostnames valid") + last := scoped.ForReplica(scoped.Replicas - 1) + require.Empty(t, validation.IsDNS1123Label(materializedPodHostname(last.ConductorClique, 0))) + require.LessOrEqual(t, len(pcsName)+len(scoped.ScalingGroupTemplate)+len(scoped.ConductorTemplate), commonconsts.MaxCombinedGroveResourceNameLength) + for _, agent := range last.Agents { + require.LessOrEqual(t, len(pcsName)+len(scoped.ScalingGroupTemplate)+len(agent.TemplateName), commonconsts.MaxCombinedGroveResourceNameLength) + require.Empty(t, validation.IsDNS1123Label(materializedPodHostname(agent.CliqueName, agent.Replicas-1))) + } + } + require.Equal(t, before, plan) + }) + } + + t.Log("Use the remaining Grove budget for named workloads") + for _, test := range []struct { + name string + pcsLength int + component string + wantError bool + }{ + {name: "readable at limit", pcsLength: 28, component: "target"}, + {name: "hashed at limit", pcsLength: 28, component: "TARGET"}, + {name: "sole-workload name leaves too little room", pcsLength: 38, component: "target", wantError: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Scope the workload without renaming its PCS") + plan, err := workload.PlanNodeLocalMaterialization(strings.Repeat("p", test.pcsLength)) + require.NoError(t, err) + scoped, err := plan.WithGroup(test.component) + if test.wantError { + require.ErrorContains(t, err, "shorten the deployment name") + require.Equal(t, strings.Repeat("p", test.pcsLength), plan.PodCliqueSetName) + return + } + require.NoError(t, err) + require.Equal(t, commonconsts.MaxCombinedGroveResourceNameLength, + len(scoped.PodCliqueSetName)+len(scoped.ScalingGroupTemplate)+len(scoped.ConductorTemplate)) + }) + } +} diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index 41ead7f7985d..38932090422a 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -10,7 +10,6 @@ import ( "fmt" "slices" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "k8s.io/apimachinery/pkg/runtime" ) @@ -22,8 +21,6 @@ const ( DGDGenerationAnnotation = "scheduling.lpu.nvidia.com/dgd-generation" // DGDUIDAnnotation records the source DGD UID on rendered objects. DGDUIDAnnotation = "scheduling.lpu.nvidia.com/dgd-uid" - // ExecutionBackendAnnotation selects the LPX execution backend on rendered objects. - ExecutionBackendAnnotation = commonconsts.KubeAnnotationLPXExecutionBackend // WorkloadModeAnnotation records the projected LPX workload mode on rendered objects. WorkloadModeAnnotation = "scheduling.lpu.nvidia.com/workload-mode" ) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index eb362459e124..f43bec799851 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -31,37 +31,44 @@ const StageLabel = "lpx.nvidia.com/stage" type RenderInput struct { // CyborgConfigMap is rendered before the user's Cyborg defaults are merged. CyborgConfigMap *corev1.ConfigMap - // MinAvailable is the minimum number of complete engine replicas in the gang. + // MinAvailable is the minimum number of complete workload replicas in the gang. MinAvailable *int32 // Stages contains an independently merged LPU template for every projected stage. Stages map[string]corev1.PodTemplateSpec // Conductor supplies a fresh, independently merged template for the shared LPU // conductor. Required for non-hybrid pipelines; hybrid pipelines leave it nil - // and supply their independently rendered Cyborg clique in pcs. + // and supply their independently rendered Cyborg clique below. Conductor *corev1.PodTemplateSpec + // Cyborg supplies the GPU clique for hybrid pipelines; nil for LPU-only workloads. + Cyborg *grovev1alpha1.PodCliqueTemplateSpec } -// RenderSelectedNodeLocal consumes fresh graph and render inputs. pcs, workload, -// and plan must be non-nil. The function mutates pcs and may mutate the -// reference-backed stage and conductor template contents in input; callers must -// pass independently owned values. pcs contains only the optional Cyborg clique -// rendered from the same workload, with no scaling groups. workload and plan are -// read without mutation. +// WorkloadTemplates contains only the templates and resources contributed by one workload. +type WorkloadTemplates struct { + Cliques []*grovev1alpha1.PodCliqueTemplateSpec + ScalingGroup grovev1alpha1.PodCliqueScalingGroupConfig + Resources []client.Object +} + +// RenderNodeLocal renders a workload without constructing a PCS. workload +// and plan must be non-nil and are read without mutation. The function may mutate +// the reference-backed templates in input; callers must pass fresh owned values. +// The caller assigns namespaces to the returned resources. // //nolint:gocyclo // Rendering is one transactional validation-and-materialization pass. -func RenderSelectedNodeLocal( - pcs *grovev1alpha1.PodCliqueSet, - workload *SelectedWorkload, +func RenderNodeLocal( + workload *Workload, plan *MaterializationPlan, input RenderInput, -) ([]client.Object, error) { +) (*WorkloadTemplates, error) { projections := workload.modelProjections - // Hybrid input owns one GPU clique before the LPU roles are appended. + // Keep the hybrid GPU clique before the workload's LPU roles. hybrid := projections[0].pipeline == PipelineLPX - var cyborg *grovev1alpha1.PodCliqueTemplateSpec + rendered := &WorkloadTemplates{} + cyborg := input.Cyborg if hybrid { - cyborg = pcs.Spec.Template.Cliques[0] + rendered.Cliques = append(rendered.Cliques, cyborg) } workloadDigest := workload.Digest().String() @@ -74,10 +81,9 @@ func RenderSelectedNodeLocal( if projections[0].pipeline == PipelineSpecDecode { allocation = strings.Join(agentTemplateNames, ":") } - namespace := pcs.Namespace // The serving component owns conductor metadata and storage independently of model order. - conductorStage := projections[len(projections)-1].stage + conductorStage := workload.ServingComponentName() conductorTemplate := input.Conductor storageTemplate := input.Stages[conductorStage] if !hybrid { @@ -90,7 +96,7 @@ func RenderSelectedNodeLocal( if err != nil { return nil, err } - configMap, err := renderRuntimeConfigMap(namespace, plan.PodCliqueSetName+"-lpu", resolvedPartitionData(projections)) + configMap, err := renderRuntimeConfigMap(plan.ResourcePrefix+"-lpu", resolvedPartitionData(projections)) if err != nil { return nil, err } @@ -133,7 +139,7 @@ func RenderSelectedNodeLocal( StartsAfter: agentTemplateNames, }, } - pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, conductor) + rendered.Cliques = append(rendered.Cliques, conductor) } // Canonical projections keep each component together; consume its last Agent instance. @@ -177,7 +183,7 @@ func RenderSelectedNodeLocal( annotations[WorkloadModeAnnotation] = string(projection.schedulerWorkloadMode()) agent := plan.Agents[index] replicas := int32(agent.Replicas) - pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, &grovev1alpha1.PodCliqueTemplateSpec{ + rendered.Cliques = append(rendered.Cliques, &grovev1alpha1.PodCliqueTemplateSpec{ Name: agent.TemplateName, Labels: maps.Clone(template.Labels), Annotations: annotations, @@ -190,16 +196,13 @@ func RenderSelectedNodeLocal( }) } - pcs.Annotations = workloadAnnotations(pcs.Annotations, workloadDigest) - // Node-local uses canonical annotation absence for backward compatibility. - delete(pcs.Annotations, ExecutionBackendAnnotation) selectedTemplateNames := agentTemplateNames if conductorTemplateName != "" { selectedTemplateNames = append([]string{conductorTemplateName}, selectedTemplateNames...) } if hybrid { - // Bound GPU hostnames using the rendered width of the last engine replica. + // Bound GPU hostnames using the rendered width of the last workload replica. if err := plan.validatePodHostname("Cyborg", plan.CyborgTemplate, int(cyborg.Spec.Replicas)-1); err != nil { return nil, err } @@ -224,32 +227,31 @@ func RenderSelectedNodeLocal( } } - // Seed the complete-engine group at its immutable minimum; live capacity is managed through /scale. + // Each workload contributes its own scaling group to the shared PCS. members := selectedTemplateNames if hybrid { members = append(members, plan.CyborgTemplate) } - pcs.Spec.Template.PodCliqueScalingGroupConfigs = []grovev1alpha1.PodCliqueScalingGroupConfig{{ - Name: lpxScalingGroupTemplateName, + rendered.ScalingGroup = grovev1alpha1.PodCliqueScalingGroupConfig{ + Name: plan.ScalingGroupTemplate, CliqueNames: members, Annotations: map[string]string{WorkloadDigestAnnotation: workloadDigest}, Replicas: ptr.To(ptr.Deref(input.MinAvailable, 1)), MinAvailable: ptr.To(ptr.Deref(input.MinAvailable, 1)), - }} + } // Keep every LPX role in one backend gang, including the KAI fallback roles. - for _, clique := range pcs.Spec.Template.Cliques { + for _, clique := range rendered.Cliques { clique.Spec.PodSpec.SchedulerName = SchedulerName } - explicit := grovev1alpha1.CliqueStartupTypeExplicit - pcs.Spec.Template.StartupType = &explicit - return extraResources, nil + rendered.Resources = extraResources + return rendered, nil } // configureLPURolePods consumes fresh, independently owned Agent and conductor // specs. Agent and workload are nonnil; nil conductor means no emitted launcher. -func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, workload *SelectedWorkload, configMapName, allocation string) error { +func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, workload *Workload, configMapName, allocation string) error { if err := withLPUConfigVolume(agentPodSpec, configMapName, workload.BuildFamily() == BuildFamilyXT); err != nil { return err } @@ -275,7 +277,6 @@ func roleAnnotations( annotations := workloadAnnotations(maps.Clone(base), workloadDigest) // Remove controller-owned role metadata before stamping canonical values. for _, key := range []string{ - ExecutionBackendAnnotation, WorkloadModeAnnotation, lpxv1alpha1.CompilerSnapshotDigestAnnotation, lpxv1alpha1.PodModelAnnotation, diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index b5b7d1ef0c22..5343001ff7bf 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -36,7 +36,7 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model if err != nil { return nil, err } - workload := &SelectedWorkload{ + workload := &Workload{ modelProjections: projections, digest: digest, scalingGroupReplicas: 1, @@ -46,14 +46,20 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model return nil, err } if workload.BuildFamily() == BuildFamilyXT && workload.Pipeline() == PipelineLPX { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(pcs.Namespace, plan) + input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(plan) if err != nil { return nil, err } } - if _, err := RenderSelectedNodeLocal(pcs, workload, plan, input); err != nil { + if workload.Pipeline() == PipelineLPX { + input.Cyborg = pcs.Spec.Template.Cliques[0] + } + templates, err := RenderNodeLocal(workload, plan, input) + if err != nil { return nil, err } + pcs.Spec.Template.Cliques = templates.Cliques + pcs.Spec.Template.PodCliqueScalingGroupConfigs = []grovev1alpha1.PodCliqueScalingGroupConfig{templates.ScalingGroup} return pcs, nil } @@ -99,12 +105,8 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { projection := projectRenderFixture(t, test.pipeline, test.snapshot) require.NotEqual(t, projection.Digest().String(), projection.CompilerSnapshotDigest()) pcs := renderTestPCS(test.pipeline == PipelineLPX) - pcs.Annotations = map[string]string{ - ExecutionBackendAnnotation: "stale", - } for _, clique := range pcs.Spec.Template.Cliques { clique.Annotations = map[string]string{ - ExecutionBackendAnnotation: "stale", lpxv1alpha1.CompilerSnapshotDigestAnnotation: "stale", } } @@ -113,7 +115,6 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { "user": "kept", lpxv1alpha1.PodModelAnnotation: "spoofed", lpxv1alpha1.CompilerSnapshotDigestAnnotation: "stale", - ExecutionBackendAnnotation: "stale", }}, Spec: renderTestPodSpec(), } @@ -147,11 +148,9 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { }) require.NoError(t, err) require.Same(t, pcs, rendered) - require.Equal(t, projection.Digest().String(), rendered.Annotations[WorkloadDigestAnnotation]) - require.NotContains(t, rendered.Annotations, ExecutionBackendAnnotation) + require.Equal(t, projection.Digest().String(), rendered.Spec.Template.PodCliqueScalingGroupConfigs[0].Annotations[WorkloadDigestAnnotation]) for _, clique := range rendered.Spec.Template.Cliques { require.Equal(t, SchedulerName, clique.Spec.PodSpec.SchedulerName) - require.NotContains(t, clique.Annotations, ExecutionBackendAnnotation) if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { require.Equal(t, projection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) } else { @@ -251,7 +250,7 @@ func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { targetSnapshot := acquireTestSnapshot(t, writeCompilerFixture(t, targetFixture)) t.Log("Keep independent draft, target and conductor templates") - draft := testLPXComponent(draftStageName, "draft-build", + draft := testLPXComponent(runtimeModelDraft, "draft-build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: &corev1.PodTemplateSpec{Spec: renderTestPodSpec()}}, ) draft.Replicas = ptr.To(int32(2)) @@ -284,20 +283,21 @@ func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { before := source.DeepCopy() t.Log("Resolve and render the authored speculative workload") - selected, err := ResolveSelectedWorkload(t.Context(), source, staticBuildSnapshotSource{ + selected, err := ResolveWorkload(t.Context(), source, singleGroupComponents(t, source), staticBuildSnapshotSource{ "draft-build": draftSnapshot, "target-build": targetSnapshot, }) require.NoError(t, err) pcs := renderTestPCS(false) plan, err := selected.PlanNodeLocalMaterialization(pcs.Name) require.NoError(t, err) - extraResources, err := RenderSelectedNodeLocal(pcs, selected, plan, RenderInput{ + templates, err := RenderNodeLocal(selected, plan, RenderInput{ Stages: stages, Conductor: conductorTemplate.DeepCopy(), }) require.NoError(t, err) t.Log("Retain each authored component's image and metadata on its Agent cliques") - for index, stage := range []string{draftStageName, draftStageName, testTargetStageName} { + pcs.Spec.Template.Cliques = templates.Cliques + for index, stage := range []string{runtimeModelDraft, runtimeModelDraft, testTargetStageName} { agent := namedClique(t, pcs, plan.Agents[index].TemplateName) require.Equal(t, stage+"-runtime", agent.Spec.PodSpec.Containers[0].Image) require.Equal(t, stage, agent.Labels["owner"]) @@ -321,7 +321,7 @@ func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { require.Equal(t, conductorBefore, &conductor.Spec.PodSpec) t.Log("Preserve runtime settings in the conductor template and share only partition data") - configMap, ok := extraResources[0].(*corev1.ConfigMap) + configMap, ok := templates.Resources[0].(*corev1.ConfigMap) require.True(t, ok) require.NotContains(t, configMap.Data, "model_config.toml") for _, env := range conductorTemplate.Spec.Containers[0].Env { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_workload.go similarity index 75% rename from deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go rename to deploy/operator/internal/dynamo/lpx/resolve_workload.go index aa70f11eeef0..08fc93d49540 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload.go @@ -20,6 +20,14 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) +const ( + // MaxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. + MaxSpecDecodeNumDrafts = 8 + + runtimeModelDraft = "draft" + runtimeModelTarget = "target" +) + // ErrUnsupportedRuntime identifies input that the supported LPX runtimes cannot materialize. var ErrUnsupportedRuntime = errors.New("unsupported LPX runtime") @@ -33,23 +41,40 @@ type BuildSnapshotSource interface { AcquireBuildSnapshot(context.Context, string) (*BuildSnapshot, error) } -// ResolveSelectedWorkload projects admitted LPX intent against immutable builds. -// dgd and source must be non-nil; dgd must select LPX and have passed webhook validation. -// The function reads but does not mutate dgd. -func ResolveSelectedWorkload( +// ResolveWorkload projects one admitted component group against immutable builds. +// dgd and source must be non-nil. componentNames must contain exactly the members +// of one ComponentGroups entry from the admitted dgd, in any order. +// Inputs are read without mutation; the full dgd supplies validation field paths. +func ResolveWorkload( ctx context.Context, dgd *dynamov1beta1.DynamoGraphDeployment, + componentNames []string, source BuildSnapshotSource, -) (*SelectedWorkload, error) { - components := Components(dgd) - projections := make([]*ModelProjection, 0, len(components)) - var snapshot NormalizedBuildSnapshot - var pipeline Pipeline - for index, stage := range components { +) (*Workload, error) { + // Resolve only this group's native components from the unchanged graph. + components := make([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, 0, len(componentNames)) + for _, name := range componentNames { + components = append(components, dgd.GetComponentByName(name)) + } + + // Preserve draft-before-target runtime identities independently of authored order. + if len(components) == 2 && components[0].ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { + components[0], components[1] = components[1], components[0] + } + + // Acquire each build and expand its models in canonical runtime order. + var ( + snapshot NormalizedBuildSnapshot + pipeline Pipeline + + projections = make([]*ModelProjection, 0, len(components)) + ) + + for _, stage := range components { model := stage.LPX configuredModel := stage.ComponentName - // A validated selection has at most two models, so only the preceding projection can share its build. + // An admitted group has at most two components, so only its preceding component can share a build. if len(projections) == 0 || projections[len(projections)-1].runtimeBuildRef != model.BuildID { rawSnapshot, acquireErr := source.AcquireBuildSnapshot(ctx, model.BuildID) if acquireErr != nil { @@ -83,12 +108,13 @@ func ResolveSelectedWorkload( } } - if err := validateSelectedConductor(dgd, stage, pipeline, snapshot.build.CompilationMode); err != nil { + if err := validateWorkloadConductor(dgd, stage, pipeline, snapshot.build.CompilationMode); err != nil { return nil, err } // Project the component once into the resolver-owned aggregate destination. - modelNames := expandedSelectedModelNames(index, len(components), int(ptr.Deref(stage.Replicas, 1))) + hasConductor := stage.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil + modelNames := expandedModelNames(len(components), hasConductor, int(ptr.Deref(stage.Replicas, 1))) intent := ModelProjectionInput{ Pipeline: pipeline, Models: modelNames, @@ -122,15 +148,15 @@ func ResolveSelectedWorkload( if err != nil { return nil, err } - return &SelectedWorkload{ + return &Workload{ modelProjections: projections, digest: digest, scalingGroupReplicas: scalingGroupReplicas, }, nil } -// validateSelectedConductor checks role requirements that depend on the immutable build. -func validateSelectedConductor( +// validateWorkloadConductor checks role requirements that depend on the immutable build. +func validateWorkloadConductor( dgd *dynamov1beta1.DynamoGraphDeployment, component *dynamov1beta1.DynamoComponentDeploymentSharedSpec, pipeline Pipeline, @@ -164,7 +190,7 @@ func validateSelectedConductor( } // Only the LPU-only serving template materializes a renamed conductor container. - if component != ServingComponent(dgd) { + if conductor == nil { return nil } @@ -182,7 +208,7 @@ func validateSelectedConductor( return err } - // An LPU-only conductor is a singleton even when the engine replica count is larger. + // An LPU-only conductor is a singleton even when the workload replica count is larger. if ptr.Deref(conductor.Replicas, 1) != 1 { return fmt.Errorf("%w: component %q conductor replicas must be one for LPU-only execution", ErrUnsupportedRuntime, component.ComponentName) } @@ -213,3 +239,18 @@ func validateRolePodSpecContainerNames(spec *corev1.PodSpec, fldPath *field.Path } return allErrs } + +func expandedModelNames(componentCount int, hasConductor bool, draftCount int) []string { + // Component membership retains Nova's existing default/draft/target identities. + if componentCount == 1 { + return []string{"default"} + } + if hasConductor { + return []string{runtimeModelTarget} + } + names := make([]string, draftCount) + for index := range names { + names[index] = fmt.Sprintf("draft%d", index) + } + return names +} diff --git a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go similarity index 78% rename from deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go rename to deploy/operator/internal/dynamo/lpx/resolve_workload_test.go index 33075413e469..845954dcf800 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_selected_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go @@ -20,7 +20,7 @@ import ( "k8s.io/utils/ptr" ) -func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { +func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { t.Log("Create one LPX component beside an unrelated conventional decode") dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor-runtime")})) dgd.Annotations = nil @@ -30,14 +30,14 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "ordinary"}}}}, }) - t.Log("Resolve an LPU-only engine without selecting the conventional decode") + t.Log("Resolve an LPU-only workload without selecting the conventional decode") hxSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) - hx, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": hxSnapshot}) + hx, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), staticBuildSnapshotSource{"build": hxSnapshot}) require.NoError(t, err) require.Equal(t, PipelineSingle, hx.Pipeline()) require.Equal(t, BuildFamilyHX, hx.BuildFamily()) require.Equal(t, lpxv1alpha1.WorkloadModeV3HxLPUOnly, hx.modelProjections[0].RequestSpec(&MaterializationPlan{}, "agents").WorkloadMode) - require.Equal(t, "LPX", hx.LPXComponentName()) + require.Equal(t, "LPX", hx.ServingComponentName()) plan, err := hx.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) require.Equal(t, "test-pcs-0-lpx", plan.LPXScalingGroup) @@ -45,10 +45,10 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin require.Empty(t, plan.CyborgTemplate) require.NotEmpty(t, plan.ConductorClique) - t.Log("Scale Nova engines without changing their model or workload digest") + t.Log("Scale Nova workloads without changing their model or workload digest") for _, replicas := range []int32{2, 10, 12, 123} { dgd.Spec.Components[0].Replicas = ptr.To(replicas) - scaled, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": hxSnapshot}) + scaled, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), staticBuildSnapshotSource{"build": hxSnapshot}) require.NoError(t, err) require.Equal(t, hx.Digest(), scaled.Digest()) require.Equal(t, replicas, scaled.scalingGroupReplicas) @@ -62,11 +62,11 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin fixture.partitions = append(fixture.partitions, testV3CapnpPartition{id: 11, deviceType: manifestcapnp.DeviceType_cuda}) snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) source := staticBuildSnapshotSource{"build": snapshot} - _, err = ResolveSelectedWorkload(t.Context(), dgd, source) + _, err = ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.ErrorContains(t, err, "requires a declared resourceClaim or a positive nvidia.com/gpu request") conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) conductor.PodTemplate = testLPXPodTemplate("cyborg-runtime") - _, err = ResolveSelectedWorkload(t.Context(), dgd, source) + _, err = ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.ErrorContains(t, err, "requires a declared resourceClaim or a positive nvidia.com/gpu request") t.Log("Validate claim consumption by the Cyborg main container") @@ -106,7 +106,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin before := candidate.DeepCopy() t.Log("Require scalar GPUs or a claim consumed by main without rewriting the template") - _, err := ResolveSelectedWorkload(t.Context(), candidate, source) + _, err := ResolveWorkload(t.Context(), candidate, singleGroupComponents(t, candidate), source) if test.wantErr { require.ErrorContains(t, err, "conductor main container requires a declared resourceClaim") } else { @@ -122,7 +122,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin dgd.Spec.Components[0].Replicas = ptr.To(int32(2)) t.Log("Project two complete hybrid replicas") - xt, err := ResolveSelectedWorkload(t.Context(), dgd, source) + xt, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) require.Equal(t, PipelineLPX, xt.Pipeline()) require.Equal(t, BuildFamilyXT, xt.BuildFamily()) @@ -138,7 +138,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin t.Log("A scheduling deadline and annotation do not change hybrid launch") dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} dgd.Annotations = map[string]string{commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend} - scheduled, err := ResolveSelectedWorkload(t.Context(), dgd, source) + scheduled, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) require.Equal(t, xt, scheduled) scheduledPlan, err := scheduled.PlanNodeLocalMaterialization("test-pcs") @@ -148,7 +148,7 @@ func TestResolveSelectedWorkloadDerivesRuntimeShapeFromCompilationMode(t *testin } -func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { +func TestResolveWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Define revision-specific SpecDecode compiler snapshots") tests := []struct { name string @@ -195,7 +195,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Project the selected SpecDecode workload") before := dgd.DeepCopy() - selected, err := ResolveSelectedWorkload(t.Context(), dgd, source) + selected, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) t.Log("Project the selected family, workload mode, and pipeline") require.NoError(t, err) @@ -203,7 +203,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { require.Equal(t, test.family, selected.BuildFamily()) require.Equal(t, test.wantMode, selected.modelProjections[0].RequestSpec(&MaterializationPlan{}, "agents").WorkloadMode) require.Equal(t, PipelineSpecDecode, selected.Pipeline()) - require.Equal(t, "lpx", selected.LPXComponentName()) + require.Equal(t, "lpx", selected.ServingComponentName()) t.Log("Expand draft fanout while preserving model and template identity order") projections := selected.ModelProjections() @@ -232,17 +232,20 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { t.Log("Reordering authored components does not change build identities or resources") reordered := dgd.DeepCopy() slices.Reverse(reordered.Spec.Components) - reselected, err := ResolveSelectedWorkload(t.Context(), reordered, source) + members := singleGroupComponents(t, reordered) + slices.Reverse(members) + reselected, err := ResolveWorkload(t.Context(), reordered, members, source) require.NoError(t, err) require.Equal(t, selected.Digest(), reselected.Digest()) reorderedPlan, err := reselected.PlanNodeLocalMaterialization("test-pcs") require.NoError(t, err) require.Equal(t, plan, reorderedPlan) + require.Equal(t, []string{"small", "lpx"}, reselected.ComponentNames()) t.Log("Agent replica assertions count one compiled model instance, not draft fanout") invalidCount := dgd.DeepCopy() invalidCount.Spec.Components[1].ComponentRole(v1beta1.ComponentRoleLPXAgent).Replicas = ptr.To(compiledAgentCount * 2) - _, err = ResolveSelectedWorkload(t.Context(), invalidCount, source) + _, err = ResolveWorkload(t.Context(), invalidCount, singleGroupComponents(t, invalidCount), source) require.ErrorContains(t, err, "must match the compiled count") t.Log("Derive an aggregate digest and reject mixed-family aggregation") @@ -277,7 +280,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { if expansion.count > 1 { draft.Replicas = ptr.To(expansion.count) } - expanded, err := ResolveSelectedWorkload(t.Context(), dgd, source) + expanded, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) models := make([]string, 0, len(expansion.models)) for _, projection := range expanded.ModelProjections() { @@ -294,7 +297,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { draft := &dgd.Spec.Components[1] draft.Replicas = nil draft.LPX.BuildID = "target-build" - shared, err := ResolveSelectedWorkload(t.Context(), dgd, source) + shared, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) projections := shared.ModelProjections() require.Len(t, projections, 2) @@ -307,7 +310,7 @@ func TestResolveSelectedWorkloadSpecDecodeV2AndV3(t *testing.T) { } } -func TestResolveSelectedWorkloadPreservesAuthoredLaunch(t *testing.T) { +func TestResolveWorkloadPreservesAuthoredLaunch(t *testing.T) { t.Log("Author independent wrappers without exposing launch syntax to the operator") conductor := testLPXPodTemplate("conductor-runtime") conductor.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} @@ -323,12 +326,12 @@ func TestResolveSelectedWorkloadPreservesAuthoredLaunch(t *testing.T) { before := dgd.DeepCopy() t.Log("Validate placement and role ownership without parsing either command line") - _, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": snapshot}) + _, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), staticBuildSnapshotSource{"build": snapshot}) require.NoError(t, err) require.Equal(t, before, dgd) } -func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t *testing.T) { +func TestResolveWorkloadChecksConductorContainerNamesForSelectedBuild(t *testing.T) { t.Log("Acquire LPU-only and hybrid builds that select different runtime container identities") lpuSnapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) hybridFixture := newV2CompilerFixture() @@ -428,11 +431,11 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t before := dgd.DeepCopy() t.Log("Defer conductor-name checks until the immutable build has been acquired") - _, err := ResolveSelectedWorkload(t.Context(), dgd, unreachableBuildSnapshotSource{}) + _, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), unreachableBuildSnapshotSource{}) require.ErrorIs(t, err, ErrBuildSnapshotAcquisition) t.Log("Reject only actual conductor collisions for the selected runtime") - selected, err := ResolveSelectedWorkload(t.Context(), dgd, staticBuildSnapshotSource{"build": snapshot}) + selected, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), staticBuildSnapshotSource{"build": snapshot}) if test.wantForbidden { namePath := field.NewPath("spec", "components").Index(componentIndex).Child("roles").Index(roleIndex).Child("podTemplate", "spec", containerList).Index(containerIndex).Child("name") want := field.Forbidden(namePath, `LPX reserves "conductor" for the materialized role container`) @@ -449,3 +452,52 @@ func TestResolveSelectedWorkloadChecksConductorContainerNamesForSelectedBuild(t } } } + +func TestResolveWorkloadIsolatesComponentGroups(t *testing.T) { + t.Log("Author two LPU-only workloads with different builds and replica counts") + dgd := newSelectedTestDGD(t, "graph", v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, + }) + for index, name := range []string{"first", "second"} { + component := testLPXComponent(name, name+"-build", + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("agent")}, + v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor")}, + ) + component.Replicas = ptr.To(int32(index + 2)) + dgd.Spec.Components = append(dgd.Spec.Components, component) + } + snapshot := acquireTestSnapshot(t, writeV3CompilerFixture(t)) + before := dgd.DeepCopy() + + t.Log("Resolve each group using only its own build, replicas and component identity") + groups := ComponentGroups(dgd) + for index, name := range []string{"first", "second"} { + workload, err := ResolveWorkload(t.Context(), dgd, groups[name], staticBuildSnapshotSource{name + "-build": snapshot}) + require.NoError(t, err) + require.Equal(t, PipelineSingle, workload.Pipeline()) + require.Equal(t, name, workload.ServingComponentName()) + require.Equal(t, []string{name}, workload.ComponentNames()) + plan, err := workload.PlanNodeLocalMaterialization("pcs") + require.NoError(t, err) + require.EqualValues(t, index+2, plan.Replicas) + } + require.Equal(t, before, dgd) + + t.Log("Validate the second conductor and report its index in the complete graph") + conductor := dgd.GetComponentByName("second").ComponentRole(v1beta1.ComponentRoleLPXConductor) + conductor.PodTemplate.Spec.InitContainers = []corev1.Container{{Name: "conductor", Image: "sidecar"}} + _, err := ResolveWorkload(t.Context(), dgd, groups["second"], staticBuildSnapshotSource{"second-build": snapshot}) + require.ErrorContains(t, err, "spec.components[2].roles[1].podTemplate.spec.initContainers[0].name") + conductor.PodTemplate.Spec.InitContainers = nil + conductor.Replicas = ptr.To(int32(2)) + _, err = ResolveWorkload(t.Context(), dgd, groups["second"], staticBuildSnapshotSource{"second-build": snapshot}) + require.ErrorContains(t, err, `component "second" conductor replicas must be one`) +} + +func TestExpandedModelNames(t *testing.T) { + t.Log("Expand admitted component replicas into runtime model names") + require.Equal(t, []string{"draft0", "draft1", "draft2"}, expandedModelNames(2, false, 3)) + require.Equal(t, []string{"draft0"}, expandedModelNames(2, false, 1)) + require.Equal(t, []string{"target"}, expandedModelNames(2, true, 1)) + require.Equal(t, []string{"default"}, expandedModelNames(1, true, 2)) +} diff --git a/deploy/operator/internal/dynamo/lpx/selected_workload.go b/deploy/operator/internal/dynamo/lpx/selected_workload.go deleted file mode 100644 index 041e40d0b9d6..000000000000 --- a/deploy/operator/internal/dynamo/lpx/selected_workload.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -// SelectedWorkload is the immutable result of resolving exactly one DGD for -// LPX scheduling and runtime materialization. -// Its methods require a successfully resolved, non-nil workload with projections. -type SelectedWorkload struct { - modelProjections []*ModelProjection - digest WorkloadDigest - scalingGroupReplicas int32 -} - -// ModelProjections returns a read-only view of the selected projection list. -// Callers must not modify the slice or its projections. -func (w *SelectedWorkload) ModelProjections() []*ModelProjection { - return w.modelProjections -} - -// Digest returns the selected workload digest. -func (w *SelectedWorkload) Digest() WorkloadDigest { - return w.digest -} - -// BuildFamily returns the selected build family. -func (w *SelectedWorkload) BuildFamily() BuildFamily { - return w.modelProjections[0].configuredBuild.Family -} - -// Pipeline returns the selected pipeline. -func (w *SelectedWorkload) Pipeline() Pipeline { - return w.modelProjections[0].pipeline -} - -// LPXComponentName returns the serving component name from the final projection. -func (w *SelectedWorkload) LPXComponentName() string { - return w.modelProjections[len(w.modelProjections)-1].stage -} diff --git a/deploy/operator/internal/dynamo/lpx/selection.go b/deploy/operator/internal/dynamo/lpx/selection.go deleted file mode 100644 index dbcbcefef525..000000000000 --- a/deploy/operator/internal/dynamo/lpx/selection.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - - dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" -) - -const ( - // MaxSpecDecodeNumDrafts bounds the supported speculative-decoding draft fanout. - MaxSpecDecodeNumDrafts = 8 - - draftStageName = "draft" - runtimeModelDraft = "draft" - runtimeModelTarget = "target" -) - -// Components returns LPX components. In a supported pair, the worker-only draft -// precedes the serving component. dgd is non-nil and is not mutated. -// Returned component pointers remain read-only. -func Components(dgd *dynamov1beta1.DynamoGraphDeployment) []*dynamov1beta1.DynamoComponentDeploymentSharedSpec { - components := make([]*dynamov1beta1.DynamoComponentDeploymentSharedSpec, 0, 2) - for index := range dgd.Spec.Components { - component := &dgd.Spec.Components[index] - if component.IsLPX() { - components = append(components, component) - } - } - if len(components) == 2 && components[0].ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { - components[0], components[1] = components[1], components[0] - } - return components -} - -// ServingComponent returns the sole LPX component or the component declaring -// the shared conductor role. It returns nil when a pair has no conductor owner. -// dgd is non-nil and the returned component is read-only. -func ServingComponent(dgd *dynamov1beta1.DynamoGraphDeployment) *dynamov1beta1.DynamoComponentDeploymentSharedSpec { - var soleComponent *dynamov1beta1.DynamoComponentDeploymentSharedSpec - componentCount := 0 - for index := range dgd.Spec.Components { - component := &dgd.Spec.Components[index] - if !component.IsLPX() { - continue - } - if component.ComponentRole(dynamov1beta1.ComponentRoleLPXConductor) != nil { - return component - } - soleComponent = component - componentCount++ - } - if componentCount == 1 { - return soleComponent - } - return nil -} - -func expandedSelectedModelNames(index, componentCount, draftCount int) []string { - // Component membership retains Nova's existing default/draft/target identities. - if componentCount == 1 { - return []string{"default"} - } - if index == 1 { - return []string{runtimeModelTarget} - } - names := make([]string, draftCount) - for index := range names { - names[index] = fmt.Sprintf("draft%d", index) - } - return names -} diff --git a/deploy/operator/internal/dynamo/lpx/selection_test.go b/deploy/operator/internal/dynamo/lpx/selection_test.go deleted file mode 100644 index 7f020fe923d3..000000000000 --- a/deploy/operator/internal/dynamo/lpx/selection_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "testing" - - dynamov1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { - t.Log("Select components by their declared type, independently of build or templates") - canonicalLPX := []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentType: dynamov1beta1.ComponentTypeLPX}} - - t.Log("Define default-selection scenarios") - tests := []struct { - name string - annotations map[string]string - components []dynamov1beta1.DynamoComponentDeploymentSharedSpec - want bool - }{ - {name: "canonical LPX", components: canonicalLPX, want: true}, - { - name: "component provider", - annotations: map[string]string{ - commonconsts.KubeAnnotationWorkloadProvider: commonconsts.WorkloadProviderComponent, - }, - components: canonicalLPX, - want: true, - }, - { - name: "Grove opt-out", - annotations: map[string]string{ - commonconsts.KubeAnnotationEnableGrove: "FALSE", - }, - components: canonicalLPX, - want: true, - }, - {name: "non-LPX", components: []dynamov1beta1.DynamoComponentDeploymentSharedSpec{{ComponentName: dynamov1beta1.ComponentRoleLPXAgent}}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Construct the DGD selection input") - dgd := &dynamov1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Annotations: test.annotations}, - Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ - Components: test.components, - Scheduling: &dynamov1beta1.SchedulingSpec{}, - }, - } - - t.Log("Resolve and verify scheduler selection") - selected := dgd.HasLPXComponent() - require.Equal(t, test.want, selected) - }) - } -} - -func TestExpandedSelectedModelNames(t *testing.T) { - t.Log("Expand admitted component replicas into runtime model names") - require.Equal(t, []string{"draft0", "draft1", "draft2"}, expandedSelectedModelNames(0, 2, 3)) - require.Equal(t, []string{"draft0"}, expandedSelectedModelNames(0, 2, 1)) - require.Equal(t, []string{"target"}, expandedSelectedModelNames(1, 2, 1)) - require.Equal(t, []string{"default"}, expandedSelectedModelNames(0, 1, 2)) -} diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml new file mode 100644 index 000000000000..8ed2e304d688 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: lpu-cyborg-specdec + namespace: test + uid: 22222222-2222-4222-8222-222222222222 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: draft + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected + roles: + - name: conductor + podTemplate: + spec: + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/lpu-conductor:test + command: [/opt/dynamo-lpu] + args: [serve] + - name: agent + podTemplate: + spec: + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/lpu-agent:test + command: [/opt/dynamo-lpu] + args: [serve] + - name: target + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected-lpx + roles: + - name: conductor + replicas: 1 + podTemplate: + spec: + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/cyborg-specdecode:test + command: [python, -m, dynamo.cyborg] + args: [--draft-endpoint, test-lpu-cyborg-specdec/backend/generate] + resources: + claims: + - name: gpu + - name: agent + podTemplate: + spec: + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/lpu-agent:test + command: [/opt/dynamo-lpu] + args: [serve] diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml new file mode 100644 index 000000000000..5d81ddbdfb1e --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml @@ -0,0 +1,482 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +data: + nodes_per_partition: |- + 2 + 2 + partition_ids: |- + 0 + 1 + partition_node_offsets: |- + 0 + 2 + partition_paths: |- + part-0 + part-1 + topologies: |- + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + name: lpu-cyborg-specdec-5f16-draft-lpu-6bcf1f4cf22aaf55 + namespace: test +--- +data: + lpu_servers: target-${GROVE_PCSG_INDEX}-target-agt-0 +immutable: true +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + name: lpu-cyborg-specdec-5f16-target-decode-32805b2198371ed2 + namespace: test +--- +data: + nodes_per_partition: "4" + partition_ids: "0" + partition_indices: "0" + partition_models: default + partition_node_offsets: "0" + partition_paths: part-0 + topologies: URSA_V2__Q8__32C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + name: lpu-cyborg-specdec-5f16-target-lpu-355e8ac6768c54e3 + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:31a0ee5e56b73e39b7c7b2c93678df6bf858dedaa77350422c3f557fbdb7d5ad + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec + name: lpu-cyborg-specdec-5f16 + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec + nvidia.com/dynamo-namespace: test-lpu-cyborg-specdec + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-cyborg-specdec-draft + name: draft-agt + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-agent:test + name: agent + resources: + limits: + lpu.nvidia.com/lpu: "8" + requests: + lpu.nvidia.com/lpu: "8" + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-cyborg-specdec-5f16-draft-lpu-6bcf1f4cf22aaf55 + name: config + replicas: 4 + roleName: draft-agt + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec + nvidia.com/dynamo-namespace: test-lpu-cyborg-specdec + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-cyborg-specdec-draft + name: draft-cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPX_ALLOCATION + value: draft-agt + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-conductor:test + name: conductor + resources: {} + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-cyborg-specdec-5f16-draft-lpu-6bcf1f4cf22aaf55 + name: config + replicas: 1 + roleName: draft-cond + startsAfter: + - draft-agt + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid + labels: + lpx.nvidia.com/stage: target + nvidia.com/dynamo-component: target + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec + nvidia.com/dynamo-namespace: test-lpu-cyborg-specdec + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-cyborg-specdec-target + name: target-agt + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected-lpx + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-agent:test + name: agent + resources: + limits: + lpu.nvidia.com/lpu: "8" + requests: + lpu.nvidia.com/lpu: "8" + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-cyborg-specdec-5f16-target-lpu-355e8ac6768c54e3 + name: config + replicas: 4 + roleName: target-agt + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 32805b2198371ed2ff837e4776e3e59448937c6274efe6602681e42f78f14d41 + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/role: cyborg-worker + labels: + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: target + nvidia.com/dynamo-component: target + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec + nvidia.com/dynamo-namespace: test-lpu-cyborg-specdec + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-cyborg-specdec-target + name: target-cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - --draft-endpoint + - test-lpu-cyborg-specdec/backend/generate + command: + - python + - -m + - dynamo.cyborg + env: + - name: DYN_COMPONENT + value: decode + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-cyborg-specdec + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: GBUILD_MANIFEST_PATH + value: /models/node-local-v2-connected-lpx/manifest.v2.capnp.bin + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/cyborg-specdecode:test + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + resources: + claims: + - name: gpu + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + resourceClaims: + - name: gpu + resourceClaimTemplateName: decode-gpu + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-cyborg-specdec-5f16-target-decode-32805b2198371ed2 + name: config + replicas: 1 + roleName: target-cond + startsAfter: + - target-agt + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + cliqueNames: + - draft-cond + - draft-agt + minAvailable: 1 + name: draft + replicas: 1 + - annotations: + lpx.nvidia.com/deployment-name: lpu-cyborg-specdec + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + cliqueNames: + - target-agt + - target-cond + minAvailable: 1 + name: target + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml new file mode 100644 index 000000000000..d5d75f59ff13 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: lpu-gpu-specdecode + namespace: test + uid: 22222222-2222-4222-8222-222222222222 + generation: 1 + annotations: + scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler +spec: + components: + - name: draft + type: lpx + replicas: 1 + lpx: + buildId: node-local-v2-connected + roles: + - name: conductor + podTemplate: + spec: + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/lpu-conductor:test + command: [/opt/dynamo-lpu] + args: [serve] + - name: agent + podTemplate: + spec: + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + containers: + - name: main + volumeMounts: + - name: model-storage + mountPath: /models + image: example.invalid/lpu-agent:test + command: [/opt/dynamo-lpu] + args: [serve] + - name: target + type: worker + replicas: 1 + podTemplate: + spec: + containers: + - name: main + image: example.invalid/vllm-target:test + command: [python, -m, dynamo.vllm] + args: [--experimental-external-draft-endpoint, test-lpu-gpu-specdecode/backend/generate] + resources: + limits: + nvidia.com/gpu: '1' diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml new file mode 100644 index 000000000000..23d9266863b3 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +data: + nodes_per_partition: |- + 2 + 2 + partition_ids: |- + 0 + 1 + partition_node_offsets: |- + 0 + 2 + partition_paths: |- + part-0 + part-1 + topologies: |- + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA + URSA_V2__Q8__16C__G_96_25__KP_FEC__GHZ_1_0__NO_FPGA +immutable: true +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-gpu-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + name: lpu-gpu-specdecode-bc86-lpu-6bcf1f4cf22aaf55 + namespace: test +--- +metadata: + annotations: + lpx.nvidia.com/deployment-name: lpu-gpu-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:26e70c69aa2a43744d6a89c9fffb8c9bbd2711cd6203561d0a97af75e4080798 + labels: + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode + name: lpu-gpu-specdecode-bc86 + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - annotations: + lpx.nvidia.com/deployment-name: lpu-gpu-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/model: default + scheduling.lpu.nvidia.com/role: agent + scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly + labels: + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode + nvidia.com/dynamo-namespace: test-lpu-gpu-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-gpu-specdecode-draft + name: agt + spec: + minAvailable: 4 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-agent:test + name: agent + resources: + limits: + lpu.nvidia.com/lpu: "8" + requests: + lpu.nvidia.com/lpu: "8" + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-gpu-specdecode-bc86-lpu-6bcf1f4cf22aaf55 + name: config + replicas: 4 + roleName: agt + - annotations: + lpx.nvidia.com/deployment-name: lpu-gpu-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/role: conductor + labels: + lpx.nvidia.com/serving: "true" + lpx.nvidia.com/stage: draft + nvidia.com/dynamo-component: draft + nvidia.com/dynamo-component-type: lpx + nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode + nvidia.com/dynamo-namespace: test-lpu-gpu-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-gpu-specdecode-draft + name: cond + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - serve + command: + - /opt/dynamo-lpu + env: + - name: DYN_COMPONENT + value: lpx + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_NAMESPACE + value: test-lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: ETCD_ENDPOINTS + value: etcd-address + - name: LPX_ALLOCATION + value: agt + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected + - name: NATS_SERVER + value: nats-address + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/lpu-conductor:test + name: conductor + resources: {} + volumeMounts: + - mountPath: /models + name: model-storage + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + schedulerName: lpx-scheduler + terminationGracePeriodSeconds: 60 + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: model-storage + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + - configMap: + name: lpu-gpu-specdecode-bc86-lpu-6bcf1f4cf22aaf55 + name: config + replicas: 1 + roleName: cond + startsAfter: + - agt + headlessServiceConfig: + publishNotReadyAddresses: true + podCliqueScalingGroups: + - annotations: + lpx.nvidia.com/deployment-name: lpu-gpu-specdecode + lpx.nvidia.com/deployment-uid: lpx-render-uid + scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + cliqueNames: + - cond + - agt + minAvailable: 1 + name: lpx + replicas: 1 + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 +--- +metadata: + labels: + nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode + name: lpu-gpu-specdecode + namespace: test +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeAnyOrder + cliques: + - labels: + nvidia.com/dynamo-component: target + nvidia.com/dynamo-component-type: worker + nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode + nvidia.com/dynamo-namespace: test-lpu-gpu-specdecode + nvidia.com/metrics-enabled: "true" + nvidia.com/selector: lpu-gpu-specdecode-target + name: target + spec: + minAvailable: 1 + podSpec: + containers: + - args: + - --experimental-external-draft-endpoint + - test-lpu-gpu-specdecode/backend/generate + command: + - python + - -m + - dynamo.vllm + env: + - name: DYN_COMPONENT + value: worker + - name: DYN_DISCOVERY_BACKEND + value: kubernetes + - name: DYN_FORWARDPASS_METRIC_PORT + value: "20380" + - name: DYN_HEALTH_CHECK_ENABLED + value: "false" + - name: DYN_NAMESPACE + value: test-lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAME + value: lpu-gpu-specdecode + - name: DYN_PARENT_DGD_K8S_NAMESPACE + value: test + - name: DYN_SYSTEM_ENABLED + value: "true" + - name: DYN_SYSTEM_PORT + value: "9090" + - name: DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS + value: '["generate"]' + - name: ETCD_ENDPOINTS + value: etcd-address + - name: NATS_SERVER + value: nats-address + - name: NIXL_TELEMETRY_ENABLE + value: "n" + - name: NIXL_TELEMETRY_EXPORTER + value: prometheus + - name: NIXL_TELEMETRY_PROMETHEUS_PORT + value: "19090" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: example.invalid/vllm-target:test + livenessProbe: + failureThreshold: 1 + httpGet: + path: /live + port: system + periodSeconds: 5 + timeoutSeconds: 4 + name: main + ports: + - containerPort: 9090 + name: system + protocol: TCP + - containerPort: 19090 + name: nixl + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: system + periodSeconds: 10 + timeoutSeconds: 4 + resources: + limits: + nvidia.com/gpu: "1" + startupProbe: + failureThreshold: 720 + httpGet: + path: /live + port: system + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + restartPolicy: Always + securityContext: + fsGroup: 1000 + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: shared-memory + replicas: 1 + roleName: target + headlessServiceConfig: + publishNotReadyAddresses: true + terminationDelay: 15m0s +status: + availableReplicas: 0 + updatedReplicas: 0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 42146c7fa849..ea328c95c71c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -28,7 +28,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-decode- + name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 namespace: inference-engine --- data: @@ -166,7 +166,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-lpx-v2-vllm-94cd-lpu- + name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 namespace: inference-engine --- metadata: @@ -174,7 +174,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:2b28cb9f0acada1e36c2e2f52a29876f852308f21d2d753da6095dc4261589d6 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm @@ -188,7 +188,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: e586b76cc8ccd9a1ed2e1b0297032eb1da420881dcfdcb632a4f00f4971308ff scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 @@ -392,14 +392,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-lpu- + name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -571,7 +571,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-lpx-v2-vllm-94cd-decode- + name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 88da944b83a4..3b9e0c0a2352 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -11,7 +11,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-decode- + name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f namespace: test --- data: @@ -30,7 +30,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-hybrid-9d06-lpu- + name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 namespace: test --- metadata: @@ -38,7 +38,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:db59c694410ca3a5c6502ab3b9d4de7ec12d4a5c0488fba792685fd35f08fd67 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid @@ -52,7 +52,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 @@ -207,14 +207,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-lpu- + name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/role: cyborg-worker @@ -360,7 +360,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-hybrid-9d06-decode- + name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 0efcf8a5352c..7646f7b8f1a2 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -25,7 +25,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-lpu-only-33b1-lpu- + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 namespace: test --- metadata: @@ -33,7 +33,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:561ba3854afee2dd805b99728289b13411f9ebfe379fa60b1262d373c1565f96 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only @@ -47,7 +47,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc @@ -176,14 +176,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-lpu-only-33b1-lpu- + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 name: config replicas: 4 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor @@ -348,7 +348,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-lpu-only-33b1-lpu- + name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index f90a5b94f840..9b34d61480cb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -59,7 +59,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v2-specdecode-63df-lpu- + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 namespace: test --- metadata: @@ -67,7 +67,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:eeb1e1a79f8739fa99975c047dbffde2c51cf1d1d5cfedae40ee7a3857a55ab2 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -81,7 +81,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b0943511c591eb7201003374eacf778c4545a959d7be251547399296e35ee1ae @@ -214,14 +214,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu- + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d19c12c440463a3ebbc8c86f3fe882a33e7d5f48de493146aaf6d0acfa4fdba7 @@ -354,14 +354,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu- + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b06e7ae90a96bd569bf228eb8b1d4b1192c4b0df3af7e33202d1fc1965a5247e @@ -494,14 +494,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v2-specdecode-63df-lpu- + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 4 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 scheduling.lpu.nvidia.com/role: conductor @@ -673,7 +673,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v2-specdecode-63df-lpu- + name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index e3fe7b5fc710..2f06a4ad0f77 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -17,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-hybrid-be18-lpu- + name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd namespace: test --- metadata: @@ -25,7 +25,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f7d69156466e0ef0f00c871c90da8cd98e6956cb5da9745ec86b462b19dd5d26 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid @@ -39,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 @@ -194,7 +194,7 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-hybrid-be18-lpu- + name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 57a214ecbe6e..ac76985f7be3 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -17,7 +17,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-lpu-only-d43c-lpu- + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd namespace: test --- metadata: @@ -25,7 +25,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:4de325afa19b861c0592d5037388b4e878bdc0f2991f5d75ef485a933ae7e868 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only @@ -39,7 +39,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 @@ -182,14 +182,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu- + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/role: conductor @@ -373,7 +373,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-lpu-only-d43c-lpu- + name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 0a5e46e8c048..4e2c60b65926 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -38,7 +38,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: node-local-v3-hx-specdecode-1f7a-lpu- + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 namespace: test --- metadata: @@ -46,7 +46,7 @@ metadata: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:34698082a40a6249bd1416b3f21bf429f2aac81064563dd6ebf40bcbd61bbd6d labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -60,7 +60,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6b42344f7916f4778ffdef73249a7395bbc7a082466cc457953af5de5544c80f @@ -207,14 +207,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu- + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b94a35c4a019e3213f4497e105522208d2a2707e8469ec13c22d46b99d53c227 @@ -361,14 +361,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu- + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8440780b5cdc30ae4bbcf80fda3cce06027325884a8b69751577112153ab2cf5 @@ -515,14 +515,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu- + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 scheduling.lpu.nvidia.com/role: conductor @@ -714,7 +714,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: node-local-v3-hx-specdecode-1f7a-lpu- + name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 0642a6b31ddd..318a50f2021a 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -100,7 +100,7 @@ metadata: scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid - name: dgd-single-v2-3a98-lpu- + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 namespace: inference-engine --- metadata: @@ -108,7 +108,7 @@ metadata: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a917077bc269f6b8d4770d022b50dc6208c4b2cc64800c813422b3768321613f labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 @@ -122,7 +122,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 @@ -255,14 +255,14 @@ spec: sizeLimit: 8Gi name: shared-memory - configMap: - name: dgd-single-v2-3a98-lpu- + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 name: config replicas: 17 roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 lpx.nvidia.com/deployment-uid: lpx-render-uid - nvidia.com/extra-resources-hash: + nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/role: conductor @@ -446,7 +446,7 @@ spec: defaultMode: 384 secretName: ssh-secret - configMap: - name: dgd-single-v2-3a98-lpu- + name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 name: config replicas: 1 roleName: cond diff --git a/deploy/operator/internal/dynamo/lpx/workload.go b/deploy/operator/internal/dynamo/lpx/workload.go new file mode 100644 index 000000000000..a7dfee8029b1 --- /dev/null +++ b/deploy/operator/internal/dynamo/lpx/workload.go @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package lpx + +// Workload is the immutable result of resolving one LPX component group +// for scheduling and runtime materialization. +// Its methods require a successfully resolved, non-nil workload with projections. +type Workload struct { + modelProjections []*ModelProjection + digest WorkloadDigest + scalingGroupReplicas int32 +} + +// ModelProjections returns a read-only view of the workload's model projections. +// Callers must not modify the slice or its projections. +func (w *Workload) ModelProjections() []*ModelProjection { + return w.modelProjections +} + +// Digest returns the workload's digest. +func (w *Workload) Digest() WorkloadDigest { + return w.digest +} + +// BuildFamily returns the workload's build family. +func (w *Workload) BuildFamily() BuildFamily { + return w.modelProjections[0].configuredBuild.Family +} + +// Pipeline returns the workload's runtime pipeline. +func (w *Workload) Pipeline() Pipeline { + return w.modelProjections[0].pipeline +} + +// ComponentNames returns this workload's members in canonical runtime order. +func (w *Workload) ComponentNames() []string { + // Draft fanout can contribute several consecutive models from one component. + var names []string + for _, projection := range w.modelProjections { + if len(names) == 0 || names[len(names)-1] != projection.stage { + names = append(names, projection.stage) + } + } + return names +} + +// ServingComponentName returns the conductor owner, placed last by ResolveWorkload. +func (w *Workload) ServingComponentName() string { + return w.modelProjections[len(w.modelProjections)-1].stage +} diff --git a/deploy/operator/internal/podcache/transform.go b/deploy/operator/internal/podcache/transform.go index daf6295b10eb..3691e666f8dd 100644 --- a/deploy/operator/internal/podcache/transform.go +++ b/deploy/operator/internal/podcache/transform.go @@ -91,8 +91,8 @@ func projectSpec(in corev1.PodSpec) corev1.PodSpec { Containers: projectContainers(in.Containers), // GMS Pod replacement requires native init-sidecar identity. InitContainers: projectInitContainers(in.InitContainers), - // Topology label discovery inspects DownwardAPI label field paths. - Volumes: projectTopologyVolumes(in.Volumes), + // Topology uses DownwardAPI fields; LPX eviction uses runtime ConfigMap references. + Volumes: projectVolumes(in.Volumes), } } @@ -124,9 +124,20 @@ func projectInitContainers(in []corev1.Container) []corev1.Container { return nil } -func projectTopologyVolumes(in []corev1.Volume) []corev1.Volume { +func projectVolumes(in []corev1.Volume) []corev1.Volume { out := make([]corev1.Volume, 0, len(in)) for i := range in { + // Keep only ConfigMap identity, not its projected keys or mount settings. + if in[i].ConfigMap != nil { + out = append(out, corev1.Volume{ + Name: in[i].Name, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: in[i].ConfigMap.LocalObjectReference, + }}, + }) + continue + } + if in[i].DownwardAPI == nil { continue } diff --git a/deploy/operator/internal/podcache/transform_test.go b/deploy/operator/internal/podcache/transform_test.go index 7afc9fae1ae9..0fd29f272441 100644 --- a/deploy/operator/internal/podcache/transform_test.go +++ b/deploy/operator/internal/podcache/transform_test.go @@ -300,3 +300,21 @@ func TestConfigureRejectsDuplicatePodEntries(t *testing.T) { func resourceQuantity(value string) resource.Quantity { return resource.MustParse(value) } + +func TestProjectRetainsRuntimeConfigMapIdentity(t *testing.T) { + t.Log("Project a runtime ConfigMap volume without retaining its file layout") + pod := &corev1.Pod{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{ + Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "pcs-workload-lpu-hash"}, + Items: []corev1.KeyToPath{{Key: "partition_models", Path: "models"}}, + Optional: ptr.To(false), DefaultMode: ptr.To(int32(0444)), + }}, + }}}} + require.Equal(t, []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "pcs-workload-lpu-hash"}}, + }}}, Project(pod).Spec.Volumes) + + t.Log("Reapplying the cache projection preserves the same reference") + before := pod.DeepCopy() + require.Equal(t, before, Project(pod)) +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index ef64a3805a1c..08fc5d69b67b 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -303,10 +303,13 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( allErrs := field.ErrorList{} // LPX cardinality determines the root overrides and shared draft/target contract. - lpxComponentCount := 0 + lpxComponentCount, independentEngineCount := 0, 0 for i := range spec.Components { if spec.Components[i].IsLPX() { lpxComponentCount++ + if spec.Components[i].ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) != nil { + independentEngineCount++ + } } } @@ -333,7 +336,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( if len(spec.Components) == 0 { allErrs = append(allErrs, field.Required(componentsPath, "must have at least one component")) } - if lpxComponentCount > 2 { + if lpxComponentCount > 2 && independentEngineCount != lpxComponentCount { allErrs = append(allErrs, field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")) } components := componentsByName(spec.Components) @@ -343,7 +346,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( componentPath := componentsPath.Index(i) // Draft fanout expands models; the target owns the shared endpoint and availability. - if lpxComponentCount == 2 && component.IsLPX() { + if lpxComponentCount == 2 && independentEngineCount == 1 && component.IsLPX() { replicas := k8sptr.Deref(component.Replicas, 1) if component.ComponentRole(nvidiacomv1beta1.ComponentRoleLPXConductor) == nil { if replicas > dynamolpx.MaxSpecDecodeNumDrafts { @@ -447,8 +450,8 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( } } } - if hasLPXComponent && conductorCount != 1 { - allErrs = append(allErrs, field.Forbidden(conductorComponentsPath, "LPX components must declare exactly one conductor role")) + if hasLPXComponent && conductorCount != lpxComponentCount && !(lpxComponentCount == 2 && conductorCount == 1) { + allErrs = append(allErrs, field.Forbidden(conductorComponentsPath, "LPX components must each declare a conductor role or form a shared draft and target pair")) } if spec.Restart != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index d35be8b82e82..a7562b307728 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -14,9 +14,9 @@ import ( // lpxDGDAdmissionCases builds fresh LPX scenarios for the single native DGD admission table. func lpxDGDAdmissionCases() []dgdAdmissionTestCase { const longLPXComponentName = "abcdefghijklmnopqrstuvwxyzabcd" - const conductorRoleErr = "spec.components: Forbidden: LPX components must declare exactly one conductor role" + const conductorRoleErr = "spec.components: Forbidden: LPX components must each declare a conductor role or form a shared draft and target pair" const conductorTemplateErr = "spec.components[0].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" - const alphaConductorRoleErr = "spec.services: Forbidden: LPX components must declare exactly one conductor role" + const alphaConductorRoleErr = "spec.services: Forbidden: LPX components must each declare a conductor role or form a shared draft and target pair" const alphaConductorTemplateErr = "spec.services[lpx].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" // Keep LPX inputs and oracles together without a separate admission execution path. @@ -489,7 +489,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { extra.ComponentName = "extra-draft" dgd.Spec.Components = append(dgd.Spec.Components, *extra) }), - wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair"}, + wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair", conductorRoleErr}, }, { name: "LPX rejects zero draft replicas", @@ -682,14 +682,43 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantCELErr: "spec.services[lpx]: Invalid value: lpx may only be set when componentType is lpx", }, { - name: "LPX rejects more than one conductor across components", + name: "LPX admits independent conductors across components", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { second := dgd.Spec.Components[0] second.ComponentName = "lpx-2" dgd.Spec.Components = append(dgd.Spec.Components, second) }), - wantWebhookErrs: []string{conductorRoleErr}, }, + { + name: "LPX admits three independent engines with unrelated replica counts", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Replicas = k8sptr.To(int32(3)) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName, second.Replicas = "second", nil + third := dgd.Spec.Components[0].DeepCopy() + third.ComponentName, third.Replicas = "third", k8sptr.To(int32(9)) + dgd.Spec.Components = append(dgd.Spec.Components, *second, *third) + }), + }, + { + name: "v1alpha1 LPX admits independent conductors", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + second := dgd.Spec.Services["lpx"].DeepCopy() + second.Replicas = k8sptr.To(int32(3)) + dgd.Spec.Services["second"] = second + }), + }, + { + name: "LPX rejects ambiguous shared and independent engines", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, nil) + second := dgd.Spec.Components[1].DeepCopy() + second.ComponentName = "independent" + dgd.Spec.Components = append(dgd.Spec.Components, *second) + }), + wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair", conductorRoleErr}, + }, + { name: "shared LPX draft does not consume the target Grove name budget", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index d3726f688d5b..abfac117dde8 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -338,7 +338,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. A DGD may contain independent
LPX components, each with its own conductor role, or a shared draft and
target pair with a conductor role only on the target. Every LPX role
requires its own podTemplate. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -385,7 +385,7 @@ _Appears in:_ | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| | `minAvailable` _integer_ | MinAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | Multinode configures worker, prefill, or decode components that span
multiple Pods. | | | -| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | Roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. A DGD may contain independent
LPX components, each with its own conductor role, or a shared draft and
target pair with a conductor role only on the target. Every LPX role
requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| @@ -1902,7 +1902,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. | | Optional: \{\}
| +| `roles` _[ComponentRoleSpec](#componentrolespec) array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. A DGD may contain independent
LPX components, each with its own conductor role, or a shared draft and
target pair with a conductor role only on the target. Every LPX role
requires its own podTemplate. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| @@ -1936,7 +1936,7 @@ _Appears in:_ | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodCliqueScalingGroup minAvailable for
components rendered as a scaling group (multi-node, inter-pod GMS, or
`experimental.grove.forceScalingGroup`; see `UsesPCSG`) and to Grove
PodClique minAvailable for all other single-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures worker, prefill, or decode components that span
multiple Pods. | | Optional: \{\}
| -| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. All LPX components in a DGD
must declare exactly one conductor role in total. Every LPX role requires
its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| +| `roles` _object array_ | roles expose the named Pod-producing parts inside a compound component.
When set for a multinode component, this list must contain exactly one
leader and one worker role. Admission defaults omitted replicas to 1 for
leader and multinode.nodeCount minus 1 for worker. Omitting the roles list
preserves the implicit multinode role layout.
LPX components each require an agent role. A DGD may contain independent
LPX components, each with its own conductor role, or a shared draft and
target pair with a conductor role only on the target. Every LPX role
requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. | | Optional: \{\}
| | `sharedMemorySize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api)_ | sharedMemorySize controls the size of the tmpfs mounted at `/dev/shm`.
`nil` selects the operator default (8Gi), a positive quantity sets a
custom size, and `"0"` disables the shared-memory volume entirely.
Simpler replacement for v1alpha1's `SharedMemorySpec` struct with its
`disabled bool` + `size Quantity` pattern. | | Optional: \{\}
| | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 5804c83ef3a7..799b269ba98a 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -76,8 +76,6 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Desired number of complete component instances. For a single-node component, each instance is one pod. For a multinode component, each instance contains all of its roles. Minimum `0`. When `scalingAdapter` is set, this field is owned by the [DynamoGraphDeploymentScalingAdapter](full-api-reference.mdx#dynamographdeploymentscalingadapter) and should not be modified directly. For a single `type: lpx` component in a DGD, this counts complete engine replicas, not individual role Pods. Hybrid and LPU-only engines allow up to `2496` replicas; generated Pod hostname limits may lower that maximum. In a speculative pair, draft replicas count model instances (`1–8`), and target replicas must be `1`. LPX does not support scaling adapters or scale-to-zero. - - When LPX serving-component `replicas` is omitted, the controller follows the live Grove `PodCliqueScalingGroup` (PCSG) replica count without writing its Scale subresource. This externally managed count is lost when the `PodCliqueSet` (PCS) or PCSG is replaced, including PCS replacement for immutable workload or compiler-provenance changes. The new group starts from the template's `minAvailable` seed (default `1`); the external scaler must reapply its desired capacity. Explicit DGD `replicas` are reapplied by the operator after replacement. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 0e7c728034e2..c8c003fbc4d3 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -91,7 +91,7 @@ spec: - Best-effort deadline for each LPU pipeline request's scheduling cycle. The timer starts at the scheduler's `status.schedulingStartedAt`; without that timestamp, no deadline is running. A later scheduling cycle supplies a new timestamp. Valid values are `1` through `9,223,372,036`; omission means unlimited. This field is not a solver budget or an overall deployment readiness timeout. + Best-effort deadline for each LPU pipeline request to leave Pending, including waiting for the scheduler to start. The timer uses a nonzero `status.schedulingStartedAt` when available; otherwise it uses `metadata.creationTimestamp`. A scheduler-provided timestamp for a later scheduling cycle takes precedence over the request's original creation time. Valid values are `1` through `9,223,372,036`; omission means unlimited. This field is not a solver budget or an overall deployment readiness timeout. Current-generation `Bound`, `NoFit`, and `Unsupported` receipts are exempt. `Degraded`, `Releasing`, and `Released` receipts are also exempt when their committed execution was accepted for the current request generation. `NoFit` can therefore leave the deployment Pending indefinitely, and a `Bound` request can still wait indefinitely for runtime readiness. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index ccacd66f6cfa..5aa9bab147ba 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -471,7 +471,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role requires its own podTemplate. See [ComponentRoleSpec](#componentrolespec). **Validation:** Optional: \{\} @@ -644,7 +644,7 @@ See [MultinodeSpec](#multinodespec). -Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +Roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} @@ -2689,7 +2689,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role requires its own podTemplate. See [ComponentRoleSpec](#v1beta1-componentrolespec). **Validation:** Optional: \{\} @@ -2804,7 +2804,7 @@ See [MultinodeSpec](#v1beta1-multinodespec). -roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. All LPX components in a DGD must declare exactly one conductor role in total. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. +roles expose the named Pod-producing parts inside a compound component. When set for a multinode component, this list must contain exactly one leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role requires its own podTemplate. Standalone DCD roles accept only `name` and `replicas`; `providerOverride` is a DGD-only provider context. **Validation:** Optional: \{\} From 6a7ded8bbbed73cf920df97b38a1128168d55a8a Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:53:40 -0700 Subject: [PATCH 112/196] fix(lpx): allowing scaling of cyborg replicas --- .../internal/controller/lpx/AGENTS.md | 8 +- .../internal/controller/lpx/controller.go | 126 ++- .../controller/lpx/controller_test.go | 895 ++++++++++++------ .../internal/controller/lpx/eviction_test.go | 6 +- .../controller/lpx/pipeline_request.go | 22 +- .../lpx/pipeline_request_deadline.go | 17 +- .../lpx/pipeline_request_deadline_test.go | 56 +- .../controller/lpx/pipeline_request_test.go | 27 +- .../internal/controller/lpx/render_test.go | 42 +- .../operator/internal/controller/lpx/util.go | 103 +- .../internal/controller/lpx/util_test.go | 191 +++- .../operator/internal/controller/lpx/watch.go | 22 +- deploy/operator/internal/dynamo/graph_lpx.go | 10 +- deploy/operator/internal/dynamo/grove_lpx.go | 79 +- .../internal/dynamo/grove_lpx_test.go | 8 +- .../internal/dynamo/lpx/cyborg_runtime.go | 17 + deploy/operator/internal/dynamo/lpx/render.go | 7 + 17 files changed, 1149 insertions(+), 487 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/AGENTS.md b/deploy/operator/internal/controller/lpx/AGENTS.md index f604a1ba5829..15d92da77f41 100644 --- a/deploy/operator/internal/controller/lpx/AGENTS.md +++ b/deploy/operator/internal/controller/lpx/AGENTS.md @@ -13,6 +13,7 @@ SPDX-License-Identifier: Apache-2.0 - Recovery from direct edits, force-deletion, or orphaning of managed resources are out of scope unless explicitly requested. - Do not add migration or recovery machinery solely for unsupported cases. +- Minimize or exclude self-evident comments. # Reconciliation @@ -95,11 +96,11 @@ SPDX-License-Identifier: Apache-2.0 PCS replacement. - Seed the PCS's scaling-group template from immutable `MinAvailable` (default one), never from live or desired capacity. Apply explicit desired capacity - through PCSG `/scale` before publishing requests. The seed stays positive and - unchanged even when live capacity is zero. + through the PCSG `/scale` subresource before publishing requests. The seed stays + positive and unchanged even when live capacity is zero. - Explicit DGD replicas give this controller capacity ownership. Omitted replicas leave capacity externally managed: derive request ordinals from the - observed PCSG and never write `/scale`, including during deadline cleanup. + observed PCSG and never change its capacity, including during deadline cleanup. - External capacity is not persisted across PCS or PCSG replacement. A new group starts from the immutable template seed; its external scaler must reapply the desired count. Explicit DGD replicas are reapplied by this controller. @@ -238,3 +239,4 @@ SPDX-License-Identifier: Apache-2.0 using the repository's envtest assets. Run LPX race tests for lifecycle changes. - Regenerate the condition golden with `go test ./internal/controller/lpx -run TestPipelineRequestReadyConditionGolden -args -update`. +- Do not write tautological tests. diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index db2a78abd689..c066bae4266d 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -19,6 +19,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -81,6 +82,7 @@ func (r *graphReconciler) GetRecorder() events.EventRecorder { // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update // +kubebuilder:rbac:groups=grove.io,resources=podcliquesets/finalizers,verbs=update +// +kubebuilder:rbac:groups=grove.io,resources=podcliques/scale;podcliquescalinggroups/scale,verbs=update // +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete // Reconcile observes dependencies once, then persists the final child status. @@ -151,7 +153,8 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu var ( pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup - requests []lpxv1alpha1.LPUPipelineRequest + pclqs map[string]*grovev1alpha1.PodClique + requests map[string]*lpxv1alpha1.LPUPipelineRequest ) if pcs != nil { @@ -166,7 +169,11 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, nil } - // Foreground PCS replacement waits for its blocking LPRs; discover them only by owner UID. + pclqs, err = getPodCliques(ctx, r.Client, pcs, pcsgs) + if err != nil { + return ctrl.Result{}, err + } + requests, err = r.getPipelineRequests(ctx, pcs) if err != nil { return ctrl.Result{}, err @@ -177,20 +184,21 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return result, err } - return r.reconcileWorkloads(ctx, deployment, dgd, pcs, pcsgs, requests) + return r.reconcileWorkloads(ctx, deployment, dgd, pcs, pcsgs, pclqs, requests) } // reconcileWorkloads consumes owned observations; DGD and deployment are non-nil. -// A nil pcs means initial creation, with no pcsgs or requests. Otherwise pcsgs -// contains every configured, non-deleting group. On error, Reconcile persists -// status before applying deadline retries. +// A nil pcs means initial creation, with no pcsgs, pclqs or requests. Otherwise +// pcsgs contains every configured group; pclqs contains owned, non-deleting cliques. +// On error, Reconcile persists status before applying deadline retries. func (r *graphReconciler) reconcileWorkloads( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, dgd *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, - requests []lpxv1alpha1.LPUPipelineRequest, + pclqs map[string]*grovev1alpha1.PodClique, + requests map[string]*lpxv1alpha1.LPUPipelineRequest, ) (result ctrl.Result, err error) { // Resolve and render every workload before deleting the running PCS. workloads, plans, err := r.resolveWorkloads(ctx, deployment, dgd) @@ -204,7 +212,7 @@ func (r *graphReconciler) reconcileWorkloads( return ctrl.Result{}, err } - // Immutable composition changes replace the PCS; replica-only changes use /scale. + // Immutable composition changes replace the PCS; replica-only changes patch live capacity. if pcs != nil && pcs.Annotations[lpx.WorkloadDigestAnnotation] != desiredPCS.Annotations[lpx.WorkloadDigestAnnotation] { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") return ctrl.Result{}, deletePodCliqueSet(ctx, r, pcs) @@ -219,13 +227,6 @@ func (r *graphReconciler) reconcileWorkloads( missingRequests []*lpxv1alpha1.LPUPipelineRequest ) - // Index the graph's observed requests once, retaining scheduler-owned receipts. - requestsByName := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(requests)) - for i := range requests { - request := &requests[i] - requestsByName[request.Name] = request - } - // Resolve each workload in the same order used for request publication. for _, groupName := range groupNames { plan := plans[groupName] @@ -241,7 +242,7 @@ func (r *graphReconciler) reconcileWorkloads( } } - desired, missing, intentChanged := resolvePipelineRequests(deployment, requestsByName, workload, plan) + desired, missing, intentChanged := resolvePipelineRequests(deployment, requests, workload, plan) if intentChanged { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") @@ -259,11 +260,15 @@ func (r *graphReconciler) reconcileWorkloads( }() // Scale only existing groups with explicitly managed capacity. + var capacityChanged bool for groupName, pcsg := range pcsgs { if replicas := explicitReplicas[groupName]; replicas != nil { - if err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, *replicas); err != nil { + changed, err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, *replicas) + if err != nil { return ctrl.Result{}, err } + + capacityChanged = capacityChanged || changed } } @@ -273,9 +278,14 @@ func (r *graphReconciler) reconcileWorkloads( return ctrl.Result{}, err } + if capacityChanged { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting to observe LPX capacity changes") + return ctrl.Result{}, nil + } + // Failure blocks publication while each workload independently cleans up expired suffixes. if len(expiredRequests) > 0 || isSchedulingFailedConditionCurrent(deployment) { - retained := slices.DeleteFunc(slices.Clone(requests), func(request lpxv1alpha1.LPUPipelineRequest) bool { + retained := slices.DeleteFunc(slices.Collect(maps.Values(requests)), func(request *lpxv1alpha1.LPUPipelineRequest) bool { _, desired := desiredRequests[request.Name] return !desired }) @@ -305,18 +315,26 @@ func (r *graphReconciler) reconcileWorkloads( } // Observe the created or updated PCS before scaling or publishing requests. - if pcs == nil || modified { + if modified { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for Grove to observe the workload") return ctrl.Result{}, nil } // Scale-out proceeds after deleting old names, without waiting for Pods. for _, groupName := range groupNames { + workload := workloads[groupName] plan := plans[groupName] - if replicas := explicitReplicas[plan.LPXScalingGroup]; replicas != nil { - if err := scalePodCliqueScalingGroup(ctx, r, pcsgs[plan.LPXScalingGroup], *replicas); err != nil { - return ctrl.Result{}, err - } + pcsg := pcsgs[plan.LPXScalingGroup] + component := dgd.GetComponentByName(groupName) + + changed, err := r.reconcileWorkloadCapacity(ctx, component, pcsg, pclqs, workload, plan) + if err != nil { + return ctrl.Result{}, err + } + + if changed { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting to observe LPX capacity changes") + return ctrl.Result{}, nil } } @@ -324,14 +342,50 @@ func (r *graphReconciler) reconcileWorkloads( return ctrl.Result{}, r.reconcilePipelineRequests(ctx, deployment, pcs, missingRequests) } - defer func() { - // Old configmaps remain available until all workloads are Ready. - if err == nil { - err = r.deleteUnusedConfigMaps(ctx, deployment, resources) + result = r.reconcileReadiness(ctx, deployment, dgd, pcs, pcsgs, pclqs, plans, desiredRequests) + // Old configmaps remain available until all workloads are Ready. + return result, r.deleteUnusedConfigMaps(ctx, deployment, resources) +} + +// reconcileWorkloadCapacity applies explicit capacity and validates external Cyborg counts. +// Component, pcsg, workload and plan are non-nil. Omitted replica counts are never written. +// The returned bool reports successful scale writes that require a fresh observation. +func (r *graphReconciler) reconcileWorkloadCapacity( + ctx context.Context, + component *v1beta1.DynamoComponentDeploymentSharedSpec, + pcsg *grovev1alpha1.PodCliqueScalingGroup, + pclqs map[string]*grovev1alpha1.PodClique, + workload *lpx.Workload, + plan *lpx.MaterializationPlan, +) (bool, error) { + if replicas := component.Replicas; replicas != nil { + changed, err := scalePodCliqueScalingGroup(ctx, r, pcsg, *replicas) + if changed || err != nil { + return changed, err } - }() + } - return r.reconcileReadiness(ctx, deployment, dgd, pcs, pcsgs, plans, desiredRequests) + if plan.CyborgTemplate == "" { + return false, nil + } + + if replicas := component.ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas; replicas != nil { + return scalePodCliques(ctx, r, pcsg, pclqs, plan.CyborgTemplate, *replicas) + } + + for index := range pcsg.Spec.Replicas { + name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(index)}, plan.CyborgTemplate) + pclq := pclqs[name] + if pclq == nil { + continue + } + + if err := workload.ValidateCyborgReplicas(pclq.Spec.Replicas); err != nil { + return false, err + } + } + + return false, nil } // reconcileRuntimeResources synchronizes additional resources for the LPX deployment. @@ -354,9 +408,10 @@ func (r *graphReconciler) reconcileReadiness( dgd *v1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, + pclqs map[string]*grovev1alpha1.PodClique, plans map[string]*lpx.MaterializationPlan, requests map[string]*lpxv1alpha1.LPUPipelineRequest, -) (ctrl.Result, error) { +) ctrl.Result { readiness := dynamo.GroveReadiness{Ready: true} deployment.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) @@ -365,10 +420,7 @@ func (r *graphReconciler) reconcileReadiness( for _, groupName := range slices.Sorted(maps.Keys(plans)) { plan := plans[groupName] - observed, err := dynamo.EvaluateLPXGroveReadiness(ctx, r.Client, dgd, groupName, componentGroups[groupName], pcs, pcsgs[plan.LPXScalingGroup]) - if err != nil { - return ctrl.Result{}, err - } + observed := dynamo.EvaluateLPXGroveReadiness(ctx, dgd, groupName, componentGroups[groupName], pcs, pcsgs[plan.LPXScalingGroup], pclqs) maps.Copy(deployment.Status.Components, observed.ComponentStatuses) @@ -378,12 +430,12 @@ func (r *graphReconciler) reconcileReadiness( } if !setPipelineRequestReadyCondition(deployment, requests) { - return ctrl.Result{}, nil + return ctrl.Result{} } if !readiness.Ready { setReadyCondition(deployment, v1beta1.DGDStatePending, readiness.Message) - return ctrl.Result{}, nil + return ctrl.Result{} } setReadyCondition(deployment, v1beta1.DGDStateSuccessful, readiness.Message) @@ -393,7 +445,7 @@ func (r *graphReconciler) reconcileReadiness( delay = max(modelDownloadRequeueAfter, time.Until(download.LastCheckedAt.Add(modelDownloadRefreshInterval))) } - return ctrl.Result{RequeueAfter: delay}, nil + return ctrl.Result{RequeueAfter: delay} } // deleteUnusedConfigMaps runs only after readiness so existing pods keep their diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 78fcdcf5d114..c4370ef49214 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -47,6 +47,276 @@ import ( "sigs.k8s.io/yaml" ) +func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { + t.Log("Materialize two hybrid workloads, each with two backbones") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + dgd.Spec.Components[0].Replicas = ptr.To(int32(2)) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName = "second" + second.ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(3)) + dgd.Spec.Components = append(dgd.Spec.Components, *second) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + pcs, _, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + objects := materializeLPXTestPCS(t, child, pcs, plans["lpx"], plans[second.ComponentName]) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 2, 2 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 2, 2 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + } + } + createLPXTestObjects(t, t.Context(), r.Client, objects...) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 3 { + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + } + + t.Log("Accept scheduler requests and observe initial Cyborg capacity independently of the seed") + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 4) + requestUIDs := make(map[string]types.UID) + for _, request := range requests { + requestUIDs[request.Name] = request.UID + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(t.Context(), request)) + } + for _, object := range objects { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) + if clique, ok := object.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker { + want := int32(1) + if clique.Labels[lpx.StageLabel] == second.ComponentName { + want = 3 + } + require.Equal(t, want, clique.Spec.Replicas) + clique.Status.ObservedGeneration = ptr.To(clique.Generation) + clique.Status.Replicas, clique.Status.UpdatedReplicas = want, want + clique.Status.ReadyReplicas, clique.Status.ScheduledReplicas = want, want + require.NoError(t, r.Update(t.Context(), clique)) + } + } + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + + for _, replicas := range []*int32{ptr.To(int32(3)), ptr.To(int32(1)), nil} { + t.Log("Change only the first workload's Cyborg count, then wait for its workers") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) + dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = replicas + require.NoError(t, r.Update(t.Context(), dgd)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(t.Context(), child)) + want := ptr.Deref(replicas, 4) + if replicas == nil { + t.Log("With an omitted count, external scaling may choose a different capacity on each backbone") + for ordinal := range plans["lpx"].Replicas { + clique := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: pcs.Namespace, Name: plans["lpx"].ForReplica(ordinal).CyborgClique}, clique)) + clique.Spec.Replicas = want + ordinal + clique.Generation++ + require.NoError(t, r.Update(t.Context(), clique)) + } + } + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + + t.Log("Observe the capacity write before calculating each workload's readiness") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.False(t, child.Status.Components["lpx"].Ready) + require.True(t, child.Status.Components[second.ComponentName].Ready) + + t.Log("Preserve PCS identity and spec, both groups, all Agents, and the other workload's workers") + for _, original := range objects { + if clique, ok := original.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker && clique.Labels[lpx.StageLabel] == "lpx" { + continue + } + actual := original.DeepCopyObject().(client.Object) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(original), actual)) + require.Equal(t, original, actual) + } + requests, err = r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 4) + for _, request := range requests { + require.Equal(t, requestUIDs[request.Name], request.UID) + } + + t.Log("Observe the requested live capacity and prove Ready without changing the template seed") + var workers []*grovev1alpha1.PodClique + for ordinal := range plans["lpx"].Replicas { + clique := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: pcs.Namespace, Name: plans["lpx"].ForReplica(ordinal).CyborgClique}, clique)) + expected := want + if replicas == nil { + expected += ordinal + } + require.Equal(t, expected, clique.Spec.Replicas) + require.Equal(t, ptr.To(int32(1)), clique.Spec.MinAvailable) + clique.Status.ObservedGeneration = ptr.To(clique.Generation) + clique.Status.Replicas, clique.Status.UpdatedReplicas = expected, expected + clique.Status.ReadyReplicas, clique.Status.ScheduledReplicas = expected, expected + require.NoError(t, r.Update(t.Context(), clique)) + workers = append(workers, clique) + } + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + for _, worker := range workers { + actual := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(worker), actual)) + require.Equal(t, worker, actual, "converged capacity must not write again") + } + } + + t.Log("Allow a user to resize an Agent clique while its new pods become ready") + agent := &grovev1alpha1.PodClique{} + agentKey := client.ObjectKey{Namespace: pcs.Namespace, Name: plans["lpx"].ForReplica(1).Agents[0].CliqueName} + require.NoError(t, r.Get(t.Context(), agentKey, agent)) + agent.Spec.Replicas++ + agent.Generation++ + require.NoError(t, r.Update(t.Context(), agent)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.False(t, child.Status.Components["lpx"].Ready) + require.True(t, child.Status.Components[second.ComponentName].Ready) + observedAgent := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), agentKey, observedAgent)) + require.Equal(t, agent, observedAgent, "reconciliation must preserve user-managed Agent capacity") + + t.Log("Accept the edited Agent capacity once Grove reports its live replica count ready") + agent.Status.ObservedGeneration = ptr.To(agent.Generation) + agent.Status.Replicas, agent.Status.UpdatedReplicas = agent.Spec.Replicas, agent.Spec.Replicas + agent.Status.ReadyReplicas, agent.Status.ScheduledReplicas = agent.Spec.Replicas, agent.Spec.Replicas + require.NoError(t, r.Update(t.Context(), agent)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + require.NoError(t, r.Get(t.Context(), agentKey, observedAgent)) + require.Equal(t, agent, observedAgent) +} + +func TestLPXExternalCyborgCapacityValidation(t *testing.T) { + t.Log("Use two explicitly managed backbones with externally managed workers in groups of four") + root := t.TempDir() + const buildID = "split-io" + writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, buildID, testV2GraphManifestFixture{ + topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", + partitionCount: 1, numChips: 8, devicesPerNode: 8, + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + ioFPGACount: 2, ioFanoutFactor: 2, + })) + registry, err := lpx.NewModelRegistry(root, nil) + require.NoError(t, err) + dgd := loadTestDGD(t, lpx.PipelineLPX, buildID) + dgd.Spec.Components[0].Replicas = ptr.To(int32(2)) + dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = nil + child := newLPXTestDeployment(t, dgd) + r := newLPXTestReconciler(t, registry, child, dgd) + workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) + require.NoError(t, err) + pcs, _, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) + require.NoError(t, err) + objects := materializeLPXTestPCS(t, child, pcs, plans["lpx"]) + for _, object := range objects { + switch live := object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + live.Status.Replicas, live.Status.UpdatedReplicas = 2, 2 + live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 2, 2 + case *grovev1alpha1.PodClique: + live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas + live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas + } + } + + t.Log("Leave the second backbone's worker clique missing while scheduler requests become bound") + worker := getResource[*grovev1alpha1.PodClique](t, objects, plans["lpx"].ForReplica(1).CyborgClique) + objects = slices.DeleteFunc(objects, func(object client.Object) bool { return object.GetName() == worker.Name }) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + for range 2 { + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + } + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 2) + for _, request := range requests { + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(t.Context(), request)) + } + result, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.Zero(t, result, "a missing clique waits for its watch") + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, v1alpha1.LPXReadyReasonPending, ready.Reason) + require.Contains(t, ready.Message, worker.Name) + + t.Log("Observe the missing clique and preserve the first backbone's independent worker capacity") + require.NoError(t, r.Create(t.Context(), worker)) + firstWorker := &grovev1alpha1.PodClique{} + firstWorkerKey := client.ObjectKey{Namespace: pcs.Namespace, Name: plans["lpx"].ForReplica(0).CyborgClique} + require.NoError(t, r.Get(t.Context(), firstWorkerKey, firstWorker)) + for _, step := range []struct { + replicas int32 + wantError string + }{ + {replicas: 8}, + {replicas: 5, wantError: "must be divisible by ioFpgaCount 2"}, + {replicas: 6, wantError: "must provide fanoutFactor 2 clients"}, + {replicas: 12}, + } { + t.Logf("Externally scale the second backbone to %d workers and report all of them ready", step.replicas) + worker.Spec.Replicas = step.replicas + worker.Generation++ + worker.Status.ObservedGeneration = ptr.To(worker.Generation) + worker.Status.Replicas, worker.Status.UpdatedReplicas = step.replicas, step.replicas + worker.Status.ReadyReplicas, worker.Status.ScheduledReplicas = step.replicas, step.replicas + require.NoError(t, r.Update(t.Context(), worker)) + _, err = r.Reconcile(t.Context(), key) + if step.wantError != "" { + require.ErrorContains(t, err, step.wantError) + } else { + require.NoError(t, err) + } + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + ready = meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, ready) + if step.wantError != "" { + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, ready.Reason) + require.Contains(t, ready.Message, step.wantError) + } else { + require.Equal(t, metav1.ConditionTrue, ready.Status) + } + + t.Log("Validation must not write either externally managed worker clique") + observed := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(worker), observed)) + require.Equal(t, worker, observed) + require.NoError(t, r.Get(t.Context(), firstWorkerKey, observed)) + require.Equal(t, firstWorker, observed) + } +} + func TestLPXWorkloadErrorDoesNotAcknowledgeGeneration(t *testing.T) { for _, tc := range []struct { name string @@ -198,7 +468,8 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { if failStatus { return statusErr } - } else if subresource == "scale" { + } + if subresource == "scale" { //nolint:goconst // Keep API subresource names inline in tests. scaleWrites++ } return delegated.SubResource(subresource).Update(ctx, object, opts...) @@ -268,8 +539,8 @@ func TestLPXDeadlineWaitsForScalingGroup(t *testing.T) { base := r.Client r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { - groups.Items = nil + if pcsgs, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + pcsgs.Items = nil return nil } return delegated.List(ctx, list, opts...) @@ -297,9 +568,9 @@ func TestLPXDeadlineWaitsForScalingGroup(t *testing.T) { t.Log("A later pass lowers capacity and removes the expired request") _, err = r.Reconcile(ctx, key) require.NoError(t, err) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: desired.plan.LPXScalingGroup}, group)) - require.Zero(t, group.Spec.Replicas) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: desired.plan.LPXScalingGroup}, pcsg)) + require.Zero(t, pcsg.Spec.Replicas) requirePipelineRequestNotFound(t, ctx, r.Client, expired.Namespace, expired.Name) } @@ -314,11 +585,10 @@ func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { createLPXTestObjects(t, ctx, r.Client, objects...) publishSelectedLPXForTest(t, ctx, r, child, desired) pcs := findLPXTestPodCliqueSet(t, objects) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) requests, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) - for index := range requests { - request := &requests[index] + for _, request := range requests { phase := lpxv1alpha1.RequestPhasePending if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex == 1 { phase = lpxv1alpha1.RequestPhaseBound @@ -336,8 +606,8 @@ func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { result, err = r.Reconcile(ctx, key) require.NoError(t, err) require.Zero(t, result) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(2), pcsg.Spec.Replicas) current, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) require.True(t, apiequality.Semantic.DeepEqual(requests, current), "interior failures must not mutate requests") @@ -350,8 +620,8 @@ func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { result, err = r.Reconcile(ctx, key) require.NoError(t, err) require.Zero(t, result) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Zero(t, group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Zero(t, pcsg.Spec.Replicas) current, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) require.Empty(t, current) @@ -662,11 +932,11 @@ func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { t.Log("Persist failure before changing Grove or deleting scheduler intent") request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) _, err := r.Reconcile(ctx, request) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(2), pcsg.Spec.Replicas) pending = getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) require.True(t, pending.DeletionTimestamp.IsZero()) @@ -676,8 +946,8 @@ func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { require.NoError(t, err) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), &grovev1alpha1.PodCliqueSet{})) require.True(t, apiequality.Semantic.DeepEqual(serving, getTestPipelineRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(1), pcsg.Spec.Replicas) } t.Log("A later edit still waits for the failed request's scheduler finalizer") pending = getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) @@ -687,8 +957,8 @@ func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { require.NoError(t, r.Update(ctx, child)) _, err = r.Reconcile(ctx, request) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(1), pcsg.Spec.Replicas) t.Log("After cleanup the later edit can retry without replacing the serving workload") retiredUID := pending.UID @@ -701,8 +971,8 @@ func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { replacement := getTestPipelineRequest(t, ctx, r.Client, pending.Namespace, pending.Name) require.NotEqual(t, retiredUID, replacement.UID) require.True(t, replacement.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(2), pcsg.Spec.Replicas) require.True(t, apiequality.Semantic.DeepEqual(serving, getTestPipelineRequest(t, ctx, r.Client, serving.Namespace, serving.Name)), "serving request changed") }) } @@ -762,7 +1032,7 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { createLPXTestObjects(t, ctx, r.Client, objects...) publishSelectedLPXForTest(t, ctx, r, child, selected) pcs := findLPXTestPodCliqueSet(t, objects) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) before := make([]*lpxv1alpha1.LPUPipelineRequest, len(selected.requests)) for index, request := range selected.requests { observed := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, request.Name) @@ -785,8 +1055,8 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { for range 3 { _, err := r.Reconcile(ctx, request) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.EqualValues(t, 3, group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.EqualValues(t, 3, pcsg.Spec.Replicas) tail := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, before[3].Name) require.False(t, tail.DeletionTimestamp.IsZero()) for _, original := range before[:3] { @@ -803,8 +1073,8 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { require.NoError(t, r.Update(ctx, remainingTail)) _, err := r.Reconcile(ctx, request) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.EqualValues(t, 1, group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.EqualValues(t, 1, pcsg.Spec.Replicas) for _, original := range before[1:3] { requirePipelineRequestNotFound(t, ctx, r.Client, original.Namespace, original.Name) } @@ -824,8 +1094,8 @@ func TestLPXExternalScaleRejectsInvalidReplicaCount(t *testing.T) { dgd.GetComponentByName("lpx").Replicas = nil r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) - group.Spec.Replicas = 2497 + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + pcsg.Spec.Replicas = 2497 createLPXTestObjects(t, ctx, r.Client, objects...) t.Log("Reject the observed replica count before publishing requests and leave external capacity unchanged") @@ -834,8 +1104,8 @@ func TestLPXExternalScaleRejectsInvalidReplicaCount(t *testing.T) { requests := &lpxv1alpha1.LPUPipelineRequestList{} require.NoError(t, r.List(ctx, requests)) require.Empty(t, requests.Items) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.EqualValues(t, 2497, group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.EqualValues(t, 2497, pcsg.Spec.Replicas) } func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { @@ -863,7 +1133,7 @@ func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) pcs := findLPXTestPodCliqueSet(t, objects) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) require.NoError(t, r.reconcilePipelineRequests(ctx, child, pcs, missing)) before := make([]*lpxv1alpha1.LPUPipelineRequest, len(missing)) for index, request := range missing { @@ -885,10 +1155,10 @@ func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { require.Equal(t, metav1.ConditionTrue, failure.Status) t.Log("Lower capacity externally while keeping the same failed input revision") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - group.Spec.Replicas = tc.replicas - require.NoError(t, r.Update(ctx, group)) - beforeGroup := group.DeepCopy() + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + pcsg.Spec.Replicas = tc.replicas + require.NoError(t, r.Update(ctx, pcsg)) + beforeGroup := pcsg.DeepCopy() deleted := make(map[client.ObjectKey]bool) r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { @@ -914,19 +1184,19 @@ func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { requirePipelineRequestNotFound(t, ctx, r.Client, original.Namespace, original.Name) } } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, beforeGroup, group) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, beforeGroup, pcsg) require.NoError(t, r.Get(ctx, request.NamespacedName, child)) require.Equal(t, failure, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) require.Equal(t, v1alpha1.LPXReadyReasonFailed, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) t.Log("A later external scale-out does not bypass the failed generation's publication gate") - group.Spec.Replicas = 4 - require.NoError(t, r.Update(ctx, group)) + pcsg.Spec.Replicas = 4 + require.NoError(t, r.Update(ctx, pcsg)) _, err = r.Reconcile(ctx, request) require.NoError(t, err) - for index := int(tc.replicas); index < len(before); index++ { - requirePipelineRequestNotFound(t, ctx, r.Client, before[index].Namespace, before[index].Name) + for _, retired := range before[tc.replicas:] { + requirePipelineRequestNotFound(t, ctx, r.Client, retired.Namespace, retired.Name) } }) } @@ -993,11 +1263,7 @@ func TestLPXInvalidEditsPreserveExistingWorkload(t *testing.T) { t.Log("Report the actionable failure without destroying existing resources") for range 3 { _, err = r.Reconcile(t.Context(), request) - if err != nil { - require.ErrorContains(t, err, scenario.message) - } else { - require.NoError(t, err) - } + require.ErrorContains(t, err, scenario.message) require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").Message, scenario.message) } @@ -1129,9 +1395,9 @@ func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { require.NoError(t, r.Update(ctx, pcs)) retiredTemplate := selected.plan.Agents[1].TemplateName originalCliqueCount := len(pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) + require.Contains(t, pcsg.Spec.CliqueNames, retiredTemplate) t.Log("Retire the immutable PCS without mutating Grove membership or directly deleting its LPRs") pcsDeleteObserved := false @@ -1158,8 +1424,8 @@ func TestLPXModelScaleDownRecreatesPodCliqueSet(t *testing.T) { require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) require.False(t, pcs.DeletionTimestamp.IsZero()) require.Contains(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].CliqueNames, retiredTemplate) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Contains(t, group.Spec.CliqueNames, retiredTemplate) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Contains(t, pcsg.Spec.CliqueNames, retiredTemplate) stale = getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) require.True(t, stale.DeletionTimestamp.IsZero()) @@ -1211,10 +1477,10 @@ func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { requestUID := request.UID t.Log("Scale the native Grove group to zero and derive empty scheduler intent") - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: initial.plan.LPXScalingGroup}, group)) - group.Spec.Replicas = 0 - require.NoError(t, r.Update(ctx, group)) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: initial.plan.LPXScalingGroup}, pcsg)) + pcsg.Spec.Replicas = 0 + require.NoError(t, r.Update(ctx, pcsg)) t.Log("Delete the stale LPR before reconciling the stable PCS template") _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) @@ -1222,16 +1488,16 @@ func TestLPXNativeScaleToZeroPreservesPodCliqueSetTemplate(t *testing.T) { require.True(t, apierrors.IsNotFound(r.Get(ctx, client.ObjectKeyFromObject(request), &lpxv1alpha1.LPUPipelineRequest{}))) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Zero(t, group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Zero(t, pcsg.Spec.Replicas) storedPCS := &grovev1alpha1.PodCliqueSet{} require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcs), storedPCS)) require.Equal(t, pcsUID, storedPCS.UID) require.Equal(t, templateReplicas, ptr.Deref(storedPCS.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas, 0)) t.Log("Scale the native group back up and publish a fresh request through the same PCS") - group.Spec.Replicas = 1 - require.NoError(t, r.Update(ctx, group)) + pcsg.Spec.Replicas = 1 + require.NoError(t, r.Update(ctx, pcsg)) _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) replacement := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, initial.requests[0].Name) @@ -1312,9 +1578,9 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { require.NoError(t, err) require.NoError(t, r.List(t.Context(), requests)) require.Len(t, requests.Items, 2) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas, "an incomplete handoff must not scale Grove") + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(2), pcsg.Spec.Replicas, "an incomplete handoff must not scale Grove") require.NoError(t, r.Get(t.Context(), request.NamespacedName, child)) if editDGD { require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, "Ready").Reason) @@ -1327,10 +1593,10 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { t.Log("Converge the new inputs without replacing the retained replica's request") _, err = r.Reconcile(t.Context(), request) require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.Equal(t, *dgd.GetComponentByName("lpx").Replicas, group.Spec.Replicas) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, *dgd.GetComponentByName("lpx").Replicas, pcsg.Spec.Replicas) require.NoError(t, r.List(t.Context(), requests)) - require.Len(t, requests.Items, int(group.Spec.Replicas)) + require.Len(t, requests.Items, int(pcsg.Spec.Replicas)) require.Equal(t, retained.UID, getTestPipelineRequest(t, t.Context(), r.Client, retained.Namespace, retained.Name).UID) }) } @@ -1418,44 +1684,208 @@ func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { require.Contains(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").Message, readError.Error()) } -func TestLPXScaleDownDoesNotRepeatWriteFromStaleCache(t *testing.T) { - t.Log("Observe an externally inflated group through a cache that stays stale after scale") - ctx := t.Context() +func TestLPXCorrectsTopLevelReplicaDrift(t *testing.T) { + t.Log("Materialize an extra PCS ordinal after the top-level replica count drifts to two") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, selected) - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) - group.Spec.Replicas = 9 - createLPXTestObjects(t, ctx, r.Client, objects...) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - stale := group.DeepCopy() - scaled, updates := false, 0 + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + pcs := findLPXTestPodCliqueSet(t, objects) + pcs.Spec.Replicas = 2 + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + extraGroup := pcsg.DeepCopy() + extraGroup.Name = grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 1}, desired.plan.ScalingGroupTemplate) + extraGroup.UID = types.UID(extraGroup.Name + "-uid") + extraGroup.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "1" + extraClique := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.ConductorClique).DeepCopy() + extraClique.Name = grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: extraGroup.Name, Replica: 0}, desired.plan.ConductorTemplate) + extraClique.UID = types.UID(extraClique.Name + "-uid") + extraClique.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "1" + extraClique.Labels[grovecommon.LabelPodCliqueScalingGroup] = extraGroup.Name + extraClique.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(extraGroup, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))} + objects = append(objects, extraGroup, extraClique) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(extraClique), extraClique)) + + t.Log("Ignore extra PCS ordinals so normal synchronization can restore one top-level replica") + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.NoError(t, err) + require.Zero(t, result) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.EqualValues(t, 1, pcs.Spec.Replicas) + observed := &grovev1alpha1.PodClique{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(extraClique), observed)) + require.Equal(t, extraClique, observed, "Grove owns cleanup of the extra ordinal") +} + +func TestLPXReconcileRejectsForeignPodClique(t *testing.T) { + t.Log("Materialize a workload whose workers need scaling") + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(3)) + r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, desired) + worker := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique) + worker.OwnerReferences[0].UID = "foreign" + createLPXTestObjects(t, t.Context(), r.Client, objects...) + writes := 0 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if observed, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok && scaled { - observed.Items = []grovev1alpha1.PodCliqueScalingGroup{*stale.DeepCopy()} - return nil - } - return delegated.List(ctx, list, opts...) - }, SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - if subresource != lpxTestScaleSubresource { - return delegated.SubResource(subresource).Update(ctx, object, opts...) - } - updates++ - if err := delegated.SubResource(subresource).Update(ctx, object, opts...); err != nil { - return err + if subresource == "scale" { + writes++ + return errors.New("unexpected capacity write") } - scaled = true - return nil + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + writes++ + return errors.New("unexpected resource publication") }, }) - t.Log("Continue reconciliation without issuing a second stale-resource-version write") - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.Equal(t, 1, updates) + t.Log("Reject foreign cliques before scaling or publishing requests") + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + require.ErrorContains(t, err, "is not controlled by") + require.Zero(t, writes) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + condition := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, condition) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, condition.Reason) +} + +func TestLPXScalingWaitsForObservedCapacity(t *testing.T) { + for _, tc := range []struct { + name string + groupReplicas int32 + wantGroupUpdates int + }{ + {name: "group scale down", groupReplicas: 9, wantGroupUpdates: 1}, + {name: "group scale up", groupReplicas: 1, wantGroupUpdates: 1}, + {name: "clique scale", groupReplicas: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe a hybrid workload with group or clique capacity still to apply") + ctx := t.Context() + child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) + dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(3)) + r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) + for _, object := range objects { + if pclq, ok := object.(*grovev1alpha1.PodClique); ok { + pclq.Status.Replicas, pclq.Status.UpdatedReplicas = pclq.Spec.Replicas, pclq.Spec.Replicas + pclq.Status.ReadyReplicas, pclq.Status.ScheduledReplicas = pclq.Spec.Replicas, pclq.Spec.Replicas + } + } + createLPXTestObjects(t, ctx, r.Client, objects...) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + _, err := r.Reconcile(ctx, key) + require.NoError(t, err) + publishSelectedLPXForTest(t, ctx, r, child, selected) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + pcs := findLPXTestPodCliqueSet(t, objects) + for _, desired := range selected.requests { + request := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.Name) + request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status + require.NoError(t, r.Update(ctx, request)) + } + pcsg.Spec.Replicas = tc.groupReplicas + pcsg.Status.Replicas, pcsg.Status.UpdatedReplicas = tc.groupReplicas, tc.groupReplicas + pcsg.Status.AvailableReplicas, pcsg.Status.ScheduledReplicas = tc.groupReplicas, tc.groupReplicas + require.NoError(t, r.Update(ctx, pcsg)) + cachedPclqs := &grovev1alpha1.PodCliqueList{} + require.NoError(t, r.List(ctx, cachedPclqs, client.InNamespace(pcs.Namespace), client.MatchingLabels{grovecommon.LabelPartOfKey: pcs.Name})) + cacheStale := true + groupUpdates, cliqueUpdates, cliqueLists := 0, 0, 0 + base := r.Client.(client.WithWatch) + r.Client = interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*grovev1alpha1.PodClique); ok { + t.Fatal("PodCliques must be listed once at the observation boundary") + } + return delegated.Get(ctx, key, object, opts...) + }, + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if observed, ok := list.(*grovev1alpha1.PodCliqueList); ok { + cliqueLists++ + options := (&client.ListOptions{}).ApplyOptions(opts) + require.Equal(t, pcs.Namespace, options.Namespace) + require.Equal(t, grovecommon.LabelPartOfKey+"="+pcs.Name+","+grovecommon.LabelPodCliqueSetReplicaIndex+"=0", options.LabelSelector.String()) + if cacheStale { + *observed = *cachedPclqs.DeepCopy() + return nil + } + } + return delegated.List(ctx, list, opts...) + }, + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if subresource == "scale" { + switch object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + groupUpdates++ + case *grovev1alpha1.PodClique: + cliqueUpdates++ + } + } + return delegated.SubResource(subresource).Update(ctx, object, opts...) + }, + Patch: func(ctx context.Context, delegated client.WithWatch, object client.Object, patch client.Patch, opts ...client.PatchOption) error { + switch object.(type) { + case *grovev1alpha1.PodCliqueScalingGroup, *grovev1alpha1.PodClique: + t.Fatal("capacity writes must use the scale subresource") + } + return delegated.Patch(ctx, object, patch, opts...) + }, + }) + + t.Log("A capacity write stops reconciliation before the cached ready 1/1 state can be reported") + result, err := r.Reconcile(ctx, key) + require.NoError(t, err) + require.Zero(t, result, "capacity observations resume reconciliation through watches") + require.Equal(t, tc.wantGroupUpdates, groupUpdates) + require.Equal(t, 1, cliqueLists) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) + if tc.wantGroupUpdates > 0 { + t.Log("Observe the group scale before updating its workers") + require.Zero(t, cliqueUpdates) + _, err = r.Reconcile(ctx, key) + require.NoError(t, err) + } + require.Equal(t, 2, cliqueUpdates) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + + t.Log("Once the cache observes capacity, wait for the new pods without repeating writes") + cacheStale = false + _, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.Equal(t, tc.wantGroupUpdates, groupUpdates) + require.Equal(t, 2, cliqueUpdates) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.False(t, child.Status.Components["lpx"].Ready) + require.Equal(t, ptr.To(int32(0)), child.Status.Components["lpx"].AvailableReplicas) + + t.Log("Report readiness only after Grove observes the new generation and all requested replicas") + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + pcsg.Status.ObservedGeneration = ptr.To(pcsg.Generation) + pcsg.Status.Replicas, pcsg.Status.UpdatedReplicas = 2, 2 + pcsg.Status.AvailableReplicas, pcsg.Status.ScheduledReplicas = 2, 2 + require.NoError(t, base.Update(ctx, pcsg)) + liveCliques := &grovev1alpha1.PodCliqueList{} + require.NoError(t, base.List(ctx, liveCliques)) + for i := range liveCliques.Items { + clique := &liveCliques.Items[i] + clique.Status.ObservedGeneration = ptr.To(clique.Generation) + clique.Status.Replicas, clique.Status.UpdatedReplicas = clique.Spec.Replicas, clique.Spec.Replicas + clique.Status.ReadyReplicas, clique.Status.ScheduledReplicas = clique.Spec.Replicas, clique.Spec.Replicas + require.NoError(t, base.Update(ctx, clique)) + } + _, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + }) + } } func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { @@ -1488,8 +1918,8 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { base := r.Client r.Client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { - groups.Items = nil + if pcsgs, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + pcsgs.Items = nil return nil } return delegated.List(ctx, list, opts...) @@ -1506,14 +1936,14 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { result, err = r.Reconcile(ctx, request) require.NoError(t, err) require.Zero(t, result, "request deletion progresses through watches") - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Equal(t, int32(1), group.Spec.Replicas) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) + require.Equal(t, int32(1), pcsg.Spec.Replicas) t.Log("Let request finalization proceed asynchronously after the lower scale is persisted") stale = getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) require.False(t, stale.DeletionTimestamp.IsZero()) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(1), pcsg.Spec.Replicas) require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(findLPXTestPodCliqueSet(t, objects)), &grovev1alpha1.PodCliqueSet{})) t.Log("Reverse the scale-down while the old request still protects its pods") @@ -1530,8 +1960,8 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { result, err = r.Reconcile(ctx, request) require.NoError(t, err) require.Zero(t, result, "terminating current names stop scale-out without polling") - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(1), group.Spec.Replicas, "Grove must not recreate pods protected by the retiring request") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(1), pcsg.Spec.Replicas, "Grove must not recreate pods protected by the retiring request") } t.Log("Allow scale-up and a fresh request once the scheduler finishes cleanup") @@ -1543,8 +1973,8 @@ func TestLPXScaleDownUpdatesGroveBeforeDeletingStaleRequest(t *testing.T) { _, err = r.Reconcile(ctx, request) require.NoError(t, err) } - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(2), group.Spec.Replicas) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(2), pcsg.Spec.Replicas) replacement := getTestPipelineRequest(t, ctx, r.Client, stale.Namespace, stale.Name) require.NotEqual(t, retiredUID, replacement.UID) require.True(t, replacement.DeletionTimestamp.IsZero()) @@ -1585,7 +2015,7 @@ func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { require.Empty(t, requests) t.Log("Observe Grove's group initialized from the PCS template") - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, groupTemplate.Name), Namespace: pcs.Namespace, @@ -1594,15 +2024,15 @@ func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { }, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: seed}, } - require.NoError(t, r.Create(t.Context(), group)) + require.NoError(t, r.Create(t.Context(), pcsg)) t.Log("Publish requests in ordinal order only after live capacity reaches the explicit desired count") var published []int64 r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { if request, ok := object.(*lpxv1alpha1.LPUPipelineRequest); ok { - require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(group), group)) - require.Equal(t, int32(3), group.Spec.Replicas) + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, int32(3), pcsg.Spec.Replicas) require.True(t, metav1.IsControlledBy(request, pcs)) require.True(t, ptr.Deref(metav1.GetControllerOf(request).BlockOwnerDeletion, false)) published = append(published, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex) @@ -1612,7 +2042,12 @@ func TestLPXStartupScalesMinimumSeedBeforePublishingRequests(t *testing.T) { }) result, err = r.Reconcile(t.Context(), key) require.NoError(t, err) - require.Zero(t, result, "Request creation events resume reconciliation") + require.Zero(t, result, "the PCSG watch resumes reconciliation") + require.Empty(t, published, "the scale response cannot be used as a complete PCSG observation") + + t.Log("Publish after the PCSG watch supplies its updated capacity") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) require.Equal(t, []int64{0, 1, 2}, published) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) @@ -1640,9 +2075,9 @@ func TestLPXTerminatingGroveResourcesBlockPublication(t *testing.T) { pending := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) pending.Status = newTestPipelineRequest(child, pcs, pending.Name, time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhasePending).Status require.NoError(t, r.Update(ctx, pending)) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - for _, object := range []client.Object{pcs, group} { + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) + for _, object := range []client.Object{pcs, pcsg} { object.SetFinalizers([]string{"example.com/cleanup"}) require.NoError(t, r.Update(ctx, object)) require.NoError(t, r.Delete(ctx, object)) @@ -1668,8 +2103,6 @@ func TestLPXTerminatingGroveResourcesBlockPublication(t *testing.T) { } } -const lpxTestScaleSubresource = "scale" - // updateTestDGD emulates the DGD controller publishing a new child revision. func updateTestDGD(t *testing.T, r *graphReconciler, child *v1alpha1.LPXGraphDeployment, desired *v1beta1.DynamoGraphDeployment) { t.Helper() @@ -1705,35 +2138,33 @@ func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { pcs := findLPXTestPodCliqueSet(t, objects) t.Log("Observe each draft condition against an independent copy of the ready baseline") - for _, scenario := range []string{"ready", "partial", "missing", "foreign", "old revision", "unobserved"} { + for _, scenario := range []string{"ready", "partial", "missing", "old revision", "unobserved"} { t.Run(scenario, func(t *testing.T) { - t.Log("Seed isolated Grove observations and fetch the first draft") - kubeClient := newLPXTestClient(t) - createLPXTestObjects(t, t.Context(), kubeClient, objects...) - firstDraft := getResource[*grovev1alpha1.PodClique](t, objects, selected.plan.Agents[0].CliqueName).DeepCopy() - require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(firstDraft), firstDraft)) + t.Log("Copy the observed cliques and select the first draft") + pclqs := make(map[string]*grovev1alpha1.PodClique) + for _, object := range objects { + if pclq, ok := object.(*grovev1alpha1.PodClique); ok { + pclqs[pclq.Name] = pclq.DeepCopy() + } + } + firstDraft := pclqs[selected.plan.Agents[0].CliqueName] require.Equal(t, "draft", firstDraft.Labels[lpx.StageLabel]) t.Log("Make only the first draft incomplete while retaining a fully ready second draft and target") switch scenario { case "partial": firstDraft.Status.ReadyReplicas-- - case "foreign": - firstDraft.OwnerReferences = []metav1.OwnerReference{{UID: "foreign", Controller: ptr.To(true)}} case "old revision": firstDraft.Status.CurrentPodCliqueSetGenerationHash = ptr.To("old") case "unobserved": firstDraft.Status.ObservedGeneration = ptr.To(firstDraft.Generation - 1) } if scenario == "missing" { - require.NoError(t, kubeClient.Delete(t.Context(), firstDraft)) - } else { - require.NoError(t, kubeClient.Update(t.Context(), firstDraft)) + delete(pclqs, firstDraft.Name) } t.Log("Project logical draft counts and make both authored components await the complete pair") - readiness, err := dynamo.EvaluateLPXGroveReadiness(t.Context(), kubeClient, dgd, dgd.GetComponentByName("lpx").ComponentName, singleGroupComponents(t, dgd), pcs, getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup)) - require.NoError(t, err) + readiness := dynamo.EvaluateLPXGroveReadiness(t.Context(), dgd, dgd.GetComponentByName("lpx").ComponentName, singleGroupComponents(t, dgd), pcs, getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup), pclqs) require.Len(t, readiness.ComponentStatuses, 2) draft, target := readiness.ComponentStatuses["draft"], readiness.ComponentStatuses["lpx"] require.Equal(t, v1beta1.ComponentKindPodClique, draft.ComponentKind) @@ -1954,93 +2385,6 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { require.Equal(t, before, pcs) } -func TestSelectedNodeLocalLPXReadinessUsesOneFixedScalingGroup(t *testing.T) { - for _, pipeline := range []lpx.Pipeline{lpx.PipelineSingle, lpx.PipelineLPX} { - t.Run(string(pipeline), func(t *testing.T) { - ctx := t.Context() - deployment, dgd, registry := newLPXTestDGD(t, pipeline) - if pipeline == lpx.PipelineLPX { - dgd.Spec.Components[0].Replicas = ptr.To(int32(1)) - dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(2)) - } - reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) - objects := lpxMaterializedObjects(t, reconciler, deployment, dgd, desired) - for _, object := range objects { - switch live := object.(type) { - case *grovev1alpha1.PodCliqueScalingGroup: - live.Status.Replicas, live.Status.UpdatedReplicas = 1, 1 - live.Status.AvailableReplicas, live.Status.ScheduledReplicas = 1, 1 - case *grovev1alpha1.PodClique: - live.Status.Replicas, live.Status.UpdatedReplicas = live.Spec.Replicas, live.Spec.Replicas - live.Status.ReadyReplicas, live.Status.ScheduledReplicas = live.Spec.Replicas, live.Spec.Replicas - require.Equal(t, desired.workload.ServingComponentName(), live.Labels[consts.KubeLabelDynamoComponent]) - } - } - createLPXTestObjects(t, ctx, reconciler.Client, objects...) - pcs := findLPXTestPodCliqueSet(t, objects) - rootReads := 0 - base, ok := reconciler.Client.(client.WithWatch) - require.True(t, ok) - reader := interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { - if _, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - rootReads++ - } - return reader.Get(ctx, key, object, opts...) - }}) - observe := func() dynamo.GroveReadiness { - groups, err := getPodCliqueScalingGroups(ctx, reconciler.Client, pcs) - group := groups[desired.plan.LPXScalingGroup] - require.NoError(t, err) - ready, err := dynamo.EvaluateLPXGroveReadiness(ctx, reader, dgd, dgd.GetComponentByName("lpx").ComponentName, singleGroupComponents(t, dgd), pcs, group) - require.NoError(t, err) - require.Len(t, ready.ComponentStatuses, 1) - require.Zero(t, rootReads, "reuse the synchronized PCS observation") - return ready - } - t.Log("Report one complete workload, never separate GPU or Agent component capacity") - ready := observe() - require.True(t, ready.Ready) - status := ready.ComponentStatuses[desired.workload.ServingComponentName()] - require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, status.ComponentKind) - require.Equal(t, []string{desired.plan.LPXScalingGroup}, status.ComponentNames) - require.Equal(t, int32(1), status.Replicas) - require.Equal(t, int32(1), status.UpdatedReplicas) - require.Equal(t, ptr.To(int32(1)), status.AvailableReplicas) - require.Equal(t, ptr.To(int32(1)), status.ScheduledReplicas) - if desired.plan.CyborgClique != "" { - t.Log("A stale GPU-role scale cannot satisfy complete-workload readiness") - gpu := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique).DeepCopy() - require.NoError(t, reconciler.Get(ctx, client.ObjectKeyFromObject(gpu), gpu)) - gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 1, 1, 1 - gpu.Status.ReadyReplicas, gpu.Status.ScheduledReplicas = 1, 1 - require.NoError(t, reconciler.Update(ctx, gpu)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, v1beta1.DGDReadyReasonUpdating, ready.Classification) - t.Log("Partial GPU readiness contributes zero complete workload replicas") - gpu.Spec.Replicas, gpu.Status.Replicas, gpu.Status.UpdatedReplicas = 2, 2, 2 - gpu.Status.ScheduledReplicas = 2 - require.NoError(t, reconciler.Update(ctx, gpu)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, v1beta1.DGDReadyReasonPodsNotReady, ready.Classification) - require.Equal(t, ptr.To(int32(0)), ready.ComponentStatuses[desired.workload.ServingComponentName()].AvailableReplicas) - gpu.Status.ReadyReplicas = 2 - require.NoError(t, reconciler.Update(ctx, gpu)) - require.True(t, observe().Ready) - } - t.Log("Reject stale complete-workload capacity even when every existing role is ready") - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, reconciler.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: desired.plan.LPXScalingGroup}, group)) - group.Spec.Replicas = 2 - require.NoError(t, reconciler.Update(ctx, group)) - ready = observe() - require.False(t, ready.Ready) - require.Equal(t, v1beta1.DGDReadyReasonUpdating, ready.Classification) - }) - } -} - func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { for _, scenario := range []struct{ name, message string }{ {"LPX", "LPX integration is disabled"}, @@ -2262,6 +2606,32 @@ func newLPXTestClient(t testing.TB, objects ...client.Object) client.WithWatch { Build() nextLPRUID := 0 return interceptor.NewClient(base, interceptor.Funcs{ + // The fake client implements /scale only for built-in Kubernetes workloads. + SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + if subresource != "scale" { + return delegated.SubResource(subresource).Update(ctx, object, opts...) + } + scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) + live := object.DeepCopyObject().(client.Object) + if err := delegated.Get(ctx, client.ObjectKeyFromObject(object), live); err != nil { + return err + } + switch resource := live.(type) { + case *grovev1alpha1.PodCliqueScalingGroup: + resource.Spec.Replicas = scale.Spec.Replicas + case *grovev1alpha1.PodClique: + resource.Spec.Replicas = scale.Spec.Replicas + default: + return fmt.Errorf("unsupported test scale resource %T", object) + } + live.SetResourceVersion(scale.ResourceVersion) + live.SetGeneration(live.GetGeneration() + 1) + if err := delegated.Update(ctx, live); err != nil { + return err + } + scale.ResourceVersion = live.GetResourceVersion() + return nil + }, // The fake client checks resource versions but does not enforce Delete UID preconditions. Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { options := (&client.DeleteOptions{}).ApplyOptions(opts) @@ -2276,17 +2646,6 @@ func newLPXTestClient(t testing.TB, objects ...client.Object) client.WithWatch { } return delegated.Delete(ctx, object, opts...) }, - // The fake client does not implement scale for Grove custom resources. - SubResourceUpdate: func(ctx context.Context, delegated client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok && subresource == "scale" { - options := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts) - scale := options.SubResourceBody.(*autoscalingv1.Scale) - group.ResourceVersion = scale.ResourceVersion - group.Spec.Replicas = scale.Spec.Replicas - return delegated.Update(ctx, group) - } - return delegated.SubResource(subresource).Update(ctx, object, opts...) - }, Create: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.CreateOption) error { if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok && pcs.UID == "" { pcs.UID = types.UID(pcs.Name + "-uid") @@ -2371,7 +2730,7 @@ func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment }) require.NotEqual(t, -1, groupIndex) groupTemplate := pcs.Spec.Template.PodCliqueScalingGroupConfigs[groupIndex].DeepCopy() - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ TypeMeta: metav1.TypeMeta{APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup"}, ObjectMeta: metav1.ObjectMeta{ Name: plan.LPXScalingGroup, Namespace: pcs.Namespace, UID: types.UID(plan.LPXScalingGroup + "-uid"), @@ -2386,13 +2745,13 @@ func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment ObservedGeneration: ptr.To(int64(1)), CurrentPodCliqueSetGenerationHash: ptr.To(generationHash), }, } - group.Labels[grovecommon.LabelPartOfKey] = pcs.Name - group.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" - objects = append(objects, group) + pcsg.Labels[grovecommon.LabelPartOfKey] = pcs.Name + pcsg.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" + objects = append(objects, pcsg) // Each Grove replica owns independent cliques. - for replicaIndex := int32(0); replicaIndex < group.Spec.Replicas; replicaIndex++ { - parent := grovecommon.ResourceNameReplica{Name: group.Name, Replica: int(replicaIndex)} + for replicaIndex := range pcsg.Spec.Replicas { + parent := grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(replicaIndex)} for _, template := range pcs.Spec.Template.Cliques { if !slices.Contains(groupTemplate.CliqueNames, template.Name) { continue @@ -2404,7 +2763,7 @@ func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: pcs.Namespace, UID: types.UID(name + "-uid"), Generation: 1, Labels: rendered.Labels, Annotations: rendered.Annotations, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, }, Spec: rendered.Spec, Status: grovev1alpha1.PodCliqueStatus{ @@ -2413,7 +2772,8 @@ func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment }, } clique.Labels[grovecommon.LabelPartOfKey] = pcs.Name - clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = group.Name + clique.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "0" + clique.Labels[grovecommon.LabelPodCliqueScalingGroup] = pcsg.Name clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = strconv.FormatInt(int64(replicaIndex), 10) for index, dependency := range clique.Spec.StartsAfter { clique.Spec.StartsAfter[index] = grovecommon.GeneratePodCliqueName(parent, dependency) @@ -2429,7 +2789,7 @@ func materializeLPXTestPCS(t *testing.T, deployment *v1alpha1.LPXGraphDeployment func createLPXTestScalingGroups(t *testing.T, kubeClient client.Client, pcs *grovev1alpha1.PodCliqueSet) { t.Helper() for _, config := range pcs.Spec.Template.PodCliqueScalingGroupConfigs { - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 0}, config.Name), Namespace: pcs.Namespace, @@ -2440,7 +2800,7 @@ func createLPXTestScalingGroups(t *testing.T, kubeClient client.Client, pcs *gro Replicas: ptr.Deref(config.Replicas, 1), MinAvailable: config.MinAvailable, CliqueNames: config.CliqueNames, }, } - require.NoError(t, kubeClient.Create(t.Context(), group)) + require.NoError(t, kubeClient.Create(t.Context(), pcsg)) } } @@ -2570,9 +2930,9 @@ func TestLPXCommittedDeadlineCleanupPreservesExternalCapacity(t *testing.T) { _, err = r.Reconcile(ctx, reconcileRequest) require.ErrorContains(t, err, "registry unavailable") require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Equal(t, selected.plan.Replicas, group.Spec.Replicas) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) + require.Equal(t, selected.plan.Replicas, pcsg.Spec.Replicas) request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) require.NotEmpty(t, request.Finalizers) @@ -2612,9 +2972,9 @@ func TestLPXCommittedReleasePreservesPods(t *testing.T) { require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) require.True(t, request.DeletionTimestamp.IsZero()) - group := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, group)) - require.Equal(t, selected.plan.Replicas, group.Spec.Replicas) + pcsg := &grovev1alpha1.PodCliqueScalingGroup{} + require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) + require.Equal(t, selected.plan.Replicas, pcsg.Spec.Replicas) }) } } @@ -2633,12 +2993,12 @@ func committedLPXTestPod(pod *corev1.Pod) *lpxv1alpha1.Committed { func TestLPXWaitsForAllScalingGroups(t *testing.T) { for _, tc := range []struct { name string - group string + pcsg string deleting bool }{ - {name: "missing explicit group", group: "lpx"}, - {name: "missing external group", group: "external"}, - {name: "deleting group", group: "external", deleting: true}, + {name: "missing explicit group", pcsg: "lpx"}, + {name: "missing external group", pcsg: "external"}, + {name: "deleting group", pcsg: "external", deleting: true}, } { t.Run(tc.name, func(t *testing.T) { t.Log("Create a PCS with explicit and externally managed workloads") @@ -2663,13 +3023,13 @@ func TestLPXWaitsForAllScalingGroups(t *testing.T) { if err := delegated.List(ctx, list, opts...); err != nil { return err } - if groups, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { - for i := range groups.Items { - if groups.Items[i].Name == plans[tc.group].LPXScalingGroup { + if pcsgs, ok := list.(*grovev1alpha1.PodCliqueScalingGroupList); ok { + for i := range pcsgs.Items { + if pcsgs.Items[i].Name == plans[tc.pcsg].LPXScalingGroup { if tc.deleting { - groups.Items[i].DeletionTimestamp = ptr.To(metav1.Now()) + pcsgs.Items[i].DeletionTimestamp = ptr.To(metav1.Now()) } else { - groups.Items = slices.Delete(groups.Items, i, i+1) + pcsgs.Items = slices.Delete(pcsgs.Items, i, i+1) } break } @@ -2784,8 +3144,7 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { {Name: secondPlan.LPXScalingGroup, ReplicaIndex: 2}, }, publishedTargets) original := make(map[string]types.UID) - for i := range requests { - request := &requests[i] + for _, request := range requests { original[request.Name] = request.UID require.Equal(t, "default", request.Annotations[pipelineRequestModelAnnotation]) request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status @@ -2811,8 +3170,7 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { t.Log("A missing role in the second workload leaves the first workload's readiness intact") clique := &grovev1alpha1.PodClique{} require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: secondPlan.ConductorClique}, clique)) - clique.Status.ReadyReplicas = 0 - require.NoError(t, r.Update(t.Context(), clique)) + require.NoError(t, r.Delete(t.Context(), clique)) _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) @@ -2829,10 +3187,10 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { require.NoError(t, r.Update(t.Context(), child)) _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) - groups, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) + pcsgs, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) require.NoError(t, err) - require.EqualValues(t, 1, groups[firstPlan.LPXScalingGroup].Spec.Replicas) - require.EqualValues(t, 3, groups[secondPlan.LPXScalingGroup].Spec.Replicas) + require.EqualValues(t, 1, pcsgs[firstPlan.LPXScalingGroup].Spec.Replicas) + require.EqualValues(t, 3, pcsgs[secondPlan.LPXScalingGroup].Spec.Replicas) requests, err = r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) require.Len(t, requests, 4) @@ -2845,9 +3203,9 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { require.Equal(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, observedPCS.Spec.Template.PodCliqueScalingGroupConfigs) t.Log("External scale-out publishes only the new ordinal and preserves the explicit workload") - group := groups[secondPlan.LPXScalingGroup] - group.Spec.Replicas = 4 - require.NoError(t, r.Update(t.Context(), group)) + pcsg := pcsgs[secondPlan.LPXScalingGroup] + pcsg.Spec.Replicas = 4 + require.NoError(t, r.Update(t.Context(), pcsg)) _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) requests, err = r.getPipelineRequests(t.Context(), pcs) @@ -2906,8 +3264,7 @@ func TestIndependentLPXDeadlineCleanup(t *testing.T) { requests, err := r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) require.Len(t, requests, 5) - for i := range requests { - request := &requests[i] + for _, request := range requests { target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef phase := lpxv1alpha1.RequestPhaseBound if target.Name == firstPlan.LPXScalingGroup && target.ReplicaIndex == tc.firstExpired || !tc.secondHealthy && target.Name == secondPlan.LPXScalingGroup && target.ReplicaIndex == 1 { @@ -2928,10 +3285,10 @@ func TestIndependentLPXDeadlineCleanup(t *testing.T) { t.Log("Apply each workload's suffix rule and capacity ownership independently") _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) - groups, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) + pcsgs, err := getPodCliqueScalingGroups(t.Context(), r.Client, pcs) require.NoError(t, err) - require.Equal(t, tc.wantFirst, groups[firstPlan.LPXScalingGroup].Spec.Replicas) - require.Equal(t, tc.wantSecond, groups[secondPlan.LPXScalingGroup].Spec.Replicas) + require.Equal(t, tc.wantFirst, pcsgs[firstPlan.LPXScalingGroup].Spec.Replicas) + require.Equal(t, tc.wantSecond, pcsgs[secondPlan.LPXScalingGroup].Spec.Replicas) requests, err = r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) require.Len(t, requests, tc.wantRequests) diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go index b1f65b914c16..eb5ff4f94591 100644 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ b/deploy/operator/internal/controller/lpx/eviction_test.go @@ -57,7 +57,7 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl Name: lpuEvictionTestPCS, Namespace: lpuEvictionTestNamespace, UID: "test-pcs-uid", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, }} - group := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ Name: lpuEvictionTestPCSG, Namespace: lpuEvictionTestNamespace, UID: "test-pcsg-uid", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }} @@ -69,7 +69,7 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl dynamolpx.WorkloadDigestAnnotation: "test-digest", lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, }, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(group, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, }} config := &corev1.ConfigMap{ @@ -86,7 +86,7 @@ func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, cl cb := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&corev1.Pod{}). - WithObjects(deployment, pcs, group, clique, config) + WithObjects(deployment, pcs, pcsg, clique, config) for _, o := range objs { cb = cb.WithObjects(o) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index 0371651f6b75..c287c2c95e01 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -4,13 +4,11 @@ package lpx import ( - "cmp" "context" "crypto/sha256" "encoding/binary" "fmt" "hash" - "slices" "strconv" "strings" @@ -32,16 +30,19 @@ const ( pipelineRequestModelAnnotation = "scheduling.lpu.nvidia.com/dynamo-model" ) -// getPipelineRequests returns only the non-nil PCS's requests, in stable order. +// getPipelineRequests returns only the non-nil PCS's requests, indexed by name. // The UID index establishes ownership; callers do not recheck it. The supported // foreground replacement path needs no discovery of LPRs after their PCS is gone. -func (r *graphReconciler) getPipelineRequests(ctx context.Context, pcs *grovev1alpha1.PodCliqueSet) ([]lpxv1alpha1.LPUPipelineRequest, error) { - requests := &lpxv1alpha1.LPUPipelineRequestList{} - if err := r.List(ctx, requests, client.InNamespace(pcs.Namespace), client.MatchingFields{pipelineRequestPCSOwnerUIDIndex: string(pcs.UID)}); err != nil { +func (r *graphReconciler) getPipelineRequests(ctx context.Context, pcs *grovev1alpha1.PodCliqueSet) (map[string]*lpxv1alpha1.LPUPipelineRequest, error) { + requestList := &lpxv1alpha1.LPUPipelineRequestList{} + if err := r.List(ctx, requestList, client.InNamespace(pcs.Namespace), client.MatchingFields{pipelineRequestPCSOwnerUIDIndex: string(pcs.UID)}); err != nil { return nil, err } - slices.SortFunc(requests.Items, func(a, b lpxv1alpha1.LPUPipelineRequest) int { return cmp.Compare(a.Name, b.Name) }) - return requests.Items, nil + requests := make(map[string]*lpxv1alpha1.LPUPipelineRequest) + for _, request := range requestList.Items { + requests[request.Name] = &request + } + return requests, nil } // resolvePipelineRequests combines desired intent with matching observed LPRs and @@ -193,12 +194,11 @@ func (r *graphReconciler) reconcilePipelineRequests( // pipelineRequestsPendingDeletion selects removed requests and current terminating // names that must disappear before publication. All requests belong to the observed PCS. func pipelineRequestsPendingDeletion( - requests []lpxv1alpha1.LPUPipelineRequest, + requests map[string]*lpxv1alpha1.LPUPipelineRequest, desiredRequests map[string]*lpxv1alpha1.LPUPipelineRequest, ) []*lpxv1alpha1.LPUPipelineRequest { var pending []*lpxv1alpha1.LPUPipelineRequest - for index := range requests { - request := &requests[index] + for _, request := range requests { if _, desired := desiredRequests[request.Name]; !desired || !request.DeletionTimestamp.IsZero() { pending = append(pending, request) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 92bbad5b4953..b5f66a9105ca 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -127,9 +127,9 @@ func requeueForPipelineRequestDeadline( func (r *graphReconciler) reconcileSchedulingFailure( ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, - groups map[string]*grovev1alpha1.PodCliqueScalingGroup, + pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, explicitReplicas map[string]*int32, - requests []lpxv1alpha1.LPUPipelineRequest, + requests []*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, ) (ctrl.Result, error) { // Persist evidence covering every expired cycle before deleting any expired request. @@ -145,7 +145,7 @@ func (r *graphReconciler) reconcileSchedulingFailure( name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name expiredByGroup[name] = append(expiredByGroup[name], request) } - requestsByGroup := make(map[string][]lpxv1alpha1.LPUPipelineRequest) + requestsByGroup := make(map[string][]*lpxv1alpha1.LPUPipelineRequest) for _, request := range requests { name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name requestsByGroup[name] = append(requestsByGroup[name], request) @@ -153,7 +153,7 @@ func (r *graphReconciler) reconcileSchedulingFailure( // An interior failure in one workload does not prevent another workload's suffix cleanup. for _, name := range slices.Sorted(maps.Keys(expiredByGroup)) { - if err := r.reconcileExpiredPipelineRequests(ctx, groups[name], requestsByGroup[name], expiredByGroup[name], explicitReplicas[name] != nil); err != nil { + if err := r.reconcileExpiredPipelineRequests(ctx, pcsgs[name], requestsByGroup[name], expiredByGroup[name], explicitReplicas[name] != nil); err != nil { return ctrl.Result{}, err } } @@ -167,7 +167,7 @@ func (r *graphReconciler) reconcileSchedulingFailure( func (r *graphReconciler) reconcileExpiredPipelineRequests( ctx context.Context, pcsg *grovev1alpha1.PodCliqueScalingGroup, - requests []lpxv1alpha1.LPUPipelineRequest, + requests []*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, manageReplicas bool, ) error { @@ -179,7 +179,7 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( // Persist the lower PCSG count before asynchronous LPR/pod cleanup starts. if manageReplicas { - if err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, replicas); err != nil { + if _, err := scaleDownPodCliqueScalingGroup(ctx, r, pcsg, replicas); err != nil { return err } } @@ -194,7 +194,7 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( // scale-down so a healthy higher ordinal is never removed. // For four replicas, expiry at {2,3} removes that suffix; {0,3} removes nothing. func expiredPipelineRequestSuffix( - requests []lpxv1alpha1.LPUPipelineRequest, + requests []*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, replicas int32, ) (int32, []*lpxv1alpha1.LPUPipelineRequest, error) { @@ -239,8 +239,7 @@ func expiredPipelineRequestSuffix( } siblings := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) - for index := range requests { - request := &requests[index] + for _, request := range requests { name, replica, err := pipelineRequestScalingGroupTarget(request) if err != nil { return 0, nil, err diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index b62f589965a6..2f5796789063 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "maps" "slices" "testing" "time" @@ -194,12 +195,12 @@ func TestReconcileExpiredPipelineRequests(t *testing.T) { dgd := loadTestDGD(t, lpx.PipelineSingle, "build-v2") deployment := newLPXTestDeployment(t, dgd) pcs := newTestPodCliqueSet(deployment) - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{Name: "engines", Namespace: deployment.Namespace}, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: tc.replicas}, } - requests := make([]lpxv1alpha1.LPUPipelineRequest, tc.replicas) - objects := []client.Object{pcs, group} + requests := make([]*lpxv1alpha1.LPUPipelineRequest, tc.replicas) + objects := []client.Object{pcs, pcsg} var expired []*lpxv1alpha1.LPUPipelineRequest for i := range requests { request := newTestPipelineRequest(deployment, pcs, fmt.Sprintf("engine-%d", i), time.Now().Add(-time.Hour), lpxv1alpha1.RequestPhaseBound) @@ -207,17 +208,17 @@ func TestReconcileExpiredPipelineRequests(t *testing.T) { request.Status.Phase = lpxv1alpha1.RequestPhasePending request.Status.Committed = nil } - request.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: group.Name, ReplicaIndex: int64(i)} - requests[i] = *request + request.Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: pcsg.Name, ReplicaIndex: int64(i)} + requests[i] = request objects = append(objects, request) if slices.Contains(tc.expired, i) { - expired = append(expired, &requests[i]) + expired = append(expired, request) } } r := newLPXTestReconciler(t, nil, deployment, dgd, objects...) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - for i := range requests { - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(&requests[i]), &requests[i])) + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcsg), pcsg)) + for _, request := range requests { + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(request), request)) } t.Log("Apply suffix cleanup without deleting PodCliques or changing healthy requests") @@ -226,23 +227,24 @@ func TestReconcileExpiredPipelineRequests(t *testing.T) { _, ok := object.(*lpxv1alpha1.LPUPipelineRequest) require.True(t, ok, "cleanup must delete only pipeline requests") live := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(group), live)) + require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(pcsg), live)) require.Equal(t, tc.wantReplicas, live.Spec.Replicas, "scale must precede deletion") return delegated.Delete(ctx, object, opts...) }, }) - err := r.reconcileExpiredPipelineRequests(t.Context(), group, requests, expired, tc.manageReplicas) + err := r.reconcileExpiredPipelineRequests(t.Context(), pcsg, requests, expired, tc.manageReplicas) require.NoError(t, err) - require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(group), group)) - require.Equal(t, tc.wantReplicas, group.Spec.Replicas) - for i := range requests { + require.Equal(t, tc.replicas, pcsg.Spec.Replicas, "scale writes must preserve the observation") + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcsg), pcsg)) + require.Equal(t, tc.wantReplicas, pcsg.Spec.Replicas) + for i, request := range requests { got := &lpxv1alpha1.LPUPipelineRequest{} - err := r.Get(t.Context(), client.ObjectKeyFromObject(&requests[i]), got) + err := r.Get(t.Context(), client.ObjectKeyFromObject(request), got) if slices.Contains(tc.removed, i) { require.True(t, apierrors.IsNotFound(err)) } else { require.NoError(t, err) - require.Equal(t, requests[i], *got) + require.Equal(t, request, got) } } }) @@ -266,13 +268,13 @@ func TestExpiredPipelineRequestSuffix(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { t.Log("Derive the complete trailing request set while preserving expiry evidence") - requests := make([]lpxv1alpha1.LPUPipelineRequest, len(tc.ordinals)) + requests := make([]*lpxv1alpha1.LPUPipelineRequest, len(tc.ordinals)) var expired []*lpxv1alpha1.LPUPipelineRequest for i, ordinal := range tc.ordinals { - requests[i].Name = fmt.Sprintf("request-%d", i) + requests[i] = &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("request-%d", i)}} requests[i].Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: "engines", ReplicaIndex: ordinal} if slices.Contains(tc.expired, i) { - expired = append(expired, &requests[i]) + expired = append(expired, requests[i]) } } replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, tc.replicas) @@ -338,15 +340,15 @@ func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { pcs := findLPXTestPodCliqueSet(t, objects) requests, err := r.getPipelineRequests(ctx, pcs) require.NoError(t, err) - groups, err := getPodCliqueScalingGroups(ctx, r.Client, pcs) - group := groups[desired.plan.LPXScalingGroup] + pcsgs, err := getPodCliqueScalingGroups(ctx, r.Client, pcs) + pcsg := pcsgs[desired.plan.LPXScalingGroup] require.NoError(t, err) require.Greater(t, len(requests), 1) expired := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, desired.requests[0].Name) var sibling *lpxv1alpha1.LPUPipelineRequest - for index := range requests { - if requests[index].Name != expired.Name { - sibling = &requests[index] + for _, request := range requests { + if request.Name != expired.Name { + sibling = request break } } @@ -363,7 +365,7 @@ func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { return delegated.Delete(ctx, object, opts...) }, }) - err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + err = r.reconcileExpiredPipelineRequests(ctx, pcsg, slices.Collect(maps.Values(requests)), []*lpxv1alpha1.LPUPipelineRequest{expired}, true) require.ErrorIs(t, err, deleteErr) require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(expired), &lpxv1alpha1.LPUPipelineRequest{})) @@ -372,8 +374,8 @@ func TestExpiredPipelineRequestsRetrySiblingDeletion(t *testing.T) { requests, err = r.getPipelineRequests(ctx, pcs) require.NoError(t, err) expired = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, expired.Name) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(group), group)) - err = r.reconcileExpiredPipelineRequests(ctx, group, requests, []*lpxv1alpha1.LPUPipelineRequest{expired}, true) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pcsg), pcsg)) + err = r.reconcileExpiredPipelineRequests(ctx, pcsg, slices.Collect(maps.Values(requests)), []*lpxv1alpha1.LPUPipelineRequest{expired}, true) require.NoError(t, err) for index := range desired.requests { requirePipelineRequestNotFound(t, ctx, r.Client, child.Namespace, desired.requests[index].Name) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 1fd9b8b81308..fb8b39f0d432 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -60,14 +60,14 @@ func TestGetPipelineRequests(t *testing.T) { r := &graphReconciler{Client: newLPXTestClient(t, objects...)} pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: "pcs", Namespace: tc.namespace, UID: tc.uid}} - t.Log("Return only this PCS's requests in stable name order") + t.Log("Return only this PCS's requests") requests, err := r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) var names []string for _, request := range requests { names = append(names, request.Name) } - require.Equal(t, tc.want, names) + require.ElementsMatch(t, tc.want, names) }) } } @@ -79,9 +79,6 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { dgd.Annotations[consts.KubeAnnotationLPXSchedulerBackend] = "unknown-scheduler" dgd.Annotations[consts.KubeAnnotationLPXExecutionBackend] = "unknown-execution" reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) - require.NotNil(t, desired) - require.Equal(t, "unknown-scheduler", dgd.Annotations[consts.KubeAnnotationLPXSchedulerBackend]) - require.Equal(t, "unknown-execution", dgd.Annotations[consts.KubeAnnotationLPXExecutionBackend]) require.Empty(t, desired.plan.ConductorTemplate) require.Empty(t, desired.plan.ConductorClique) require.NotEmpty(t, desired.plan.CyborgClique) @@ -90,8 +87,8 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { for _, object := range objects { require.NotEmpty(t, object.GetName()) } - group := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) - require.NotContains(t, group.Spec.CliqueNames, "") + pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, desired.plan.LPXScalingGroup) + require.NotContains(t, pcsg.Spec.CliqueNames, "") cyborg := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique) require.NotContains(t, cyborg.Spec.StartsAfter, "") require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) @@ -102,7 +99,6 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { require.Equal(t, v1alpha1.LPXReadyReasonPending, condition.Reason) request := getTestPipelineRequest(t, ctx, reconciler.Client, deployment.Namespace, desired.requests[0].Name) pcs := findLPXTestPodCliqueSet(t, objects) - require.True(t, metav1.IsControlledBy(pcs, deployment)) require.True(t, metav1.IsControlledBy(request, pcs)) require.True(t, *metav1.GetControllerOf(request).BlockOwnerDeletion) require.Empty(t, request.Finalizers) @@ -111,7 +107,7 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { } func TestPipelineRequestIdentityDigest(t *testing.T) { - t.Log("Every identity field distinguishes requests; the same tuple always has the same digest") + t.Log("Every identity field distinguishes requests") base := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "", "default", 0) firstGroup := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "first", "default", 0) secondGroup := pipelineRequestIdentityDigest("ns", "dgd", "uid-a", "second", "default", 0) @@ -137,7 +133,6 @@ func TestPipelineRequestIdentityDigest(t *testing.T) { digest := pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, "", test.model, test.replica) require.NotEmpty(t, digest) require.Equal(t, test.same, base == digest) - require.Equal(t, digest, pipelineRequestIdentityDigest(test.namespace, test.deployment, test.uid, "", test.model, test.replica)) }) } } @@ -205,7 +200,7 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( require.NoError(t, err) require.Len(t, requests, 3) - requestByModel := make(map[string]lpxv1alpha1.LPUPipelineRequest, len(requests)) + requestByModel := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(requests)) for _, request := range requests { model := request.Annotations[pipelineRequestModelAnnotation] requestByModel[model] = request @@ -310,12 +305,12 @@ func TestPipelineRequestsPendingDeletion(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { t.Log("Block publication until removed and already-terminating request names disappear") - requests := []lpxv1alpha1.LPUPipelineRequest{ - {ObjectMeta: metav1.ObjectMeta{Name: "first"}}, {ObjectMeta: metav1.ObjectMeta{Name: "second"}}, + requests := map[string]*lpxv1alpha1.LPUPipelineRequest{ + "first": {ObjectMeta: metav1.ObjectMeta{Name: "first"}}, + "second": {ObjectMeta: metav1.ObjectMeta{Name: "second"}}, } desired := make(map[string]*lpxv1alpha1.LPUPipelineRequest) - for index := range requests { - request := &requests[index] + for _, request := range requests { if slices.Contains(tc.desired, request.Name) { desired[request.Name] = request } @@ -328,7 +323,7 @@ func TestPipelineRequestsPendingDeletion(t *testing.T) { for _, request := range pipelineRequestsPendingDeletion(requests, desired) { names = append(names, request.Name) } - require.Equal(t, tc.want, names) + require.ElementsMatch(t, tc.want, names) }) } } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 5682f779a494..c6680b3a0965 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -473,16 +473,37 @@ func TestLPXRenderingMetadata(t *testing.T) { } func TestLPXReplicaChangesPreserveRenderedTemplates(t *testing.T) { - t.Log("Render a hybrid workload at its initial capacity") - child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) + t.Log("Render omitted hybrid capacity for two I/O endpoints with two clients each") + root := t.TempDir() + const buildID = "split-io" + writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, buildID, testV2GraphManifestFixture{ + topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", + partitionCount: 1, numChips: 8, devicesPerNode: 8, + compilationMode: manifestcapnpv2.CompilationMode_lpx, + nonLPUDeviceTypes: []manifestcapnpv2.DeviceType{manifestcapnpv2.DeviceType_cuda}, + ioFPGACount: 2, ioFanoutFactor: 2, + })) + registry, err := lpx.NewModelRegistry(root, nil) + require.NoError(t, err) + dgd := loadTestDGD(t, lpx.PipelineLPX, buildID) + dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = nil + child := newLPXTestDeployment(t, dgd) r := newLPXTestReconciler(t, registry, child, dgd) workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) before, beforeResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) require.NoError(t, err) + cyborgIndex := slices.IndexFunc(before.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { + return clique.Name == plans["lpx"].CyborgTemplate + }) + require.NotEqual(t, -1, cyborgIndex) + cyborg := before.Spec.Template.Cliques[cyborgIndex] + require.EqualValues(t, 4, cyborg.Spec.Replicas) + require.Equal(t, ptr.To(int32(1)), cyborg.Spec.MinAvailable) - t.Log("Changing only workload capacity preserves templates, configuration and the graph digest") + t.Log("Changing backbone and Cyborg capacity preserves templates, configuration and the graph digest") dgd.Spec.Components[0].Replicas = ptr.To(int32(12)) + dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(8)) workloads, plans, err = r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) after, afterResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) @@ -829,6 +850,19 @@ func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphM populateTestGraphBuild(t, manifest, filepath.Base(buildID)) deployment, program := newTestGraphProgram(t, manifest, fixture.compilationMode, uint32(fixture.partitionCount)*fixture.numChips/fixture.devicesPerNode, 8192) + + // Describe a complete batch when a fixture uses multiple endpoints or clients. + runtimeIO, err := deployment.RuntimeIo() + require.NoError(t, err) + if fixture.ioFPGACount != 0 { + runtimeIO.SetProtocol(1) + runtimeIO.SetIoFpgaCount(fixture.ioFPGACount) + } + if fixture.ioFanoutFactor != 0 { + runtimeIO.SetFanoutFactor(fixture.ioFanoutFactor) + } + program.SetBatchSize(runtimeIO.IoFpgaCount() * runtimeIO.FanoutFactor()) + if len(fixture.selectedPropSyncChain) != 0 { chains, err := deployment.NewSelectedPropSyncChains(1) require.NoError(t, err) @@ -888,6 +922,8 @@ type testV2GraphManifestFixture struct { selectedPropSyncChain []uint32 compilationMode manifestcapnpv2.CompilationMode nonLPUDeviceTypes []manifestcapnpv2.DeviceType + ioFPGACount uint32 + ioFanoutFactor uint32 } func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphManifestFixture) []byte { diff --git a/deploy/operator/internal/controller/lpx/util.go b/deploy/operator/internal/controller/lpx/util.go index ae55bf9ecfb7..1be5eb20d996 100644 --- a/deploy/operator/internal/controller/lpx/util.go +++ b/deploy/operator/internal/controller/lpx/util.go @@ -115,46 +115,107 @@ func getPodCliqueScalingGroups(ctx context.Context, reader client.Reader, pcs *g } // Establish each expected group's identity and lifetime once for downstream operations. - groups := make(map[string]*grovev1alpha1.PodCliqueScalingGroup, len(configs)) + pcsgs := make(map[string]*grovev1alpha1.PodCliqueScalingGroup, len(configs)) for i := range observed.Items { - group := &observed.Items[i] - if _, selected := expected[group.Name]; !selected { + pcsg := &observed.Items[i] + if _, selected := expected[pcsg.Name]; !selected { continue } - if !metav1.IsControlledBy(group, pcs) { - return nil, fmt.Errorf("PodCliqueScalingGroup %q is not controlled by PodCliqueSet %q", group.Name, pcs.Name) + if !metav1.IsControlledBy(pcsg, pcs) { + return nil, fmt.Errorf("PodCliqueScalingGroup %q is not controlled by PodCliqueSet %q", pcsg.Name, pcs.Name) } - if group.DeletionTimestamp.IsZero() { - groups[group.Name] = group + if pcsg.DeletionTimestamp.IsZero() { + pcsgs[pcsg.Name] = pcsg } } - return groups, nil + return pcsgs, nil +} + +// getPodCliques observes the cliques under the non-nil, owned PCS and pcsgs. +// pcs is non-nil. +// reader uses the cache. Ownership is validated here; +// missing or deleting cliques are omitted and wait for their existing watches. +func getPodCliques(ctx context.Context, reader client.Reader, pcs *grovev1alpha1.PodCliqueSet, pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup) (map[string]*grovev1alpha1.PodClique, error) { + observed := &grovev1alpha1.PodCliqueList{} + if err := reader.List(ctx, observed, client.InNamespace(pcs.Namespace), client.MatchingLabels{ + grovecommon.LabelPartOfKey: pcs.Name, grovecommon.LabelPodCliqueSetReplicaIndex: "0", + }); err != nil { + return nil, err + } + + // Keep every observed ordinal, including children whose group's cache entry still lags scale-out. + pclqs := make(map[string]*grovev1alpha1.PodClique, len(observed.Items)) + for i := range observed.Items { + pclq := &observed.Items[i] + pcsg := pcsgs[pclq.Labels[grovecommon.LabelPodCliqueScalingGroup]] + if pcsg == nil || !metav1.IsControlledBy(pclq, pcsg) { + return nil, fmt.Errorf("PodClique %q is not controlled by an observed LPX PodCliqueScalingGroup", pclq.Name) + } + if !pclq.DeletionTimestamp.IsZero() { + continue + } + pclqs[pclq.Name] = pclq + } + return pclqs, nil } // scaleDownPodCliqueScalingGroup lowers capacity before asynchronous request cleanup. // pcsg must be non-nil, owned and not deleting. Call only for explicitly managed // capacity. Scale-out must wait until terminating request names disappear. -func scaleDownPodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) error { +func scaleDownPodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) (bool, error) { if replicas >= pcsg.Spec.Replicas { - return nil + return false, nil } return scalePodCliqueScalingGroup(ctx, cl, pcsg, replicas) } // scalePodCliqueScalingGroup updates the non-nil, already-owned PCSG using its -// observed resource version. Call only for explicitly managed capacity. -// A successful write updates the supplied observation for the rest of this pass. -func scalePodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) error { +// observed resource version. It leaves the observation unchanged; a successful +// scale write requires another reconciliation before using the new capacity. +func scalePodCliqueScalingGroup(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, replicas int32) (bool, error) { if pcsg.Spec.Replicas == replicas { - return nil + return false, nil } - scale := &autoscalingv1.Scale{ - ObjectMeta: metav1.ObjectMeta{ResourceVersion: pcsg.ResourceVersion}, - Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, + + if err := updateScale(ctx, cl, pcsg, replicas); err != nil { + return false, fmt.Errorf("scale LPX PodCliqueScalingGroup %q: %w", pcsg.Name, err) } - if err := cl.SubResource("scale").Update(ctx, pcsg, client.WithSubResourceBody(scale)); err != nil { - return fmt.Errorf("scale LPX PodCliqueScalingGroup %q: %w", pcsg.Name, err) + + return true, nil +} + +// scalePodCliques scales the named template's clique in every replica of the +// non-nil pcsg. Call only for explicitly managed capacity. +// The provided PCLQs remain unchanged; the returned bool reports whether they were updated. +func scalePodCliques(ctx context.Context, cl client.Client, pcsg *grovev1alpha1.PodCliqueScalingGroup, pclqs map[string]*grovev1alpha1.PodClique, templateName string, replicas int32) (bool, error) { + var changed bool + + for index := range pcsg.Spec.Replicas { + name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(index)}, templateName) + pclq := pclqs[name] + if pclq == nil || pclq.Spec.Replicas == replicas { + continue + } + + if err := updateScale(ctx, cl, pclq, replicas); err != nil { + return changed, fmt.Errorf("scale PodClique %q: %w", name, err) + } + + changed = true } - pcsg.Spec.Replicas = replicas - return nil + + return changed, nil +} + +// updateScale writes only capacity using the non-nil resource's observed version. +// The scale response does not replace or mutate the complete resource observation. +func updateScale(ctx context.Context, cl client.Client, resource client.Object, replicas int32) error { + scale := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: resource.GetName(), Namespace: resource.GetNamespace(), ResourceVersion: resource.GetResourceVersion(), + }, + Spec: autoscalingv1.ScaleSpec{Replicas: replicas}, + } + + return cl.SubResource("scale").Update(ctx, resource, client.WithSubResourceBody(scale)) } diff --git a/deploy/operator/internal/controller/lpx/util_test.go b/deploy/operator/internal/controller/lpx/util_test.go index 7eddbec2fbe6..2006c40a742b 100644 --- a/deploy/operator/internal/controller/lpx/util_test.go +++ b/deploy/operator/internal/controller/lpx/util_test.go @@ -19,6 +19,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -58,7 +59,7 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { pcs.Spec.Template.PodCliqueScalingGroupConfigs[index].Name = "pending" } } - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "pcs-0-engines", Namespace: pcs.Namespace, Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name}, @@ -67,19 +68,19 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: tc.replicas}, } if tc.previousPCS { - group.OwnerReferences[0].UID = "previous-pcs" + pcsg.OwnerReferences[0].UID = "previous-pcs" } if tc.deleting { - group.Finalizers = []string{"example.com/cleanup"} + pcsg.Finalizers = []string{"example.com/cleanup"} now := metav1.Now() - group.DeletionTimestamp = &now + pcsg.DeletionTimestamp = &now } var objects []client.Object if !tc.missing { - objects = append(objects, group) + objects = append(objects, pcsg) } if tc.secondGroup { - second := group.DeepCopy() + second := pcsg.DeepCopy() second.Name, second.Spec.Replicas = "pcs-0-pending", 4 objects = append(objects, second) } @@ -88,10 +89,11 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { for _, key := range []client.ObjectKey{ {Namespace: pcs.Namespace, Name: "unrelated"}, {Namespace: pcs.Namespace, Name: "pcs-1-engines"}, - {Namespace: "other", Name: group.Name}, + {Namespace: "other", Name: pcsg.Name}, } { objects = append(objects, &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, + Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name}, }}) } lists := 0 @@ -111,13 +113,13 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { }) t.Log("Only a current non-deleting group supplies capacity, including an explicit zero") - groups, err := getPodCliqueScalingGroups(t.Context(), reader, pcs) + pcsgs, err := getPodCliqueScalingGroups(t.Context(), reader, pcs) if tc.templates == 0 { require.Zero(t, lists) } else { require.Equal(t, 1, lists) } - observed := groups[group.Name] + observed := pcsgs[pcsg.Name] if tc.wantError != "" { require.ErrorContains(t, err, tc.wantError) require.Nil(t, observed) @@ -131,7 +133,7 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { if tc.wantGroup { wantGroups++ require.NotNil(t, observed) - require.Equal(t, group.Name, observed.Name) + require.Equal(t, pcsg.Name, observed.Name) require.Equal(t, tc.replicas, observed.Spec.Replicas) require.True(t, metav1.IsControlledBy(observed, pcs)) } else { @@ -139,10 +141,84 @@ func TestGetPodCliqueScalingGroups(t *testing.T) { } if tc.secondGroup { wantGroups++ - require.NotNil(t, groups["pcs-0-pending"]) - require.EqualValues(t, 4, groups["pcs-0-pending"].Spec.Replicas) + require.NotNil(t, pcsgs["pcs-0-pending"]) + require.EqualValues(t, 4, pcsgs["pcs-0-pending"].Spec.Replicas) + } + require.Len(t, pcsgs, wantGroups) + }) + } +} + +func TestGetPodCliques(t *testing.T) { + for _, tc := range []struct { + name, wantError string + missing, deleting, foreign, listError bool + }{ + {name: "observed"}, + {name: "missing", missing: true}, + {name: "deleting", deleting: true}, + {name: "foreign", foreign: true, wantError: "is not controlled by"}, + {name: "foreign deleting", foreign: true, deleting: true, wantError: "is not controlled by"}, + {name: "list failure", listError: true, wantError: "clique list unavailable"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Observe owned cliques and ignore cliques outside this PCS and namespace") + pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: "pcs", Namespace: "test"}} + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{Name: "pcs-0-workload", Namespace: pcs.Namespace, UID: "group"}} + pclq := &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{ + Name: pcsg.Name + "-0-agent", Namespace: pcs.Namespace, + Labels: map[string]string{grovecommon.LabelPartOfKey: pcs.Name, grovecommon.LabelPodCliqueScalingGroup: pcsg.Name, grovecommon.LabelPodCliqueSetReplicaIndex: "0"}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, + }, + Spec: grovev1alpha1.PodCliqueSpec{RoleName: "agent", Replicas: 4}, + } + if tc.foreign { + pclq.OwnerReferences[0].UID = "foreign" + } + if tc.deleting { + pclq.DeletionTimestamp, pclq.Finalizers = ptr.To(metav1.Now()), []string{"test"} + } + otherPCS, otherNamespace, otherPCSReplica := pclq.DeepCopy(), pclq.DeepCopy(), pclq.DeepCopy() + otherPCS.Name, otherPCS.Labels[grovecommon.LabelPartOfKey] = "unrelated", "other-pcs" + otherNamespace.Namespace = "other-namespace" + otherPCSReplica.Name = "pcs-1-workload-0-agent" + otherPCSReplica.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "1" + otherPCSReplica.Labels[grovecommon.LabelPodCliqueScalingGroup] = "pcs-1-workload" + objects := []client.Object{otherPCS, otherNamespace, otherPCSReplica} + if !tc.missing { + objects = append(objects, pclq) + } + lists := 0 + reader := interceptor.NewClient(newLPXTestClient(t, objects...), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Fatal("PodClique observation must use one scoped List") + return nil + }, + List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if tc.listError { + return errors.New("clique list unavailable") + } + return delegated.List(ctx, list, opts...) + }, + }) + + t.Log("Validate once and omit missing or deleting cliques for watch-driven progress") + pclqs, err := getPodCliques(t.Context(), reader, pcs, map[string]*grovev1alpha1.PodCliqueScalingGroup{pcsg.Name: pcsg}) + require.Equal(t, 1, lists) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + require.Nil(t, pclqs) + return + } + require.NoError(t, err) + if tc.missing || tc.deleting { + require.Empty(t, pclqs) + } else { + require.Len(t, pclqs, 1) + require.Contains(t, pclqs, pclq.Name) } - require.Len(t, groups, wantGroups) }) } } @@ -281,32 +357,97 @@ func TestScalePodCliqueScalingGroup(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { t.Log("Use an already-owned group observation without rendering a workload") - group := &grovev1alpha1.PodCliqueScalingGroup{ + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{Name: "group", Namespace: "default", ResourceVersion: "7"}, Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 9}, } + observed := pcsg.DeepCopy() updates := 0 cl := interceptor.NewClient(newLPXTestClient(t), interceptor.Funcs{ - SubResourceUpdate: func(_ context.Context, _ client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { - require.Equal(t, "scale", subresource) - require.Same(t, group, object) + SubResourceUpdate: func(_ context.Context, _ client.Client, objectSubresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "scale", objectSubresource) scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) - require.Equal(t, group.ResourceVersion, scale.ResourceVersion) + require.Equal(t, object.GetName(), scale.Name) + require.Equal(t, "7", scale.ResourceVersion) require.Equal(t, tc.replicas, scale.Spec.Replicas) updates++ - return tc.err + if tc.err != nil { + return tc.err + } + scale.SetResourceVersion("8") + scale.SetGeneration(2) + return nil }, }) - t.Log("Only explicit changes write scale; successful writes update the in-pass observation") - err := scalePodCliqueScalingGroup(t.Context(), cl, group, tc.replicas) + t.Log("Report capacity writes while leaving the observation unchanged") + changed, err := scalePodCliqueScalingGroup(t.Context(), cl, pcsg, tc.replicas) require.ErrorIs(t, err, tc.err) require.Equal(t, tc.wantUpdates, updates) - want := int32(9) - if tc.err == nil { - want = tc.replicas + require.Equal(t, tc.wantUpdates > 0 && tc.err == nil, changed) + require.Equal(t, observed, pcsg) + }) + } +} + +func TestScalePodCliques(t *testing.T) { + conflict := errors.New("scale conflict") + for _, tc := range []struct { + name string + replicas int32 + failOnWrite int + wantWrites int + wantApplied int + }{ + {name: "converged", replicas: 1}, + {name: "scaled", replicas: 3, wantWrites: 2, wantApplied: 2}, + {name: "scale failure", replicas: 3, failOnWrite: 1, wantWrites: 1}, + {name: "partial scale failure", replicas: 3, failOnWrite: 2, wantWrites: 2, wantApplied: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Supply already-owned cliques under a two-replica group") + pcsg := &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "workload", Namespace: "test"}, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 2}, + } + pclqs := make(map[string]*grovev1alpha1.PodClique) + want := make(map[string]*grovev1alpha1.PodClique) + for _, name := range []string{"workload-0-worker", "workload-1-worker"} { + pclq := &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: pcsg.Namespace, ResourceVersion: "7", Generation: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + } + pclqs[name] = pclq + want[name] = pclq.DeepCopy() + } + writes := 0 + cl := interceptor.NewClient(newLPXTestClient(t), interceptor.Funcs{ + SubResourceUpdate: func(_ context.Context, _ client.Client, subresource string, object client.Object, opts ...client.SubResourceUpdateOption) error { + require.Equal(t, "scale", subresource) + scale := (&client.SubResourceUpdateOptions{}).ApplyOptions(opts).SubResourceBody.(*autoscalingv1.Scale) + require.Equal(t, object.GetName(), scale.Name) + require.Equal(t, "7", scale.ResourceVersion) + require.Equal(t, int32(3), scale.Spec.Replicas) + writes++ + if writes == tc.failOnWrite { + return conflict + } + scale.SetResourceVersion("8") + scale.SetGeneration(2) + return nil + }, + }) + + t.Log("Report successful writes, including partial progress, without mutating observations") + changed, err := scalePodCliques(t.Context(), cl, pcsg, pclqs, "worker", tc.replicas) + if tc.failOnWrite > 0 { + require.ErrorIs(t, err, conflict) + } else { + require.NoError(t, err) } - require.Equal(t, want, group.Spec.Replicas) + require.Equal(t, tc.wantWrites, writes) + require.Equal(t, tc.wantApplied > 0, changed) + require.Equal(t, want, pclqs) }) } } diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 6e59c7ddca2f..d0f1aa9344f4 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -157,8 +157,8 @@ func isLPXPodClique(obj client.Object) bool { } func isLPXPodCliqueScalingGroup(obj client.Object) bool { - group := obj.(*grovev1alpha1.PodCliqueScalingGroup) - return group.Annotations[lpx.WorkloadDigestAnnotation] != "" + pcsg := obj.(*grovev1alpha1.PodCliqueScalingGroup) + return pcsg.Annotations[lpx.WorkloadDigestAnnotation] != "" } func podCliquePredicate() predicate.Funcs { @@ -188,17 +188,17 @@ func podCliqueScalingGroupPredicate() predicate.Funcs { CreateFunc: func(e event.CreateEvent) bool { return isLPXPodCliqueScalingGroup(e.Object) }, DeleteFunc: func(e event.DeleteEvent) bool { return isLPXPodCliqueScalingGroup(e.Object) }, UpdateFunc: func(e event.UpdateEvent) bool { - oldGroup := e.ObjectOld.(*grovev1alpha1.PodCliqueScalingGroup) - newGroup := e.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) - if !isLPXPodCliqueScalingGroup(oldGroup) && !isLPXPodCliqueScalingGroup(newGroup) { + oldPCSG := e.ObjectOld.(*grovev1alpha1.PodCliqueScalingGroup) + newPCSG := e.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) + if !isLPXPodCliqueScalingGroup(oldPCSG) && !isLPXPodCliqueScalingGroup(newPCSG) { return false } - return commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldGroup, newGroup) || - oldGroup.Generation != newGroup.Generation || - !maps.Equal(oldGroup.Annotations, newGroup.Annotations) || - !maps.Equal(oldGroup.Labels, newGroup.Labels) || - !apiequality.Semantic.DeepEqual(oldGroup.OwnerReferences, newGroup.OwnerReferences) || - !apiequality.Semantic.DeepEqual(oldGroup.DeletionTimestamp, newGroup.DeletionTimestamp) + return commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldPCSG, newPCSG) || + oldPCSG.Generation != newPCSG.Generation || + !maps.Equal(oldPCSG.Annotations, newPCSG.Annotations) || + !maps.Equal(oldPCSG.Labels, newPCSG.Labels) || + !apiequality.Semantic.DeepEqual(oldPCSG.OwnerReferences, newPCSG.OwnerReferences) || + !apiequality.Semantic.DeepEqual(oldPCSG.DeletionTimestamp, newPCSG.DeletionTimestamp) }, GenericFunc: func(event.GenericEvent) bool { return false }, } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index dabfb73ed4f0..b02af4de1a28 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -298,8 +298,12 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn continue } - // Hybrid conductors use their authored GPU template and replica count. - template, replicas := conductor.PodTemplate, ptr.Deref(conductor.Replicas, 1) + // Omitted hybrid capacity starts with one complete compiled client group. + minimumReplicas, err := workload.MinimumCyborgReplicas() + if err != nil { + return nil, err + } + template, replicas := conductor.PodTemplate, ptr.Deref(conductor.Replicas, minimumReplicas) role := lpxRoleComponent(component, template, p.dynamoDeployment, p.discoveryBackend) role.ComponentType = v1beta1.ComponentTypeDecode role.Replicas = ptr.To(replicas) @@ -328,8 +332,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn return nil, fmt.Errorf("rendering %s.conductor: %w", component.ComponentName, err) } - // Every GPU worker is required by the same compiled workload replica. - clique.Spec.MinAvailable = ptr.To(clique.Spec.Replicas) clique.Labels[commonconsts.KubeLabelDynamoComponentType] = string(v1beta1.ComponentTypeLPX) clique.Labels[dynamolpx.StageLabel] = component.ComponentName input.Cyborg = clique diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 9636c0fc1116..dc7a9628f5a8 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -13,20 +13,18 @@ import ( grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) -// EvaluateLPXGroveReadiness observes every role and replica of one component group. +// EvaluateLPXGroveReadiness evaluates every observed role and replica of one component group. // source is non-nil and admitted; groupName and componentNames identify one entry -// from ComponentGroups. pcs and group may be nil while materializing; otherwise -// the caller has verified ownership and deletion state. reader may be nil until -// both pcs and group are available. Inputs remain read-only. -func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source *v1beta1.DynamoGraphDeployment, groupName string, componentNames []string, pcs *grovev1alpha1.PodCliqueSet, pcsg *grovev1alpha1.PodCliqueScalingGroup) (GroveReadiness, error) { +// from ComponentGroups. pcs and pcsg may be nil while materializing. The caller +// verifies ownership and deletion state before passing pclqs. +// Missing cliques are pending. Inputs remain read-only. +func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphDeployment, groupName string, componentNames []string, pcs *grovev1alpha1.PodCliqueSet, pcsg *grovev1alpha1.PodCliqueScalingGroup, pclqs map[string]*grovev1alpha1.PodClique) GroveReadiness { component := source.GetComponentByName(groupName) status := v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, RuntimeNamespace: source.GetDynamoNamespaceForComponent(component)} // Draft instances are counted from their own complete Agent cliques. @@ -56,25 +54,25 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source return result(false, v1beta1.DGDReadyReasonSomeResourcesNotReady, message) } if pcs == nil { - return pending("Waiting for the exact LPX PodCliqueSet"), nil + return pending("Waiting for the exact LPX PodCliqueSet") } hash := getAcceptedPCSRevisionHash(pcs) if hash == nil { - return pending("Waiting for Grove to accept the LPX PodCliqueSet revision"), nil + return pending("Waiting for Grove to accept the LPX PodCliqueSet revision") } if pcsg == nil { - return pending("Waiting for the LPX scaling group"), nil + return pending("Waiting for the LPX scaling group") } status.ComponentNames = []string{pcsg.Name} status.Replicas, status.UpdatedReplicas = pcsg.Status.Replicas, pcsg.Status.UpdatedReplicas status.AvailableReplicas = ptr.To(pcsg.Status.AvailableReplicas) if pcsg.Status.ObservedGeneration == nil || *pcsg.Status.ObservedGeneration != pcsg.Generation { - return pending("Waiting for the exact observed LPX scaling group"), nil + return pending("Waiting for the exact observed LPX scaling group") } status.ScheduledReplicas = ptr.To(pcsg.Status.ScheduledReplicas) replicas := ptr.Deref(component.Replicas, pcsg.Spec.Replicas) if pcsg.Spec.Replicas != replicas || pcsg.Status.CurrentPodCliqueSetGenerationHash == nil || *pcsg.Status.CurrentPodCliqueSetGenerationHash != *hash { - return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity"), nil + return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity") } // Observe every member before returning so partial draft readiness remains visible. unreadyClassification, unreadyMessage := "", "" @@ -83,7 +81,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source unreadyClassification, unreadyMessage = classification, message } } - for replica := int32(0); replica < replicas; replica++ { + for replica := range replicas { replicaReady := true for _, template := range pcs.Spec.Template.Cliques { if !slices.Contains(pcsg.Spec.CliqueNames, template.Name) { @@ -91,9 +89,17 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source } name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(replica)}, template.Name) memberName := template.Labels[lpx.StageLabel] - readiness, err := observeLPXRole(ctx, reader, pcsg, name, template.Spec.Replicas) - if err != nil { - return GroveReadiness{}, err + + // Revision checks precede the pure per-clique readiness calculation. + pclq := pclqs[name] + readiness := groveComponentReadiness{} + switch { + case pclq == nil: + readiness = readiness.withResult(false, fmt.Sprintf("Waiting for LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady) + case pclq.Status.CurrentPodCliqueSetGenerationHash == nil || *pclq.Status.CurrentPodCliqueSetGenerationHash != *hash: + readiness = readiness.withResult(false, fmt.Sprintf("LPX role %s has not applied the desired revision", name), v1beta1.DGDReadyReasonUpdating) + default: + readiness = observeLPXRole(ctx, pclq) } // Sum complete model instances, never physical Agent Pod counts. if draft, isDraft := statuses[memberName]; isDraft { @@ -116,56 +122,43 @@ func EvaluateLPXGroveReadiness(ctx context.Context, reader client.Reader, source } } if unreadyMessage != "" { - return result(false, unreadyClassification, unreadyMessage), nil + return result(false, unreadyClassification, unreadyMessage) } ready, message, classification := pcsgStatusReady(pcsg, replicas) if ready { classification = v1beta1.DGDReadyReasonAllResourcesReady } - return result(ready, classification, message), nil + return result(ready, classification, message) } -// observeLPXRole fences one clique by ownership and revision, then reports its -// counts in complete model instances. The group must already be observed at the -// accepted PCS revision; all pointer inputs are non-nil. -func observeLPXRole(ctx context.Context, reader client.Reader, group *grovev1alpha1.PodCliqueScalingGroup, name string, width int32) (groveComponentReadiness, error) { +// observeLPXRole calculates readiness and complete-instance counts from a non-nil +// PodClique at the accepted PCS revision. Capacity reconciliation has already +// applied explicit counts; omitted counts retain the observed external capacity. +func observeLPXRole(ctx context.Context, pclq *grovev1alpha1.PodClique) groveComponentReadiness { role := groveComponentReadiness{} - clique := &grovev1alpha1.PodClique{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: group.Namespace, Name: name}, clique); err != nil { - if apierrors.IsNotFound(err) { - return role.withResult(false, fmt.Sprintf("Waiting for LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - return role, err - } - if !metav1.IsControlledBy(clique, group) || !clique.DeletionTimestamp.IsZero() { - return role.withResult(false, fmt.Sprintf("Waiting for exact LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - if clique.Spec.Replicas != width || clique.Status.CurrentPodCliqueSetGenerationHash == nil || - *clique.Status.CurrentPodCliqueSetGenerationHash != *group.Status.CurrentPodCliqueSetGenerationHash { - return role.withResult(false, fmt.Sprintf("LPX role %s has not applied the desired revision and capacity", name), v1beta1.DGDReadyReasonUpdating), nil - } + replicas := pclq.Spec.Replicas // A model instance is counted only after the complete build width is observed. - readiness := podCliqueReadiness(clique, log.FromContext(ctx)) + readiness := podCliqueReadiness(pclq, log.FromContext(ctx)) role = role.withResult(readiness.ready, readiness.reason, readiness.classification) - if clique.Status.ObservedGeneration == nil || *clique.Status.ObservedGeneration != clique.Generation { - return role, nil + if pclq.Status.ObservedGeneration == nil || *pclq.Status.ObservedGeneration != pclq.Generation { + return role } - if clique.Status.Replicas >= width { + if pclq.Status.Replicas >= replicas { role.status.Replicas = 1 } - if clique.Status.UpdatedReplicas >= width { + if pclq.Status.UpdatedReplicas >= replicas { role.status.UpdatedReplicas = 1 } scheduled, ready := int32(0), int32(0) - if clique.Status.ScheduledReplicas >= width || readiness.ready { + if pclq.Status.ScheduledReplicas >= replicas || readiness.ready { scheduled = 1 } if readiness.ready { ready = 1 } role.status.ScheduledReplicas, role.status.ReadyReplicas = &scheduled, &ready - return role, nil + return role } func pcsgStatusReady(pcsg *grovev1alpha1.PodCliqueScalingGroup, desiredReplicas int32) (bool, string, string) { diff --git a/deploy/operator/internal/dynamo/grove_lpx_test.go b/deploy/operator/internal/dynamo/grove_lpx_test.go index 4be72e576b35..ff8f67944996 100644 --- a/deploy/operator/internal/dynamo/grove_lpx_test.go +++ b/deploy/operator/internal/dynamo/grove_lpx_test.go @@ -45,11 +45,9 @@ func TestEvaluateLPXGroveReadinessUsesEachComponentRuntimeNamespace(t *testing.T }, }, } { - readiness, err := EvaluateLPXGroveReadiness(t.Context(), nil, source, "serving", []string{"draft", "serving"}, pcs, nil) - require.NoError(t, err) + readiness := EvaluateLPXGroveReadiness(t.Context(), source, "serving", []string{"draft", "serving"}, pcs, nil, nil) require.False(t, readiness.Ready) - require.NotEqual(t, source.GetDynamoNamespaceForComponent(&draft), source.GetDynamoNamespaceForComponent(&serving)) - require.Equal(t, source.GetDynamoNamespaceForComponent(&draft), readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) - require.Equal(t, source.GetDynamoNamespaceForComponent(&serving), readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) + require.Equal(t, "dynamo", readiness.ComponentStatuses[draft.ComponentName].RuntimeNamespace) + require.Equal(t, "tenant-graph", readiness.ComponentStatuses[serving.ComponentName].RuntimeNamespace) } } diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index aa7dd9e57aba..fd1782a11171 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -7,6 +7,7 @@ package lpx import ( "fmt" + "math" "path/filepath" corev1 "k8s.io/api/core/v1" @@ -14,6 +15,22 @@ import ( const gbuildManifestPathEnv = "GBUILD_MANIFEST_PATH" +// MinimumCyborgReplicas returns one complete client group for a resolved hybrid workload. +func (w *Workload) MinimumCyborgReplicas() (int32, error) { + build := &w.modelProjections[0].configuredBuild + replicas := int64(build.IOFPGACount) * int64(build.IOFanoutFactor) + if replicas > math.MaxInt32 { + return 0, fmt.Errorf("minimum Cyborg replicas %d exceeds the PodClique replica limit %d", replicas, math.MaxInt32) + } + return int32(replicas), nil +} + +// ValidateCyborgReplicas ensures that the specified Cyborg replicas are in a valid configuration. +func (w *Workload) ValidateCyborgReplicas(replicas int32) error { + build := &w.modelProjections[0].configuredBuild + return validateCyborgReplicas(build, replicas) +} + // applyCyborgManifestPath projects an authoritative manifest location into one Cyborg container. func applyCyborgManifestPath(container *corev1.Container, projection *ModelProjection, modelStoragePath string) error { buildRoot, err := buildRuntimePath(lpuRuntimeBuildRef(projection, modelStoragePath), modelStoragePath) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index f43bec799851..3df7a3d38055 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -225,6 +225,13 @@ func RenderNodeLocal( ); err != nil { return nil, err } + + minimumReplicas, err := workload.MinimumCyborgReplicas() + if err != nil { + return nil, err + } + cyborg.Spec.Replicas = minimumReplicas + cyborg.Spec.MinAvailable = ptr.To(int32(1)) } // Each workload contributes its own scaling group to the shared PCS. From 9e1410720ed1e8ce621accb258c3194f0ee9c6f8 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 06:44:49 +0000 Subject: [PATCH 113/196] test(operator): restore PCSG readiness coverage Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/grove_test.go | 304 +++++++++++++++++- 1 file changed, 302 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index ab063b8056a6..3b2e8935e23b 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -681,6 +681,294 @@ func TestCheckPodCliqueReady(t *testing.T) { } } +func TestCheckPCSGReady(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + resourceName string + namespace string + existingPCSG *grovev1alpha1.PodCliqueScalingGroup + wantReady bool + wantReasonContains string + wantClassification string + wantServiceStatus v1beta1.ComponentReplicaStatus + }{ + { + name: "PCSG not found", + resourceName: "missing-pcsg", + namespace: "default", + wantReady: false, + wantReasonContains: "resource not found", + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"missing-pcsg"}, + }, + }, + { + name: "PCSG fully ready", + resourceName: "ready-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ready-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 3, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 3, + AvailableReplicas: 3, + UpdatedReplicas: 3, + ScheduledReplicas: 3, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + wantReady: true, + wantClassification: "", + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"ready-pcsg"}, + Replicas: 3, + UpdatedReplicas: 3, + AvailableReplicas: ptr.To(int32(3)), + ScheduledReplicas: ptr.To(int32(3)), + }, + }, + { + name: "PCSG with zero replicas desired", + resourceName: "zero-replicas-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "zero-replicas-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 0, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 0, + AvailableReplicas: 0, + UpdatedReplicas: 0, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + wantReady: true, + wantClassification: "", + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"zero-replicas-pcsg"}, + Replicas: 0, + UpdatedReplicas: 0, + AvailableReplicas: ptr.To(int32(0)), + ScheduledReplicas: ptr.To(int32(0)), + }, + }, + { + name: "PCSG spec not yet processed - observedGeneration < generation", + resourceName: "stale-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stale-pcsg", + Namespace: "default", + Generation: 3, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 2, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, + AvailableReplicas: 2, + UpdatedReplicas: 2, + ObservedGeneration: ptr.To(int64(2)), + }, + }, + wantReady: false, + wantReasonContains: "spec not yet processed", + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"stale-pcsg"}, + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: ptr.To(int32(2)), + }, + }, + { + name: "PCSG not ready - available replicas less than desired", + resourceName: "not-ready-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-ready-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 3, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 3, + AvailableReplicas: 1, + UpdatedReplicas: 3, + ScheduledReplicas: 3, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + wantReady: false, + wantReasonContains: "scheduled but available=1/3", + wantClassification: v1beta1.DGDReadyReasonPodsNotReady, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"not-ready-pcsg"}, + Replicas: 3, + UpdatedReplicas: 3, + AvailableReplicas: ptr.To(int32(1)), + ScheduledReplicas: ptr.To(int32(3)), + }, + }, + { + name: "PCSG not fully updated - updated replicas less than desired", + resourceName: "not-updated-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-updated-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 3, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 3, + AvailableReplicas: 3, + UpdatedReplicas: 2, + ScheduledReplicas: 3, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + wantReady: false, + wantReasonContains: "desired=3, updated=2", + wantClassification: v1beta1.DGDReadyReasonUpdating, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"not-updated-pcsg"}, + Replicas: 3, + UpdatedReplicas: 2, + AvailableReplicas: ptr.To(int32(3)), + ScheduledReplicas: ptr.To(int32(3)), + }, + }, + { + name: "PCSG performing rolling update - replicas != desired", + resourceName: "rolling-update-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rolling-update-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 3, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 4, + AvailableReplicas: 3, + UpdatedReplicas: 3, + ScheduledReplicas: 4, + ObservedGeneration: ptr.To(int64(1)), + }, + }, + wantReady: false, + wantReasonContains: "performing rolling update", + wantClassification: v1beta1.DGDReadyReasonUpdating, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"rolling-update-pcsg"}, + Replicas: 4, + UpdatedReplicas: 3, + AvailableReplicas: ptr.To(int32(3)), + ScheduledReplicas: ptr.To(int32(4)), + }, + }, + { + name: "PCSG with nil observedGeneration", + resourceName: "nil-observed-gen-pcsg", + namespace: "default", + existingPCSG: &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-observed-gen-pcsg", + Namespace: "default", + Generation: 1, + }, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{ + Replicas: 2, + }, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, + AvailableReplicas: 2, + UpdatedReplicas: 2, + ObservedGeneration: nil, + }, + }, + wantReady: false, + wantReasonContains: "observedGeneration is nil", + wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, + wantServiceStatus: v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, + ComponentNames: []string{"nil-observed-gen-pcsg"}, + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: ptr.To(int32(2)), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + s := scheme.Scheme + err := v1alpha1.AddToScheme(s) + g.Expect(err).NotTo(gomega.HaveOccurred()) + err = grovev1alpha1.AddToScheme(s) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + var objects []client.Object + if tt.existingPCSG != nil { + objects = append(objects, tt.existingPCSG) + } + + fakeKubeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objects...). + WithStatusSubresource(objects...). + Build() + + logger := log.FromContext(ctx) + ready, reason, serviceStatus, classification, checkErr := CheckPCSGReady(ctx, fakeKubeClient, tt.resourceName, tt.namespace, logger) + + g.Expect(checkErr).NotTo(gomega.HaveOccurred()) + g.Expect(ready).To(gomega.Equal(tt.wantReady)) + if tt.wantReasonContains != "" { + g.Expect(reason).To(gomega.ContainSubstring(tt.wantReasonContains)) + } else { + g.Expect(reason).To(gomega.Equal("")) + } + g.Expect(classification).To(gomega.Equal(tt.wantClassification)) + g.Expect(serviceStatus).To(gomega.Equal(tt.wantServiceStatus)) + }) + } +} + func TestEvaluateGroveReadinessUsesDeclaredLayout(t *testing.T) { tests := []struct { name string @@ -2239,9 +2527,11 @@ func newFakeGroveClient(g *gomega.WithT, objects ...client.Object) client.Client // TestGroveReadinessTransientErrorsPropagate verifies that a non-NotFound Get // error from a Grove child is returned as an error (so the reconcile retries and // does not advance ObservedGeneration), rather than folded into a normal -// not-ready result. NotFound remains a legitimate not-ready state. +// not-ready result. NotFound remains a legitimate not-ready state (covered by +// the "not found" cases in TestCheckPodCliqueReady / TestCheckPCSGReady). func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { ctx := context.Background() + logger := log.FromContext(ctx) transientErr := fmt.Errorf("transient API error") newClient := func(g *gomega.WithT) client.Client { @@ -2266,7 +2556,7 @@ func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { t.Run("CheckPodCliqueReady returns error on non-NotFound get failure", func(t *testing.T) { g := gomega.NewGomegaWithT(t) c := newClient(g) - ready, _, _, classification, err := CheckPodCliqueReady(ctx, c, "test-pc", "default", log.FromContext(ctx)) + ready, _, _, classification, err := CheckPodCliqueReady(ctx, c, "test-pc", "default", logger) g.Expect(err).To(gomega.HaveOccurred()) g.Expect(err.Error()).To(gomega.ContainSubstring("transient API error")) g.Expect(ready).To(gomega.BeFalse()) @@ -2274,6 +2564,16 @@ func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { g.Expect(classification).To(gomega.BeEmpty()) }) + t.Run("CheckPCSGReady returns error on non-NotFound get failure", func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + c := newClient(g) + ready, _, _, classification, err := CheckPCSGReady(ctx, c, "test-pcsg", "default", logger) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring("transient API error")) + g.Expect(ready).To(gomega.BeFalse()) + g.Expect(classification).To(gomega.BeEmpty()) + }) + t.Run("PCSG-backed component returns error on non-NotFound get failure", func(t *testing.T) { g := gomega.NewGomegaWithT(t) c := newClient(g) From 4655cec53e8557341452e31347530babcab4f8d0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 13:19:04 +0000 Subject: [PATCH 114/196] docs(operator): explain PCS owner-reference RBAC Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index c066bae4266d..e290525d8953 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -81,6 +81,8 @@ func (r *graphReconciler) GetRecorder() events.EventRecorder { // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=lpxgraphdeployments/finalizers,verbs=update +// SetControllerReference sets blockOwnerDeletion=true on PCS-owned LPUPipelineRequests, +// requiring update permission on the owner's finalizers subresource. // +kubebuilder:rbac:groups=grove.io,resources=podcliquesets/finalizers,verbs=update // +kubebuilder:rbac:groups=grove.io,resources=podcliques/scale;podcliquescalinggroups/scale,verbs=update // +kubebuilder:rbac:groups=scheduling.lpu.nvidia.com,resources=lpupipelinerequests,verbs=get;list;watch;create;delete From a165b48d76a628b77deea926e037e30c37f75129 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 14:05:55 +0000 Subject: [PATCH 115/196] refactor(operator): remove LPX eviction controller Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/AGENTS.md | 2 +- .../internal/controller/lpx/controller.go | 4 - .../internal/controller/lpx/eviction.go | 280 -------- .../internal/controller/lpx/eviction_test.go | 636 ------------------ .../internal/dynamo/lpx/lpu_configmap.go | 12 - .../internal/dynamo/lpx/lpu_partition.go | 65 -- .../internal/dynamo/lpx/lpu_partition_test.go | 120 ---- .../operator/internal/podcache/transform.go | 21 +- .../internal/podcache/transform_test.go | 24 +- 9 files changed, 7 insertions(+), 1157 deletions(-) delete mode 100644 deploy/operator/internal/controller/lpx/eviction.go delete mode 100644 deploy/operator/internal/controller/lpx/eviction_test.go delete mode 100644 deploy/operator/internal/dynamo/lpx/lpu_partition.go delete mode 100644 deploy/operator/internal/dynamo/lpx/lpu_partition_test.go diff --git a/deploy/operator/internal/controller/lpx/AGENTS.md b/deploy/operator/internal/controller/lpx/AGENTS.md index 15d92da77f41..33f3e276536e 100644 --- a/deploy/operator/internal/controller/lpx/AGENTS.md +++ b/deploy/operator/internal/controller/lpx/AGENTS.md @@ -129,7 +129,7 @@ SPDX-License-Identifier: Apache-2.0 cleanup; scheduler Pod references and release journals do not authorize it. - A terminating LPR is pending cleanup, not a reason to delete its PCS or Pods. Wait for scheduler finalization through watches, including when external - capacity is unchanged. The eviction controller has a separate disruption policy. + capacity is unchanged. # Scheduling Cycles diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index e290525d8953..2f08aa91d698 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -61,10 +61,6 @@ func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runti if err != nil { return err } - - if err := setupLPUEviction(mgr); err != nil { - return fmt.Errorf("unable to create LPU Eviction controller: %w", err) - } } if err := r.setupWithManager(mgr); err != nil { diff --git a/deploy/operator/internal/controller/lpx/eviction.go b/deploy/operator/internal/controller/lpx/eviction.go deleted file mode 100644 index 601f395fc298..000000000000 --- a/deploy/operator/internal/controller/lpx/eviction.go +++ /dev/null @@ -1,280 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - "fmt" - "slices" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" - groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/events" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" -) - -const ( - lpuEvictionEventReason = "LPUEviction" - podDisruptionReasonTaintManagerDeletion = "DeletionByTaintManager" - podDisruptionReasonEvictionAPI = "EvictionByEvictionAPI" -) - -type lpuEvictionReconciler struct { - client.Client - Recorder events.EventRecorder -} - -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;delete;deletecollection -func (r *lpuEvictionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var pod corev1.Pod - if err := r.Get(ctx, req.NamespacedName, &pod); err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - if !r.isTrigger(&pod) { - return ctrl.Result{}, nil - } - - pods, err := r.podsForTrigger(ctx, &pod) - if err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, r.deletePods(ctx, &pod, pods) -} - -// setupLPUEviction registers eviction of disrupted node-local LPX Agent cohorts. -func setupLPUEviction(mgr ctrl.Manager) error { - r := &lpuEvictionReconciler{Client: mgr.GetClient(), Recorder: mgr.GetEventRecorder("lpu-eviction")} - return ctrl.NewControllerManagedBy(mgr). - Named("lpu-eviction"). - Watches(&corev1.Pod{}, &handler.EnqueueRequestForObject{}, - builder.WithPredicates(r.evictionPredicate()), - ). - Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.evictionRequestsForConfigMap)). - Complete(r) -} - -func (r *lpuEvictionReconciler) evictionRequestsForConfigMap(ctx context.Context, object client.Object) []ctrl.Request { - // Ignore ConfigMaps outside the hybrid runtime contract before listing Pods. - config, ok := object.(*corev1.ConfigMap) - if !ok || config.Data["partition_models"] == "" { - return nil - } - - // A late runtime table observation must wake disrupted Agents using that exact table. - var pods corev1.PodList - if err := r.List(ctx, &pods, client.InNamespace(config.GetNamespace()), client.MatchingLabels{ - commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, - }); err != nil { - log.FromContext(ctx).Error(err, "list LPU eviction triggers for runtime ConfigMap") - return nil - } - - // Unrelated workloads and other immutable runtime revisions do not need another reconcile. - var requests []ctrl.Request - for i := range pods.Items { - pod := &pods.Items[i] - name, err := dynamolpx.LPUAgentConfigMapName(pod) - if r.isTrigger(pod) && err == nil && name == config.GetName() { - requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) - } - } - return requests -} - -func (r *lpuEvictionReconciler) podsForTrigger(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { - owner := metav1.GetControllerOf(trigger) - if owner == nil || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || - owner.Kind != groveconstants.KindPodClique || owner.Name == "" || owner.UID == "" { - return nil, nil - } - clique := &grovev1alpha1.PodClique{} - if err := r.Get(ctx, client.ObjectKey{Namespace: trigger.Namespace, Name: owner.Name}, clique); err != nil { - return nil, err - } - if clique.UID != owner.UID || !dynamolpx.OwnsPodClique(ctx, r.Client, clique) { - return nil, fmt.Errorf("cannot verify ownership of LPU eviction trigger pod %s/%s", trigger.Namespace, trigger.Name) - } - - mode := lpxv1alpha1.WorkloadMode(trigger.Annotations[dynamolpx.WorkloadModeAnnotation]) - if mode == "" { - return nil, nil - } - pods, err := r.allModelPods(ctx, trigger) - if err != nil { - return nil, err - } - switch mode { - case lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly: - case lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid: - // The Pod's content-addressed runtime table survives native template updates. - name, err := dynamolpx.LPUAgentConfigMapName(trigger) - if err != nil { - return nil, err - } - hash := trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] - if hash == "" { - return nil, fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap identity", trigger.Namespace, trigger.Name) - } - var config corev1.ConfigMap - key := client.ObjectKey{Namespace: trigger.Namespace, Name: name} - if err := r.Get(ctx, key, &config); err != nil { - return nil, fmt.Errorf("read LPU-GPU eviction runtime ConfigMap: %w", err) - } - if config.Immutable == nil || !*config.Immutable { - return nil, fmt.Errorf("LPU-GPU eviction runtime ConfigMap %s is not immutable", key) - } - - // Bind the table to the verified deployment and the Pod's full content hash. - configOwner := metav1.GetControllerOf(&config) - if configOwner == nil || configOwner.APIVersion != nvidiacomv1alpha1.GroupVersion.String() || - configOwner.Kind != nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind || - configOwner.Name != clique.Annotations[dynamolpx.DeploymentNameAnnotation] || - string(configOwner.UID) != clique.Annotations[dynamolpx.DeploymentUIDAnnotation] || - dynamolpx.LPUConfigMapHash(&config) != hash { - return nil, fmt.Errorf("cannot authenticate LPU-GPU eviction runtime ConfigMap %s", key) - } - triggerPartition, err := dynamolpx.LPUAgentRuntimePartition(trigger, &config) - if err != nil { - return nil, fmt.Errorf("parse LPU-GPU eviction trigger pod %s/%s: %w", trigger.Namespace, trigger.Name, err) - } - - // Select the full runtime partition before authorizing any sibling deletion. - selected := pods[:0] - for _, pod := range pods { - partition, err := dynamolpx.LPUAgentRuntimePartition(&pod, &config) - if err != nil { - return nil, fmt.Errorf("parse LPU-GPU eviction candidate pod %s/%s: %w", pod.Namespace, pod.Name, err) - } - if partition == triggerPartition { - selected = append(selected, pod) - } - } - pods = selected - default: - return nil, nil - } - return pods, nil -} - -func (r *lpuEvictionReconciler) allModelPods(ctx context.Context, trigger *corev1.Pod) ([]corev1.Pod, error) { - labels := trigger.GetLabels() - dgdName := labels[commonconsts.KubeLabelDynamoGraphDeploymentName] - component := labels[commonconsts.KubeLabelDynamoComponent] - pcsg := labels[grovecommon.LabelPodCliqueScalingGroup] - pcsgReplica := labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] - if dgdName == "" || component == "" || pcsg == "" || pcsgReplica == "" { - return nil, fmt.Errorf("LPU eviction trigger pod %s/%s missing required labels", trigger.Namespace, trigger.Name) - } - - var pods corev1.PodList - if err := r.List(ctx, &pods, client.InNamespace(trigger.Namespace), client.MatchingLabels{ - commonconsts.KubeLabelDynamoGraphDeploymentName: dgdName, - commonconsts.KubeLabelDynamoComponent: component, - grovecommon.LabelPodCliqueScalingGroup: pcsg, - grovecommon.LabelPodCliqueScalingGroupReplicaIndex: pcsgReplica, - }); err != nil { - return nil, fmt.Errorf("failed to list LPU pod candidates: %w", err) - } - - model := trigger.Annotations[lpxv1alpha1.PodModelAnnotation] - owner := metav1.GetControllerOf(trigger) - - return slices.DeleteFunc(pods.Items, func(pod corev1.Pod) bool { - // Native updates can replace the clique or individual Pods within it. - candidateOwner := metav1.GetControllerOf(&pod) - return pod.DeletionTimestamp != nil || !r.isLPUAgentPod(&pod) || - owner == nil || candidateOwner == nil || - owner.APIVersion != candidateOwner.APIVersion || owner.Kind != candidateOwner.Kind || - owner.Name != candidateOwner.Name || owner.UID != candidateOwner.UID || - pod.Labels[grovecommon.LabelPodTemplateHash] != trigger.Labels[grovecommon.LabelPodTemplateHash] || - pod.Annotations[commonconsts.AnnotationExtraResourcesHash] != trigger.Annotations[commonconsts.AnnotationExtraResourcesHash] || - model != "" && pod.Annotations[lpxv1alpha1.PodModelAnnotation] != model - }), nil -} - -func (r *lpuEvictionReconciler) deletePods(ctx context.Context, trigger *corev1.Pod, pods []corev1.Pod) error { - deleted := 0 - for i := range pods { - pod := &pods[i] - uid, resourceVersion := pod.UID, pod.ResourceVersion - if err := r.Delete(ctx, pod, client.GracePeriodSeconds(0), client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}); err != nil { - if errors.IsNotFound(err) { - continue - } - return fmt.Errorf("failed to delete LPU pod %s/%s: %w", pod.Namespace, pod.Name, err) - } - deleted++ - } - - // Repeated observations and concurrent deletions must not report another eviction. - if deleted == 0 { - return nil - } - - log.FromContext(ctx).Info("deleted LPU pod set", - "trigger", trigger.Name, - "deletedPods", deleted, - ) - r.Recorder.Eventf(trigger, nil, corev1.EventTypeWarning, lpuEvictionEventReason, "Delete", - "Pod %s is being disrupted; deleted %d LPU pods", - trigger.Name, deleted, - ) - return nil -} - -func (r *lpuEvictionReconciler) isLPUAgentPod(pod *corev1.Pod) bool { - componentType := pod.Labels[commonconsts.KubeLabelDynamoComponentType] - return componentType == commonconsts.ComponentTypeLPX && - pod.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent -} - -func (r *lpuEvictionReconciler) isTrigger(pod *corev1.Pod) bool { - if pod.Spec.SchedulerName != dynamolpx.SchedulerName || !r.isLPUAgentPod(pod) || pod.DeletionTimestamp == nil { - return false - } - return slices.ContainsFunc(pod.Status.Conditions, func(condition corev1.PodCondition) bool { - return condition.Type == corev1.DisruptionTarget && - condition.Status == corev1.ConditionTrue && - (condition.Reason == podDisruptionReasonTaintManagerDeletion || condition.Reason == podDisruptionReasonEvictionAPI) - }) -} - -func (r *lpuEvictionReconciler) evictionPredicate() predicate.Predicate { - shouldEnqueueDeletingPod := func(obj client.Object) bool { - pod, ok := obj.(*corev1.Pod) - return ok && r.isTrigger(pod) - } - - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return shouldEnqueueDeletingPod(e.Object) - }, - DeleteFunc: func(event.DeleteEvent) bool { return false }, - GenericFunc: func(event.GenericEvent) bool { return false }, - UpdateFunc: func(e event.UpdateEvent) bool { - return shouldEnqueueDeletingPod(e.ObjectNew) - }, - } -} diff --git a/deploy/operator/internal/controller/lpx/eviction_test.go b/deploy/operator/internal/controller/lpx/eviction_test.go deleted file mode 100644 index eb5ff4f94591..000000000000 --- a/deploy/operator/internal/controller/lpx/eviction_test.go +++ /dev/null @@ -1,636 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "context" - "sort" - "testing" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/events" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/event" - - commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" -) - -const ( - lpuEvictionTestNamespace = "test-ns" - lpuEvictionTestDGD = "test-dgd" - lpuEvictionTestLGD = "test-lgd" - lpuEvictionTestPCS = "materialization-12345678" - lpuEvictionTestComponent = "lpx-worker" - lpuEvictionTestPCSG = "lpx-worker" - lpuEvictionTestModel = "test-model" - lpuEvictionTestConfigHash = "bf665efcf39a0c79fa97ea2c05664ca7235dd525aced22f47da0633d729d6171" -) - -func newLPUEvictionReconciler(objs ...client.Object) (*lpuEvictionReconciler, client.Client) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = nvidiacomv1alpha1.AddToScheme(scheme) - _ = grovev1alpha1.AddToScheme(scheme) - - deployment := &nvidiacomv1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{ - Name: lpuEvictionTestLGD, Namespace: lpuEvictionTestNamespace, UID: "test-lgd-uid", - }} - pcs := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: lpuEvictionTestPCS, Namespace: lpuEvictionTestNamespace, UID: "test-pcs-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, - }} - pcsg := &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metav1.ObjectMeta{ - Name: lpuEvictionTestPCSG, Namespace: lpuEvictionTestNamespace, UID: "test-pcsg-uid", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, - }} - clique := &grovev1alpha1.PodClique{ObjectMeta: metav1.ObjectMeta{ - Name: "agents", Namespace: lpuEvictionTestNamespace, UID: "agents-uid", - Annotations: map[string]string{ - dynamolpx.DeploymentNameAnnotation: lpuEvictionTestLGD, - dynamolpx.DeploymentUIDAnnotation: "test-lgd-uid", - dynamolpx.WorkloadDigestAnnotation: "test-digest", - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - }, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup"))}, - }} - - config := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), Namespace: lpuEvictionTestNamespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(deployment, nvidiacomv1alpha1.LPXGraphDeploymentGVK)}, - }, - Immutable: ptr.To(true), - Data: map[string]string{ - "nodes_per_partition": "2\n1", "partition_node_offsets": "0\n2", - "partition_models": lpuEvictionTestModel + "\n" + lpuEvictionTestModel, - }, - } - cb := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&corev1.Pod{}). - WithObjects(deployment, pcs, pcsg, clique, config) - for _, o := range objs { - cb = cb.WithObjects(o) - } - c := cb.Build() - - return &lpuEvictionReconciler{Client: c, Recorder: events.NewFakeRecorder(16)}, c -} - -func newLPUEvictionPod(name string, replicaIdx string, podIndex string, deleting bool, conditionReason string) *corev1.Pod { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: lpuEvictionTestNamespace, - UID: types.UID(name + "-uid"), - OwnerReferences: []metav1.OwnerReference{{APIVersion: "grove.io/v1alpha1", Kind: "PodClique", Name: "agents", UID: "agents-uid", Controller: ptr.To(true)}}, - Labels: map[string]string{ - grovecommon.LabelPartOfKey: lpuEvictionTestPCS, - commonconsts.KubeLabelDynamoGraphDeploymentName: lpuEvictionTestDGD, - commonconsts.KubeLabelDynamoComponent: lpuEvictionTestComponent, - commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeLPX, - grovecommon.LabelPodCliqueScalingGroup: lpuEvictionTestPCSG, - grovecommon.LabelPodCliqueScalingGroupReplicaIndex: replicaIdx, - }, - Annotations: map[string]string{ - lpxv1alpha1.PodRoleAnnotation: lpxv1alpha1.PodRoleAgent, - dynamolpx.WorkloadModeAnnotation: string(lpxv1alpha1.WorkloadModeV2LPUOnly), - commonconsts.AnnotationExtraResourcesHash: lpuEvictionTestConfigHash, - }, - }, - Spec: corev1.PodSpec{ - SchedulerName: dynamolpx.SchedulerName, - Volumes: []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{ - Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), - }}, - }}}, - }, - Status: corev1.PodStatus{Phase: corev1.PodRunning}, - } - if podIndex != "" { - pod.Labels[grovecommon.LabelPodCliquePodIndex] = podIndex - } - pod.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel - if conditionReason != "" { - pod.Status.Conditions = []corev1.PodCondition{{ - Type: corev1.DisruptionTarget, - Status: corev1.ConditionTrue, - Reason: conditionReason, - }} - } - if deleting { - now := metav1.Now() - pod.DeletionTimestamp = &now - pod.Finalizers = []string{"test-finalizer"} - } - return pod -} - -func newLPUOnlyConductor() *corev1.Pod { - pod := newLPUEvictionPod("conductor-0", "0", "", false, "") - pod.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor - return pod -} - -func reconcileLPUEviction(t *testing.T, r *lpuEvictionReconciler, name string) ctrl.Result { - t.Helper() - - result, err := r.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: name, Namespace: lpuEvictionTestNamespace}, - }) - require.NoError(t, err) - return result -} - -func remainingLPUEvictionPods(t *testing.T, c client.Client) []string { - t.Helper() - - var remaining corev1.PodList - require.NoError(t, c.List(context.Background(), &remaining, client.InNamespace(lpuEvictionTestNamespace))) - - names := make([]string, 0, len(remaining.Items)) - for _, pod := range remaining.Items { - if pod.DeletionTimestamp != nil { - continue - } - names = append(names, pod.Name) - } - sort.Strings(names) - return names -} - -func lpuEvictionPodNames(pods []corev1.Pod) []string { - names := make([]string, 0, len(pods)) - for _, pod := range pods { - names = append(names, pod.Name) - } - sort.Strings(names) - return names -} - -func TestLPUEviction_PodsForTrigger(t *testing.T) { - t.Run("lpu-gpu pod without Grove index returns error", func(t *testing.T) { - t.Log("Observe a hybrid Agent without its canonical Grove index") - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - delete(trigger.Labels, grovecommon.LabelPodCliquePodIndex) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, _ := newLPUEvictionReconciler(trigger, sibling) - - t.Log("Reject the incomplete identity before selecting any deletion") - pods, err := r.podsForTrigger(context.Background(), trigger) - - require.Error(t, err) - assert.Nil(t, pods) - }) -} - -func TestLPUEvictionPreservesNativeReplacement(t *testing.T) { - t.Log("Keep the original snapshot while a new native clique coexists") - trigger := newLPUEvictionPod("agent-0", "0", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - replacement := newLPUEvictionPod("agent-1", "0", "1", false, "") - replacement.OwnerReferences[0].UID = "replacement-clique" - r, _ := newLPUEvictionReconciler(trigger, replacement) - pods, err := r.podsForTrigger(t.Context(), trigger) - require.NoError(t, err) - require.Equal(t, []string{"agent-0"}, lpuEvictionPodNames(pods)) -} - -func TestLPUEvictionPreservesNewPodTemplateRevision(t *testing.T) { - for _, mode := range []lpxv1alpha1.WorkloadMode{ - lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly, - lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid, - } { - t.Run(string(mode), func(t *testing.T) { - t.Log("OnDelete can retain old Pods and replace others under the same clique") - trigger := newLPUEvictionPod("agent-old-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) - sibling := newLPUEvictionPod("agent-old-1", "0", "1", false, "") - replacement := newLPUEvictionPod("agent-new-1", "0", "1", false, "") - trigger.Labels[grovecommon.LabelPodTemplateHash] = "old-image" - sibling.Labels[grovecommon.LabelPodTemplateHash] = "old-image" - replacement.Labels[grovecommon.LabelPodTemplateHash] = "new-image" - trigger.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:old"}} - sibling.Spec.Containers = trigger.Spec.Containers - replacement.Spec.Containers = []corev1.Container{{Name: "agent", Image: "agent:new"}} - require.Equal(t, trigger.OwnerReferences, replacement.OwnerReferences) - r, c := newLPUEvictionReconciler(trigger, sibling, replacement) - - t.Log("Evict the old revision's sibling without deleting the replacement") - reconcileLPUEviction(t, r, trigger.Name) - require.Equal(t, []string{replacement.Name}, remainingLPUEvictionPods(t, c)) - }) - } -} - -func TestLPUEviction_LPUOnlyDeletesSamePCSGReplica(t *testing.T) { - tests := []struct { - name string - reason string - deletedConcurrently bool - }{ - {name: "taint manager", reason: podDisruptionReasonTaintManagerDeletion}, - {name: "eviction api", reason: podDisruptionReasonEvictionAPI}, - {name: "deleted concurrently", reason: podDisruptionReasonTaintManagerDeletion, deletedConcurrently: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Log("Observe a disrupted Agent alongside its conductor and another replica") - trigger := newLPUEvictionPod("agent-0", "0", "0", true, tt.reason) - conductor := newLPUOnlyConductor() - agent := newLPUEvictionPod("agent-1", "0", "1", false, "") - otherReplica := newLPUEvictionPod("agent-other-replica", "1", "0", false, "") - - r, c := newLPUEvictionReconciler( - trigger, - conductor, - agent, - otherReplica, - ) - if tt.deletedConcurrently { - r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - // Another actor removes the sibling before this reconciler's delete reaches the server. - require.NoError(t, delegated.Delete(ctx, object, opts...)) - return delegated.Delete(ctx, object, opts...) - }, - }) - } - - t.Log("Delete only the same replica's Agent and report only our successful deletion") - result := reconcileLPUEviction(t, r, "agent-0") - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"agent-other-replica", "conductor-0"}, remainingLPUEvictionPods(t, c)) - recorded := r.Recorder.(*events.FakeRecorder).Events - if tt.deletedConcurrently { - require.Empty(t, recorded) - } else { - require.Len(t, recorded, 1) - require.Contains(t, <-recorded, "deleted 1 LPU pods") - } - - t.Log("Reconcile the retained deleting trigger without another mutation or warning") - assert.Equal(t, ctrl.Result{}, reconcileLPUEviction(t, r, "agent-0")) - require.Empty(t, recorded) - }) - } -} - -func TestLPUEviction_RejectsStaleLGDOwner(t *testing.T) { - t.Log("Observe an ownership snapshot with a stale LGD UID") - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) - - clique := &grovev1alpha1.PodClique{} - require.NoError(t, c.Get(t.Context(), client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: "agents"}, clique)) - clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "stale-lgd-uid" - require.NoError(t, c.Update(t.Context(), clique)) - - t.Log("Return a retryable error without deleting the sibling") - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) - require.ErrorContains(t, err, "cannot verify ownership") - assert.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) - - t.Log("Evict the sibling when the owner snapshot catches up") - clique.Annotations[dynamolpx.DeploymentUIDAnnotation] = "test-lgd-uid" - require.NoError(t, c.Update(t.Context(), clique)) - reconcileLPUEviction(t, r, trigger.Name) - assert.Empty(t, remainingLPUEvictionPods(t, c)) -} - -func TestLPUEviction_ResourceVersionFencesDeletion(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) - r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ - Delete: func(ctx context.Context, delegated client.WithWatch, object client.Object, opts ...client.DeleteOption) error { - current := &corev1.Pod{} - require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(object), current)) - current.OwnerReferences[0].Kind = "ForeignPodClique" - require.NoError(t, delegated.Update(ctx, current)) - return delegated.Delete(ctx, object, opts...) - }, - }) - - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) - require.Error(t, err) - require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(sibling), &corev1.Pod{})) -} - -func TestLPUEviction_LPUGPUDeletesSamePartitionOnly(t *testing.T) { - for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid} { - t.Run(string(mode), func(t *testing.T) { - t.Log("Observe live neighbors alongside a terminating Agent without scheduler partition metadata") - trigger := newLPUEvictionPod("stage-0-partition-3", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - samePartition := newLPUEvictionPod("stage-1-partition-3", "0", "1", false, "") - otherPartition := newLPUEvictionPod("stage-0-partition-4", "0", "2", false, "") - otherModel := newLPUEvictionPod("stage-0-draft-partition-3", "0", "0", false, "") - otherModel.Annotations[lpxv1alpha1.PodModelAnnotation] = lpuEvictionTestModel + "-other" - noPartition := newLPUOnlyConductor() - otherReplica := newLPUEvictionPod("replica-1-partition-3", "1", "0", false, "") - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) - terminating := newLPUEvictionPod("terminating-agent", "0", "", true, "") - - t.Log("Consume the real shared-cache Pod projection without deprecated row annotations") - for _, pod := range []*corev1.Pod{trigger, samePartition, otherPartition, otherModel, noPartition, otherReplica, terminating} { - podcache.Project(pod) - } - - r, c := newLPUEvictionReconciler( - trigger, - samePartition, - otherPartition, - otherModel, - noPartition, - otherReplica, - terminating, - ) - - t.Log("Evict the live neighbor in the trigger's partition") - result := reconcileLPUEviction(t, r, "stage-0-partition-3") - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"conductor-0", "replica-1-partition-3", "stage-0-draft-partition-3", "stage-0-partition-4"}, remainingLPUEvictionPods(t, c)) - - t.Log("Reconcile successfully after the partition's live neighbors are gone") - reconcileLPUEviction(t, r, trigger.Name) - }) - } -} - -func TestLPUEvictionHybridIncompleteRuntimeDoesNotDelete(t *testing.T) { - t.Log("Define missing runtime dependencies and malformed candidate identities") - tests := []struct { - name string - candidateIndex string - missingHash bool - missingVolume bool - missingModel bool - missingConfig bool - mutableConfig bool - missingOwner bool - foreignOwner bool - alteredData bool - }{ - {name: "missing candidate index", candidateIndex: ""}, - {name: "invalid candidate index", candidateIndex: "bad"}, - {name: "out of range candidate", candidateIndex: "3"}, - {name: "missing config identity", candidateIndex: "2", missingHash: true}, - {name: "missing runtime volume", candidateIndex: "2", missingVolume: true}, - {name: "missing model", candidateIndex: "2", missingModel: true}, - {name: "config not observed", candidateIndex: "2", missingConfig: true}, - {name: "mutable config", candidateIndex: "2", mutableConfig: true}, - {name: "unowned config", candidateIndex: "2", missingOwner: true}, - {name: "foreign config owner", candidateIndex: "2", foreignOwner: true}, - {name: "replaced config with altered ranges", candidateIndex: "2", alteredData: true}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Observe a valid sibling before another candidate whose identity cannot authorize deletion") - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV2StrictHybrid) - if test.missingVolume { - trigger.Spec.Volumes = nil - } - if test.missingHash { - delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) - } - if test.missingModel { - delete(trigger.Annotations, lpxv1alpha1.PodModelAnnotation) - } - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - candidate := newLPUEvictionPod("agent-2", "0", test.candidateIndex, false, "") - r, c := newLPUEvictionReconciler(trigger, sibling, candidate) - config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} - require.NoError(t, c.Get(t.Context(), key, config)) - require.Equal(t, lpuEvictionTestConfigHash, dynamolpx.LPUConfigMapHash(config)) - if test.missingConfig { - require.NoError(t, c.Delete(t.Context(), config)) - } - if test.mutableConfig { - config.Immutable = ptr.To(false) - require.NoError(t, c.Update(t.Context(), config)) - } - if test.missingOwner || test.foreignOwner || test.alteredData { - require.NoError(t, c.Delete(t.Context(), config)) - config.ResourceVersion = "" - config.UID = "replacement-config-uid" - if test.missingOwner { - config.OwnerReferences = nil - } - if test.foreignOwner { - config.OwnerReferences[0].UID = "foreign-lgd-uid" - } - if test.alteredData { - config.Data["nodes_per_partition"] = "3\n1" - config.Data["partition_node_offsets"] = "0\n3" - } - require.NoError(t, c.Create(t.Context(), config)) - } - - t.Log("Return an error without partially deleting the cohort") - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) - require.Error(t, err) - require.Equal(t, []string{sibling.Name, candidate.Name}, remainingLPUEvictionPods(t, c)) - }) - } -} - -func TestLPUEvictionConfigMapObservationWakesTrigger(t *testing.T) { - t.Log("Observe a hybrid disruption before its immutable runtime table reaches the cache") - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(lpxv1alpha1.WorkloadModeV3HxStrictHybrid) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) - config := &corev1.ConfigMap{} - key := client.ObjectKey{Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash)} - require.NoError(t, c.Get(t.Context(), key, config)) - require.NoError(t, c.Delete(t.Context(), config)) - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) - require.Error(t, err) - require.Equal(t, []string{sibling.Name}, remainingLPUEvictionPods(t, c)) - - t.Log("Wake only the disrupted Pod when its exact runtime table becomes visible") - config.ResourceVersion = "" - require.NoError(t, c.Create(t.Context(), config)) - require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(trigger)}}, r.evictionRequestsForConfigMap(t.Context(), config)) - unrelated := config.DeepCopy() - unrelated.Name = "different-runtime-config" - require.Empty(t, r.evictionRequestsForConfigMap(t.Context(), unrelated)) - - t.Log("Converge from the new cached state through the original deletion owner") - reconcileLPUEviction(t, r, trigger.Name) - require.Empty(t, remainingLPUEvictionPods(t, c)) -} - -func TestLPUEvictionLPUOnlyNeedsNoRuntimePartitionTable(t *testing.T) { - for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2LPUOnly, lpxv1alpha1.WorkloadModeV3HxLPUOnly} { - t.Run(string(mode), func(t *testing.T) { - t.Log("Observe an LPU-only disruption without hybrid runtime partition metadata") - trigger := newLPUEvictionPod("agent-0", "0", "", true, podDisruptionReasonTaintManagerDeletion) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) - sibling := newLPUEvictionPod("agent-1", "0", "", false, "") - delete(trigger.Annotations, commonconsts.AnnotationExtraResourcesHash) - delete(sibling.Annotations, commonconsts.AnnotationExtraResourcesHash) - r, c := newLPUEvictionReconciler(trigger, sibling) - config := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), - }} - require.NoError(t, c.Delete(t.Context(), config)) - - t.Log("Keep the existing model-cohort eviction behavior independent of partition lookup") - reconcileLPUEviction(t, r, trigger.Name) - require.Empty(t, remainingLPUEvictionPods(t, c)) - }) - } -} - -func TestLPUEviction_DisruptionConditionWithoutDeletionTimestampDoesNotCascade(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "0", false, podDisruptionReasonTaintManagerDeletion) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - - r, c := newLPUEvictionReconciler(trigger, sibling) - - result := reconcileLPUEviction(t, r, "agent-0") - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"agent-0", "agent-1"}, remainingLPUEvictionPods(t, c)) - - var updated corev1.Pod - require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(trigger), &updated)) - assert.Empty(t, updated.Finalizers) -} - -func TestLPUEviction_PodPredicateEnqueuesOnlyDeletingPods(t *testing.T) { - predicate := (&lpuEvictionReconciler{}).evictionPredicate() - ordinary := newLPUEvictionPod("agent-0", "0", "0", false, "") - deleting := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - - assert.False(t, predicate.Create(event.CreateEvent{Object: ordinary})) - assert.False(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: ordinary.DeepCopy()})) - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: ordinary, ObjectNew: deleting})) -} - -func TestLPUEviction_Noops(t *testing.T) { - tests := []struct { - name string - triggerName string - conditionReason string - phase corev1.PodPhase - statusReason string - missingMode bool - nonLPU bool - conductor bool - }{ - {name: "other disruption reason", conditionReason: "OtherDisruptionReason"}, - {name: "failed evicted without disruption condition", phase: corev1.PodFailed, statusReason: "Evicted"}, - {name: "missing workload mode", conditionReason: podDisruptionReasonTaintManagerDeletion, missingMode: true}, - {name: "non LPX pod", triggerName: "worker-0", conditionReason: podDisruptionReasonTaintManagerDeletion, nonLPU: true}, - {name: "conductor", triggerName: "conductor-0", conditionReason: podDisruptionReasonTaintManagerDeletion, conductor: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Log("Observe a deleting Pod outside the Agent cascade contract") - triggerName := tt.triggerName - if triggerName == "" { - triggerName = "agent-0" - } - trigger := newLPUEvictionPod(triggerName, "0", "0", true, tt.conditionReason) - trigger.Status.Phase = tt.phase - trigger.Status.Reason = tt.statusReason - if tt.nonLPU { - trigger.Labels[commonconsts.KubeLabelDynamoComponentType] = commonconsts.ComponentTypeWorker - } - if tt.conductor { - trigger.Annotations[lpxv1alpha1.PodRoleAnnotation] = lpxv1alpha1.PodRoleConductor - } - if tt.missingMode { - delete(trigger.Annotations, dynamolpx.WorkloadModeAnnotation) - } - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - r, c := newLPUEvictionReconciler(trigger, sibling) - - t.Log("Reconcile without deleting the surviving Agent") - result := reconcileLPUEviction(t, r, trigger.Name) - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"agent-1"}, remainingLPUEvictionPods(t, c)) - }) - } -} - -func TestLPUEviction_MissingGroveLabelsReturnsError(t *testing.T) { - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonTaintManagerDeletion) - delete(trigger.Labels, grovecommon.LabelPodCliqueScalingGroup) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - - r, c := newLPUEvictionReconciler(trigger, sibling) - - result, err := r.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: trigger.Name, Namespace: lpuEvictionTestNamespace}, - }) - require.Error(t, err) - assert.Equal(t, ctrl.Result{}, result) - assert.Equal(t, []string{"agent-1"}, remainingLPUEvictionPods(t, c)) -} - -func TestLPUEvictionUsesGroupScopedPodConfigMap(t *testing.T) { - for _, mode := range []lpxv1alpha1.WorkloadMode{lpxv1alpha1.WorkloadModeV2StrictHybrid, lpxv1alpha1.WorkloadModeV3HxStrictHybrid} { - t.Run(string(mode), func(t *testing.T) { - t.Log("Observe two workloads sharing a PCS and content hash but using distinct runtime tables") - trigger := newLPUEvictionPod("agent-0", "0", "0", true, podDisruptionReasonEvictionAPI) - trigger.Annotations[dynamolpx.WorkloadModeAnnotation] = string(mode) - sibling := newLPUEvictionPod("agent-1", "0", "1", false, "") - otherPartition := newLPUEvictionPod("agent-2", "0", "2", false, "") - otherWorkload := newLPUEvictionPod("other-agent-1", "0", "1", false, "") - otherWorkload.Labels[grovecommon.LabelPodCliqueScalingGroup] = "other-workload" - groupConfig := dynamolpx.LPUConfigMapName(lpuEvictionTestPCS+"-decode", lpuEvictionTestConfigHash) - for _, pod := range []*corev1.Pod{trigger, sibling, otherPartition} { - pod.Spec.Volumes[0].ConfigMap.Name = groupConfig - } - otherWorkload.Spec.Volumes[0].ConfigMap.Name = dynamolpx.LPUConfigMapName(lpuEvictionTestPCS+"-other", lpuEvictionTestConfigHash) - for _, pod := range []*corev1.Pod{trigger, sibling, otherPartition, otherWorkload} { - podcache.Project(pod) - } - r, kube := newLPUEvictionReconciler(trigger, sibling, otherPartition, otherWorkload) - config := &corev1.ConfigMap{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKey{ - Namespace: lpuEvictionTestNamespace, Name: dynamolpx.LPUConfigMapName(lpuEvictionTestPCS, lpuEvictionTestConfigHash), - }, config)) - - t.Log("An old unscoped table must neither wake nor authorize eviction of this workload") - require.Empty(t, r.evictionRequestsForConfigMap(t.Context(), config)) - _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trigger)}) - require.ErrorContains(t, err, groupConfig) - require.Equal(t, []string{sibling.Name, otherPartition.Name, otherWorkload.Name}, remainingLPUEvictionPods(t, kube)) - - t.Log("The exact group-scoped table wakes the trigger and evicts only its partition") - config.Name, config.ResourceVersion, config.UID = groupConfig, "", "" - require.NoError(t, kube.Create(t.Context(), config)) - require.Equal(t, []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(trigger)}}, r.evictionRequestsForConfigMap(t.Context(), config)) - reconcileLPUEviction(t, r, trigger.Name) - require.Equal(t, []string{otherPartition.Name, otherWorkload.Name}, remainingLPUEvictionPods(t, kube)) - }) - } -} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 8087d58e6ffb..36e9de19edd4 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -29,18 +29,6 @@ func LPUConfigMapName(root, configHash string) string { return fmt.Sprintf("%s-lpu-%.16s", root, configHash) } -// LPUAgentConfigMapName returns the runtime table referenced by a non-nil Agent Pod. -// The cached Pod projection must preserve ConfigMap volume names. Reading the Pod's -// reference keeps old runtime revisions addressable across workload template updates. -func LPUAgentConfigMapName(pod *corev1.Pod) (string, error) { - for _, volume := range pod.Spec.Volumes { - if volume.Name == lpuConfigVolumeName && volume.ConfigMap != nil && volume.ConfigMap.Name != "" { - return volume.ConfigMap.Name, nil - } - } - return "", fmt.Errorf("LPU-GPU eviction trigger pod %s/%s has no runtime ConfigMap volume", pod.Namespace, pod.Name) -} - func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition.go b/deploy/operator/internal/dynamo/lpx/lpu_partition.go deleted file mode 100644 index eb15aae93bce..000000000000 --- a/deploy/operator/internal/dynamo/lpx/lpu_partition.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "fmt" - "strconv" - "strings" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" - corev1 "k8s.io/api/core/v1" -) - -// LPUAgentRuntimePartition maps a hybrid Agent's Grove index to its runtime -// ConfigMap row. Both inputs must be non-nil and config must be the immutable -// table stamped on the Pod's template. Inputs are not mutated. Runtime rows, -// rather than scheduler partitions, preserve XT's collapsed PropSync cohorts. -func LPUAgentRuntimePartition(pod *corev1.Pod, config *corev1.ConfigMap) (int, error) { - // Consume only canonical Grove identity, never deprecated partition/rank annotations. - rawIndex := pod.Labels[grovecommon.LabelPodCliquePodIndex] - index, err := strconv.ParseUint(rawIndex, 10, 32) - if err != nil || strconv.FormatUint(index, 10) != rawIndex { - return 0, fmt.Errorf("pod %s/%s has invalid Grove Pod index %q", pod.Namespace, pod.Name, rawIndex) - } - model := pod.Annotations[lpxv1alpha1.PodModelAnnotation] - if model == "" { - return 0, fmt.Errorf("pod %s/%s has no LPU model", pod.Namespace, pod.Name) - } - - // Read the same three columns used by the hybrid Agent's partition metadata command. - counts := strings.Split(config.Data["nodes_per_partition"], "\n") - offsets := strings.Split(config.Data["partition_node_offsets"], "\n") - models := strings.Split(config.Data["partition_models"], "\n") - if len(counts) != len(offsets) || len(counts) != len(models) { - return 0, fmt.Errorf("ConfigMap %s/%s has inconsistent runtime partition columns", config.Namespace, config.Name) - } - - // Require one unambiguous runtime range for this model and Grove index. - matched := -1 - for row, rowModel := range models { - if rowModel != model { - continue - } - count, countErr := strconv.ParseUint(counts[row], 10, 32) - offset, offsetErr := strconv.ParseUint(offsets[row], 10, 32) - if countErr != nil || offsetErr != nil || count == 0 { - return 0, fmt.Errorf("ConfigMap %s/%s has invalid runtime partition row %d", config.Namespace, config.Name, row) - } - if index < offset || index-offset >= count { - continue - } - if matched >= 0 { - return 0, fmt.Errorf("Grove Pod index %d matches multiple runtime partitions for model %q", index, model) - } - matched = row - } - if matched < 0 { - return 0, fmt.Errorf("Grove Pod index %d matches no runtime partition for model %q", index, model) - } - return matched, nil -} diff --git a/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go b/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go deleted file mode 100644 index 80b32b670a38..000000000000 --- a/deploy/operator/internal/dynamo/lpx/lpu_partition_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package lpx - -import ( - "testing" - - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestLPUAgentRuntimePartition(t *testing.T) { - t.Log("Cover model-local Grove indexes and incomplete or ambiguous runtime mappings") - tests := []struct { - name string - index string - model string - data map[string]string - want int - wantError string - }{ - {name: "second rank", index: "1", model: "model", want: 0}, - {name: "next partition", index: "2", model: "model", want: 1}, - {name: "other model resets indexes", index: "1", model: "other", want: 2}, - {name: "missing index", model: "model", wantError: "invalid Grove Pod index"}, - {name: "noncanonical index", index: "01", model: "model", wantError: "invalid Grove Pod index"}, - {name: "negative index", index: "-1", model: "model", wantError: "invalid Grove Pod index"}, - {name: "overflow index", index: "4294967296", model: "model", wantError: "invalid Grove Pod index"}, - {name: "missing model", index: "0", wantError: "has no LPU model"}, - {name: "absent model", index: "0", model: "absent", wantError: "matches no runtime partition"}, - {name: "out of range", index: "3", model: "model", wantError: "matches no runtime partition"}, - {name: "missing column", index: "0", model: "model", data: map[string]string{"partition_node_offsets": ""}, wantError: "inconsistent runtime partition columns"}, - {name: "bad count", index: "0", model: "model", data: map[string]string{"nodes_per_partition": "bad\n1\n3"}, wantError: "invalid runtime partition row"}, - {name: "zero count", index: "0", model: "model", data: map[string]string{"nodes_per_partition": "0\n1\n3"}, wantError: "invalid runtime partition row"}, - {name: "bad offset", index: "0", model: "model", data: map[string]string{"partition_node_offsets": "-1\n2\n0"}, wantError: "invalid runtime partition row"}, - {name: "overlapping ranges", index: "1", model: "model", data: map[string]string{"partition_node_offsets": "0\n1\n0"}, wantError: "matches multiple runtime partitions"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Log("Build a runtime table with variable partition sizes and model-local offsets") - config := &corev1.ConfigMap{Data: map[string]string{ - "nodes_per_partition": "2\n1\n3", "partition_node_offsets": "0\n2\n0", - "partition_models": "model\nmodel\nother", - }} - for key, value := range test.data { - config.Data[key] = value - } - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{grovecommon.LabelPodCliquePodIndex: test.index}, - Annotations: map[string]string{ - lpxv1alpha1.PodModelAnnotation: test.model, - lpxv1alpha1.PodPartitionIDAnnotation: "stale", lpxv1alpha1.PodRankInPartitionAnnotation: "not-authority", - }, - }} - originalPod, originalConfig := pod.DeepCopy(), config.DeepCopy() - - t.Log("Resolve the canonical row without mutating producer-owned data") - got, err := LPUAgentRuntimePartition(pod, config) - require.Equal(t, originalPod, pod) - require.Equal(t, originalConfig, config) - if test.wantError != "" { - require.ErrorContains(t, err, test.wantError) - return - } - require.NoError(t, err) - require.Equal(t, test.want, got) - }) - } -} - -func TestLPUAgentRuntimePartitionUsesRenderedHybridTables(t *testing.T) { - for _, test := range []struct { - name string - family BuildFamily - collapsed bool - }{ - {name: "XT", family: BuildFamilyXT}, - {name: "HX", family: BuildFamilyHX}, - {name: "XT collapsed PropSync chain", family: BuildFamilyXT, collapsed: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Render a hybrid runtime table with a two-Agent partition followed by one Agent") - partitions := []BuildPartition{ - {Topology: Topology{ChipCount: 16}, HXExtent: []int64{1, 1, 1, 2}}, - {Topology: Topology{ChipCount: 8}, HXExtent: []int64{1, 1, 1, 1}}, - } - projection := &ModelProjection{ - model: "model", pipeline: PipelineLPX, partitions: partitions, - configuredBuild: Build{Family: test.family, Partitions: partitions}, - } - if test.collapsed { - projection.configuredBuild.Partitions = []BuildPartition{{Topology: Topology{ChipCount: 24}}} - } - config := &corev1.ConfigMap{Data: resolvedPartitionData([]*ModelProjection{projection})} - - t.Log("Resolve every rendered Grove index to the same runtime grouping") - for _, row := range []struct { - index string - want int - }{{"0", 0}, {"1", 0}, {"2", 1}} { - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{grovecommon.LabelPodCliquePodIndex: row.index}, - Annotations: map[string]string{lpxv1alpha1.PodModelAnnotation: "model"}, - }} - got, err := LPUAgentRuntimePartition(pod, config) - require.NoError(t, err) - if test.collapsed { - row.want = 0 - } - require.Equal(t, row.want, got) - } - }) - } -} diff --git a/deploy/operator/internal/podcache/transform.go b/deploy/operator/internal/podcache/transform.go index 3691e666f8dd..0235f6c295e5 100644 --- a/deploy/operator/internal/podcache/transform.go +++ b/deploy/operator/internal/podcache/transform.go @@ -83,15 +83,12 @@ func projectSpec(in corev1.PodSpec) corev1.PodSpec { return corev1.PodSpec{ // Topology and snapshot controllers require the assigned node. NodeName: in.NodeName, - // Node-local LPX disruption recovery distinguishes Agent Pods scheduled - // through LPX from unrelated Pods. - SchedulerName: in.SchedulerName, // Model endpoint classification requires the main container's command // and arguments. Containers: projectContainers(in.Containers), // GMS Pod replacement requires native init-sidecar identity. InitContainers: projectInitContainers(in.InitContainers), - // Topology uses DownwardAPI fields; LPX eviction uses runtime ConfigMap references. + // Topology uses DownwardAPI fields. Volumes: projectVolumes(in.Volumes), } } @@ -127,17 +124,6 @@ func projectInitContainers(in []corev1.Container) []corev1.Container { func projectVolumes(in []corev1.Volume) []corev1.Volume { out := make([]corev1.Volume, 0, len(in)) for i := range in { - // Keep only ConfigMap identity, not its projected keys or mount settings. - if in[i].ConfigMap != nil { - out = append(out, corev1.Volume{ - Name: in[i].Name, - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: in[i].ConfigMap.LocalObjectReference, - }}, - }) - continue - } - if in[i].DownwardAPI == nil { continue } @@ -180,8 +166,7 @@ func projectStatus(in corev1.PodStatus) corev1.PodStatus { // lifecycle phase. Phase: in.Phase, // Model endpoint classification requires Ready; Snapshot-aware GMS replacement - // requires the public restore outcome; node-local LPX disruption recovery - // requires DisruptionTarget and its Kubernetes reason. + // requires the public restore outcome. Conditions: projectConditions(in.Conditions), // DGDR diagnostics inspect container failures. ContainerStatuses: projectContainerStatuses(in.ContainerStatuses), @@ -199,7 +184,7 @@ func projectConditions(in []corev1.PodCondition) []corev1.PodCondition { } switch in[i].Type { case corev1.PodReady: - case corev1.PodConditionType(podcontract.RestoredCondition), corev1.DisruptionTarget: + case corev1.PodConditionType(podcontract.RestoredCondition): condition.Reason = in[i].Reason default: continue diff --git a/deploy/operator/internal/podcache/transform_test.go b/deploy/operator/internal/podcache/transform_test.go index 0fd29f272441..31ce242ae330 100644 --- a/deploy/operator/internal/podcache/transform_test.go +++ b/deploy/operator/internal/podcache/transform_test.go @@ -93,6 +93,7 @@ func TestProjectConsumerContract(t *testing.T) { }}, }, {Name: "discarded-secret", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "large"}}}, + {Name: "discarded-config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config"}}}}, }, }, Status: corev1.PodStatus{ @@ -162,8 +163,8 @@ func TestProjectConsumerContract(t *testing.T) { require.Len(t, got.Spec.Containers, 1) assert.Equal(t, corev1.Container{Name: "main", Command: []string{"python"}, Args: []string{"-m", "dynamo"}}, got.Spec.Containers[0]) }) - t.Run("snapshot and LPX retain restore and disruption state", func(t *testing.T) { - assert.Equal(t, "lpx-scheduler", got.Spec.SchedulerName) + t.Run("snapshot retains restore state", func(t *testing.T) { + assert.Empty(t, got.Spec.SchedulerName) assert.Equal(t, []corev1.PodCondition{ {Type: corev1.PodReady, Status: corev1.ConditionTrue}, { @@ -171,7 +172,6 @@ func TestProjectConsumerContract(t *testing.T) { Status: corev1.ConditionFalse, Reason: podcontract.RestoreReasonFailed, }, - {Type: corev1.DisruptionTarget, Status: corev1.ConditionTrue, Reason: "EvictionByEvictionAPI"}, }, got.Status.Conditions) }) t.Run("failover DGDR GMS replacement and Recreate retain status state", func(t *testing.T) { @@ -300,21 +300,3 @@ func TestConfigureRejectsDuplicatePodEntries(t *testing.T) { func resourceQuantity(value string) resource.Quantity { return resource.MustParse(value) } - -func TestProjectRetainsRuntimeConfigMapIdentity(t *testing.T) { - t.Log("Project a runtime ConfigMap volume without retaining its file layout") - pod := &corev1.Pod{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{ - Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "pcs-workload-lpu-hash"}, - Items: []corev1.KeyToPath{{Key: "partition_models", Path: "models"}}, - Optional: ptr.To(false), DefaultMode: ptr.To(int32(0444)), - }}, - }}}} - require.Equal(t, []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "pcs-workload-lpu-hash"}}, - }}}, Project(pod).Spec.Volumes) - - t.Log("Reapplying the cache projection preserves the same reference") - before := pod.DeepCopy() - require.Equal(t, before, Project(pod)) -} From 20042d7800fdbd884d866a1498ef4cf1c3187cd5 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 15:08:43 +0000 Subject: [PATCH 116/196] refactor(operator): keep LPX download status on the child Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 2 - .../dynamographdeployment_conversion.go | 2 - .../v1alpha1/dynamographdeployment_types.go | 3 - .../api/v1alpha1/lpx_conversion_test.go | 49 +-------- .../api/v1alpha1/zz_generated.deepcopy.go | 5 - .../v1beta1/dynamographdeployment_types.go | 4 - deploy/operator/api/v1beta1/lpx_types.go | 8 -- .../api/v1beta1/zz_generated.deepcopy.go | 25 ----- deploy/operator/cmd/crd-apply/main_test.go | 5 +- .../nvidia.com_dynamographdeployments.yaml | 38 ------- .../controller/dgd_grove_lpx_status.go | 16 +-- .../controller/dgd_grove_lpx_status_test.go | 10 +- .../internal/controller/dgd_grove_program.go | 3 +- .../dgd_lpx_handoff_envtest_test.go | 23 +--- .../controller/dgd_lpx_handoff_test.go | 103 ++---------------- .../additional-resources/api-reference-k8s.md | 35 ------ .../dynamo-graph-deployment.mdx | 10 -- .../kubernetes-api/full-api-reference.mdx | 50 +-------- .../scripts/tests/test_gen_kubernetes_api.py | 21 +--- 19 files changed, 36 insertions(+), 376 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 6ccb1fa01ca9..b4a6abf7718d 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -189,8 +189,6 @@ DynamoGraphDeploymentStatus.components.runtimeNamespace DynamoGraphDeploymentStatus.components.scheduledReplicas DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions -DynamoGraphDeploymentStatus.lpx.modelDownload.builds -DynamoGraphDeploymentStatus.lpx.modelDownload.lastCheckedAt DynamoGraphDeploymentStatus.observedGeneration DynamoGraphDeploymentStatus.placement.score DynamoGraphDeploymentStatus.placement.state diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 2b46041df31e..72d16b89ea95 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -664,7 +664,6 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds } else { dst.Placement = nil } - dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { @@ -712,7 +711,6 @@ func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStat } else { dst.Placement = nil } - dst.LPX = src.LPX if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 20fd2450aabe..96bbb0dc6bbd 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -213,9 +213,6 @@ type DynamoGraphDeploymentStatus struct { // state, and any future placement fields). // +optional Placement *PlacementStatus `json:"placement,omitempty"` - // LPX contains the status of the graph's LPX workload, when present. - // +optional - LPX *v1beta1.DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` } // ServiceCheckpointStatus contains checkpoint information for a single service. diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index 230844e518a0..aa50044ec326 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -6,7 +6,6 @@ package v1alpha1 import ( - "encoding/json" "testing" "time" @@ -93,7 +92,7 @@ func TestDynamoGraphDeploymentRoleShapesRoundTrip(t *testing.T) { } func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { - checkedAt := metav1.NewTime(time.Unix(1_800_000_000, 0)) + transitionTime := metav1.NewTime(time.Unix(1_800_000_000, 0)) src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "lpx", Namespace: "ns"}, Spec: v1beta1.DynamoGraphDeploymentSpec{ @@ -111,12 +110,10 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { }}, }, Status: v1beta1.DynamoGraphDeploymentStatus{ - LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: &v1beta1.ModelDownloadStatus{ - Builds: []string{"gs://models/draft", "gs://models/target"}, - LastCheckedAt: &checkedAt, - }, - }, + Conditions: []metav1.Condition{{ + Type: "Ready", Status: metav1.ConditionFalse, Reason: "Pending", + Message: "Waiting for model downloads to complete", LastTransitionTime: transitionTime, + }}, }, } @@ -143,24 +140,6 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { if diff := cmp.Diff(src.Spec.Scheduling, alpha.Spec.Scheduling); diff != "" { t.Fatalf("LPX scheduling conversion mismatch (-want +got):\n%s", diff) } - if diff := cmp.Diff(src.Status.LPX, alpha.Status.LPX); diff != "" { - t.Fatalf("LPX status conversion mismatch (-want +got):\n%s", diff) - } - - t.Log("Serialize the native LPX payload under one status group in both versions") - payload, err := json.Marshal(src.Status.LPX) - if err != nil { - t.Fatal(err) - } - for _, status := range []any{src.Status, alpha.Status} { - raw, err := json.Marshal(status) - if err != nil { - t.Fatal(err) - } - if want := `{"state":"","lpx":` + string(payload) + `}`; string(raw) != want { - t.Fatalf("%T JSON = %s, want %s", status, raw, want) - } - } t.Log("Edit the alpha configuration and round-trip the complete LPX payload") service.LPX = service.LPX.DeepCopy() @@ -200,22 +179,4 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { if _, exists := got.Annotations[annDGDStatus]; exists { t.Fatalf("canonical LPX status must not use sparse conversion annotations: %v", got.Annotations) } - - t.Log("Preserve an explicitly empty LPX group through native conversion") - alpha.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{} - if err := alpha.ConvertTo(got); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(alpha.Status.LPX, got.Status.LPX); diff != "" { - t.Fatalf("empty LPX group changed (-want +got):\n%s", diff) - } - - t.Log("Clearing live alpha LPX status removes the previously populated group") - alpha.Status.LPX = nil - if err := alpha.ConvertTo(got); err != nil { - t.Fatal(err) - } - if got.Status.LPX != nil { - t.Fatalf("cleared LPX status was restored: %#v", got.Status.LPX) - } } diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index dfdcbdebf529..cbd9c9e36408 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -972,11 +972,6 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(PlacementStatus) (*in).DeepCopyInto(*out) } - if in.LPX != nil { - in, out := &in.LPX, &out.LPX - *out = new(v1beta1.DynamoGraphDeploymentLPXStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 8053b05fd1a8..43fc91743b1a 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -142,10 +142,6 @@ type DynamoGraphDeploymentStatus struct { // state, and any future placement fields). // +optional Placement *PlacementStatus `json:"placement,omitempty"` - - // lpx contains the status of the graph's LPX workload, when present. - // +optional - LPX *DynamoGraphDeploymentLPXStatus `json:"lpx,omitempty"` } // DGD Ready condition reasons used to classify Grove-backed not-ready diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index 517ae757a11d..136c10095714 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -22,14 +22,6 @@ type SchedulingSpec struct { AttemptDeadlineSeconds *int64 `json:"attemptDeadlineSeconds,omitempty"` } -// DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. -// Both served DGD versions use this type, so conversion preserves its live fields natively. -type DynamoGraphDeploymentLPXStatus struct { - // modelDownload contains the model download status for remote builds. - // +optional - ModelDownload *ModelDownloadStatus `json:"modelDownload,omitempty"` -} - // ModelDownloadStatus contains the status of remote LPU model downloads. type ModelDownloadStatus struct { // builds is the sorted set of resolved remote LPU build URLs whose artifacts diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index bec6a9fb5e21..ad13def5d83e 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -508,26 +508,6 @@ func (in *DynamoGraphDeploymentExperimentalSpec) DeepCopy() *DynamoGraphDeployme return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DynamoGraphDeploymentLPXStatus) DeepCopyInto(out *DynamoGraphDeploymentLPXStatus) { - *out = *in - if in.ModelDownload != nil { - in, out := &in.ModelDownload, &out.ModelDownload - *out = new(ModelDownloadStatus) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentLPXStatus. -func (in *DynamoGraphDeploymentLPXStatus) DeepCopy() *DynamoGraphDeploymentLPXStatus { - if in == nil { - return nil - } - out := new(DynamoGraphDeploymentLPXStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DynamoGraphDeploymentList) DeepCopyInto(out *DynamoGraphDeploymentList) { *out = *in @@ -902,11 +882,6 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(PlacementStatus) (*in).DeepCopyInto(*out) } - if in.LPX != nil { - in, out := &in.LPX, &out.LPX - *out = new(DynamoGraphDeploymentLPXStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index 0430bc6c2e0d..810df0431d51 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -76,7 +76,7 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { } } - t.Log("Verify installed schemas preserve DGD placement, keep LPX status grouped, and keep private child status flat") + t.Log("Verify installed schemas preserve DGD placement and keep model downloads on the private child") client, err := apiextensionsclient.NewForConfig(config) if err != nil { t.Fatal(err) @@ -144,7 +144,8 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { if status.Properties["placement"].Type != objectType { t.Errorf("%s %s is missing the established status.placement projection", name, version.Name) } - status = status.Properties["lpx"] + require.NotContains(t, status.Properties, "lpx") + continue } else if _, exists := status.Properties["lpx"]; exists { t.Errorf("%s %s unexpectedly groups private child status", name, version.Name) } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 79008646b845..5109f2b8cbe0 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20256,25 +20256,6 @@ spec: - type type: object type: array - lpx: - description: LPX contains the status of the graph's LPX workload, when present. - properties: - modelDownload: - description: modelDownload contains the model download status for remote builds. - properties: - builds: - description: |- - builds is the sorted set of resolved remote LPU build URLs whose artifacts - were successfully downloaded into model-storage. - items: - type: string - type: array - lastCheckedAt: - description: lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. - format: date-time - type: string - type: object - type: object observedGeneration: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 @@ -38093,25 +38074,6 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map - lpx: - description: lpx contains the status of the graph's LPX workload, when present. - properties: - modelDownload: - description: modelDownload contains the model download status for remote builds. - properties: - builds: - description: |- - builds is the sorted set of resolved remote LPU build URLs whose artifacts - were successfully downloaded into model-storage. - items: - type: string - type: array - lastCheckedAt: - description: lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. - format: date-time - type: string - type: object - type: object observedGeneration: description: observedGeneration is the most recent generation observed by the controller. format: int64 diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 2b27c27a4264..54266180004d 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -17,10 +17,10 @@ func mergeLPXChildStatus( source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, ordinary ReconcileResult, -) (ReconcileResult, *v1beta1.DynamoGraphDeploymentLPXStatus) { +) ReconcileResult { components := lpx.Components(source) if len(components) == 0 && child == nil { - return ordinary, nil + return ordinary } if ordinary.ComponentStatus == nil { ordinary.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) @@ -35,11 +35,7 @@ func mergeLPXChildStatus( current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() observed := current && child.Status.ObservedGeneration == child.Generation - var status *v1beta1.DynamoGraphDeploymentLPXStatus if observed { - status = &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: child.Status.ModelDownload.DeepCopy(), - } for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { ordinary.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() @@ -55,15 +51,15 @@ func mergeLPXChildStatus( ready.Reason == v1alpha1.LPXReadyReasonFailed { ordinary.State = v1beta1.DGDStateFailed ordinary.Reason, ordinary.Message = Reason(ready.Reason), Message(ready.Message) - return ordinary, status + return ordinary } if observed && ready != nil && ready.ObservedGeneration == child.Generation && ready.Status == metav1.ConditionTrue { - return ordinary, status + return ordinary } if observed && ordinary.State != v1beta1.DGDStateFailed && ready != nil && ready.ObservedGeneration == child.Generation { ordinary.State = v1beta1.DGDStatePending ordinary.Reason, ordinary.Message = Reason(ready.Reason), Message(ready.Message) - return ordinary, status + return ordinary } } if ordinary.State != v1beta1.DGDStateFailed { @@ -71,5 +67,5 @@ func mergeLPXChildStatus( ordinary.Reason = "LPXChildPending" ordinary.Message = "Waiting for the current LPX engine revision and readiness" } - return ordinary, status + return ordinary } diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go index 52782f3eb8b9..38928cb0301c 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go @@ -63,19 +63,13 @@ func TestLPXStatusUsesOneCurrentChild(t *testing.T) { want = v1beta1.DGDStateFailed } before := child.DeepCopy() - result, status := mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStateSuccessful}) + result := mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, want, result.State) - t.Log("Publish the aggregate download only for a current child and retain per-component readiness") + t.Log("Publish per-component readiness only for a current child") if scenario == "ready" || scenario == "failure" || scenario == "no-download" { - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: before.Status.ModelDownload}, status) require.Equal(t, before.Status.Components, result.ComponentStatus) - if status.ModelDownload != nil { - status.ModelDownload.Builds[0] = "parent-copy" - status.ModelDownload.LastCheckedAt.Time = metav1.Unix(2, 0).Time - } } else { - require.Nil(t, status) for _, component := range result.ComponentStatus { require.False(t, component.Ready) require.Zero(t, component.Replicas) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index a5a7dd71e7c8..97b83f24c3fe 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -125,7 +125,6 @@ func (p *groveProgram) Reconcile( } // Do not keep reporting retired LPX capacity when ordinary reconciliation fails. - programResult.Status.LPX = nil for name := range programResult.Status.Components { if req.DGD.GetComponentByName(name) == nil { delete(programResult.Status.Components, name) @@ -194,7 +193,7 @@ func (p *groveProgram) Reconcile( } } - result, programResult.Status.LPX = mergeLPXChildStatus(req.DGD, child, result) + result = mergeLPXChildStatus(req.DGD, child, result) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) if child != nil && !child.DeletionTimestamp.IsZero() { programResult.RequeueAfter = 5 * time.Second diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 1c751215149a..73066597428e 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -235,7 +235,6 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { assert.Equal(c, failed.Reason, ready.Reason) assert.Equal(c, failed.Message, ready.Message) assert.NotNil(c, child.Status.ModelDownload) - assert.Nil(c, source.Status.LPX) }, 20*time.Second, 50*time.Millisecond) t.Log("The rejected publication created neither Grove workloads nor scheduler requests") @@ -293,8 +292,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, env.Client().Create(t.Context(), source)) t.Log("Persist the pending component projection using the real DGD status schema") - result := &ReconcileResult{} - projectLPXChildStatus(source, nil, result, &source.Status) + result := mergeLPXChildStatus(source, nil, ReconcileResult{}) source.Status.Components = result.ComponentStatus source.Status.State = result.State require.NoError(t, env.Client().Status().Update(t.Context(), source)) @@ -313,7 +311,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, err) require.Equal(t, revision, child.Spec.InputRevision) - t.Log("Project the observed download payload and its actionable failure") + t.Log("Persist the child's download payload and project its actionable failure") child.Status.ObservedGeneration = child.Generation child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, @@ -330,13 +328,10 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NotNil(t, schedulingFailed) require.True(t, failureTime.Equal(&schedulingFailed.LastTransitionTime)) require.Equal(t, child.Generation, schedulingFailed.ObservedGeneration) - projected := v1beta1.DynamoGraphDeploymentStatus{} - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, child, result, &projected) + result = mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStateFailed, result.State) require.Equal(t, Reason(child.Status.Conditions[0].Reason), result.Reason) require.Equal(t, Message(child.Status.Conditions[0].Message), result.Message) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) t.Log("Ordinary source metadata does not advance the child generation or frozen source identity") source.Labels = map[string]string{"unrelated": "metadata"} @@ -345,10 +340,8 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, err) require.Equal(t, child.ResourceVersion, unchanged.ResourceVersion) require.Equal(t, child.Annotations, unchanged.Annotations) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, unchanged, result, &projected) + result = mergeLPXChildStatus(source, unchanged, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStateFailed, result.State) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) t.Log("An LPX template edit advances the real child generation and invalidates the old observation") source.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXAgent).PodTemplate.Spec.Containers[0].Image = "lpu-runtime:next" @@ -361,11 +354,9 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, err) require.Equal(t, revision, updated.Spec.InputRevision) require.Equal(t, child.Status.ObservedGeneration, updated.Status.ObservedGeneration) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, updated, result, &projected) + result = mergeLPXChildStatus(source, updated, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStatePending, result.State) require.Equal(t, Reason("LPXChildPending"), result.Reason) - require.Nil(t, projected.LPX) t.Log("Reject malformed revision hashes in the API server") invalid := updated.DeepCopy() @@ -379,10 +370,8 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { stored := &v1alpha1.LPXGraphDeployment{} require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(updated), stored)) require.Equal(t, updated.Status, stored.Status) - result = &ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, stored, result, &projected) + result = mergeLPXChildStatus(source, stored, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStateFailed, result.State) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{ModelDownload: child.Status.ModelDownload}, projected.LPX) storedSource := &v1beta1.DynamoGraphDeployment{} require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(source), storedSource)) require.Equal(t, source.Status, storedSource.Status) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 63f6824c2fdf..92acfd5a0fcb 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -65,16 +64,6 @@ func newLPXHandoffFixture(t *testing.T, fixture string) (*v1alpha1.LPXGraphDeplo return child, source, kube } -// projectLPXChildStatus applies the production call-site composition for focused tests. -func projectLPXChildStatus( - source *v1beta1.DynamoGraphDeployment, - child *v1alpha1.LPXGraphDeployment, - result *ReconcileResult, - status *v1beta1.DynamoGraphDeploymentStatus, -) { - *result, status.LPX = mergeLPXChildStatus(source, child, *result) -} - func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { for _, test := range []struct { name string @@ -182,8 +171,7 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { require.NoError(t, err) require.Equal(t, revision, child.Spec.InputRevision) require.NotContains(t, child.Annotations, "lpx.nvidia.com/podcliqueset-name") - pending := ReconcileResult{State: v1beta1.DGDStatePending} - projectLPXChildStatus(source, child, &pending, &v1beta1.DynamoGraphDeploymentStatus{}) + pending := mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStatePending}) require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, pending.ComponentStatus["lpx"].ComponentKind) t.Log("Reject foreign references before returning a child for status projection") @@ -220,18 +208,13 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { Replicas: 3, AvailableReplicas: ptr.To(int32(3)), Ready: true, GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), }} - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"build"}} source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4)), Ready: true}}} - status := *source.Status.DeepCopy() - projectLPXChildStatus(source, child, &result, &status) + result = mergeLPXChildStatus(source, child, result) require.Equal(t, v1beta1.DGDStateSuccessful, result.State) require.Len(t, result.ComponentStatus, 2) require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) - require.Equal(t, source.Status.Placement, status.Placement) - status.LPX.ModelDownload.Builds[0] = "parent-copy" - require.Equal(t, "build", child.Status.ModelDownload.Builds[0]) t.Log("Reject missing or stale children and stale Ready conditions") for _, failure := range []string{"missing-child", "generation", "condition"} { @@ -244,30 +227,19 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { case "condition": stale.Status.Conditions[0].ObservedGeneration-- } - pending := ReconcileResult{State: v1beta1.DGDStateSuccessful} - previous := status - projectLPXChildStatus(source, stale, &pending, &previous) + pending := mergeLPXChildStatus(source, stale, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStatePending, pending.State, failure) - require.Equal(t, source.Status.Placement, previous.Placement, failure) - if failure == "missing-child" || failure == "generation" { - require.Nil(t, previous.LPX, failure) - } else { - require.NotNil(t, previous.LPX, failure) - } } t.Log("A partially ready child retains logical replica counts and GPU capacity in public status") partial := child.Status.Components["lpx"] partial.AvailableReplicas, partial.Ready = ptr.To(int32(2)), false child.Status.Components["lpx"] = partial - child.Status.ModelDownload = nil result.Reason = "CheckpointReady" meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonPending, Message: "Waiting for removed LPX requests"}) - projectLPXChildStatus(source, child, &result, &status) + result = mergeLPXChildStatus(source, child, result) require.Equal(t, v1beta1.DGDStatePending, result.State) require.Equal(t, Reason(v1alpha1.LPXReadyReasonPending), result.Reason) - require.Equal(t, &v1beta1.DynamoGraphDeploymentLPXStatus{}, status.LPX) - require.Equal(t, source.Status.Placement, status.Placement) require.Equal(t, partial, result.ComponentStatus["lpx"]) source.Status.Components = result.ComponentStatus request := &v1beta1.DynamoGraphDeploymentRequest{} @@ -276,21 +248,16 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Equal(t, ptr.To(int32(6)), request.Status.DeploymentInfo.AvailableReplicas) t.Log("Retirement remains pending in final DGD status after ordinary checkpoint readiness") - projected := workloadProgramResult{Status: status} + projected := newWorkloadProgramResult(source) projected.applyReconcileResult(source.Generation, result) require.Equal(t, v1beta1.DGDStatePending, projected.Status.State) + require.Equal(t, source.Status.Placement, projected.Status.Placement) ready := meta.FindStatusCondition(projected.Status.Conditions, "Ready") require.NotNil(t, ready) require.Equal(t, metav1.ConditionFalse, ready.Status) require.Equal(t, child.Status.Conditions[0].Reason, ready.Reason) require.Equal(t, child.Status.Conditions[0].Message, ready.Message) require.Equal(t, source.Generation, ready.ObservedGeneration) - - t.Log("Remove the complete LPX group when its components leave the source graph") - source.Spec.Components = nil - projectLPXChildStatus(source, child, &result, &status) - require.Nil(t, status.LPX) - require.Equal(t, source.Status.Placement, status.Placement) } func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { @@ -309,12 +276,9 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { t.Log("Only a current child failure may bypass the complete-observation gate") child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"stale-build"}} child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} result := ReconcileResult{State: v1beta1.DGDStateSuccessful} - status := v1beta1.DynamoGraphDeploymentStatus{LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: child.Status.ModelDownload.DeepCopy(), - }} + status := v1beta1.DynamoGraphDeploymentStatus{} switch test.name { case "observed-result": child.Status.ObservedGeneration = child.Generation @@ -329,7 +293,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { case "ordinary-failure": result = ReconcileResult{State: v1beta1.DGDStateFailed, Reason: "OrdinaryFailure", Message: "Existing ordinary failure"} } - projectLPXChildStatus(source, child, &result, &status) + result = mergeLPXChildStatus(source, child, result) require.Equal(t, test.failed, result.State == v1beta1.DGDStateFailed) if test.name == "ordinary-failure" { require.Equal(t, Reason("OrdinaryFailure"), result.Reason) @@ -343,8 +307,6 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { t.Log("A failure condition does not make stale result payloads current") if test.name == "observed-result" { - require.NotNil(t, status.LPX) - require.NotNil(t, status.LPX.ModelDownload) require.Equal(t, int32(1), result.ComponentStatus["lpx"].Replicas) t.Log("A retained ordinary rollout cannot hide the child failure in final DGD status") @@ -358,7 +320,6 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { require.Equal(t, child.Status.Conditions[0].Message, ready.Message) require.Equal(t, source.Generation, ready.ObservedGeneration) } else { - require.Nil(t, status.LPX) require.Zero(t, result.ComponentStatus["lpx"].Replicas) } }) @@ -457,7 +418,7 @@ func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { require.Equal(t, beforeChild, current) ordinary := ReconcileResult{State: v1beta1.DGDStateSuccessful, Reason: "Ready"} projected := newWorkloadProgramResult(source) - projectLPXChildStatus(source, current, &ordinary, &projected.Status) + ordinary = mergeLPXChildStatus(source, current, ordinary) projected.applyReconcileResult(source.Generation, ordinary) require.Equal(t, v1beta1.DGDStateSuccessful, projected.Status.State) require.True(t, meta.IsStatusConditionTrue(projected.Status.Conditions, "Ready")) @@ -484,12 +445,8 @@ func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { require.Equal(t, beforeChild.Status, updated.Status) // Fake clients do not advance generations; the isolated API test covers that transition. updated.Generation++ - projected := newWorkloadProgramResult(source) - require.NotNil(t, projected.Status.LPX) - pending := ReconcileResult{State: v1beta1.DGDStateSuccessful} - projectLPXChildStatus(source, updated, &pending, &projected.Status) + pending := mergeLPXChildStatus(source, updated, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, v1beta1.DGDStatePending, pending.State) - require.Nil(t, projected.Status.LPX) } func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { @@ -649,15 +606,11 @@ func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { }) source.Generation++ require.NoError(t, kube.Update(t.Context(), source)) - source.Status.LPX = &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}}, - } source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} source.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ "lpx": {Replicas: 1}, "prefill": {Replicas: 2}, } if test.childGone { - source.Status.LPX = nil source.Status.Components["lpx"] = v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup} } previousStatus := source.Status.DeepCopy() @@ -681,10 +634,8 @@ func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { require.Equal(t, previousStatus, &source.Status) require.Equal(t, source.Status.Components["prefill"], result.Status.Components["prefill"]) if !test.retainLPX { - require.Nil(t, result.Status.LPX) require.NotContains(t, result.Status.Components, "lpx") } else { - require.Equal(t, source.Status.LPX, result.Status.LPX) require.Equal(t, source.Status.Components, result.Status.Components) } require.Equal(t, source.Status.Placement, result.Status.Placement) @@ -847,37 +798,3 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) require.Equal(t, child, stored) } - -func TestOrdinaryDGDClearsLPXStatusWithoutRefresh(t *testing.T) { - t.Log("Seed an ordinary DGD with obsolete LPX download status and independent placement") - checkedAt := metav1.Unix(1, 0) - source := &v1beta1.DynamoGraphDeployment{ - ObjectMeta: metav1.ObjectMeta{Name: "download-status", Namespace: "default", Generation: 1}, - Status: v1beta1.DynamoGraphDeploymentStatus{ - State: v1beta1.DGDStateSuccessful, ObservedGeneration: 1, - Placement: &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported}, - LPX: &v1beta1.DynamoGraphDeploymentLPXStatus{ - ModelDownload: &v1beta1.ModelDownloadStatus{Builds: []string{"model/build"}, LastCheckedAt: &checkedAt}, - }, - }, - } - kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). - WithStatusSubresource(source).WithObjects(source).Build() - r := &DynamoGraphDeploymentReconciler{ - Client: kube, Recorder: events.NewFakeRecorder(10), - Config: &configv1alpha1.OperatorConfiguration{ - Namespace: configv1alpha1.NamespaceConfiguration{Restricted: source.Namespace}, - LPX: configv1alpha1.LPXConfiguration{Enabled: true}, - }, - RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, - } - - t.Log("Reconcile ordinary state without scheduling an LPX download refresh") - result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(source)}) - require.NoError(t, err) - require.Zero(t, result.RequeueAfter) - updated := &v1beta1.DynamoGraphDeployment{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(source), updated)) - require.Nil(t, updated.Status.LPX) - require.Equal(t, source.Status.Placement, updated.Status.Placement) -} diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index abfac117dde8..b4f539b1d270 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -641,7 +641,6 @@ _Appears in:_ | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| | `placement` _[PlacementStatus](#placementstatus)_ | Placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| -| `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | LPX contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| #### DynamoModel @@ -2009,24 +2008,6 @@ _Appears in:_ | `kvTransferPolicy` _[KvTransferPolicy](#kvtransferpolicy)_ | kvTransferPolicy configures topology-aware routing for KV-cache
transfers between prefill and decode workers. | | Optional: \{\}
| -#### DynamoGraphDeploymentLPXStatus - - - -DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. -Both served DGD versions use this type, so conversion preserves its live fields natively. - - - -_Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) -- [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `modelDownload` _[ModelDownloadStatus](#modeldownloadstatus)_ | modelDownload contains the model download status for remote builds. | | Optional: \{\}
| - - #### v1beta1 DynamoGraphDeploymentRequest @@ -2222,7 +2203,6 @@ _Appears in:_ | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| | `placement` _[PlacementStatus](#placementstatus)_ | placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| -| `lpx` _[DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus)_ | lpx contains the status of the graph's LPX workload, when present. | | Optional: \{\}
| #### EPPConfig @@ -2555,21 +2535,6 @@ _Appears in:_ | `pvcMountPath` _string_ | PVCMountPath is the mount path for the PVC inside the container. | /opt/model-cache | Optional: \{\}
| -#### ModelDownloadStatus - - - -ModelDownloadStatus contains the status of remote LPU model downloads. - - - -_Appears in:_ -- [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `builds` _string array_ | builds is the sorted set of resolved remote LPU build URLs whose artifacts
were successfully downloaded into model-storage. | | Optional: \{\}
| -| `lastCheckedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. | | Optional: \{\}
| #### ModelReference diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index c8c003fbc4d3..f8222ef56732 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -190,16 +190,6 @@ The operator maintains observed state under `status`. Most recent `metadata.generation` observed by the controller. - - LPX model download observations for the current configuration. Omitted when no current LPX result exists, including ordinary-only deployments. LPX component entries and aggregate readiness remain under `status.components`, `status.state`, and `status.conditions`. - - - - - Downloaded builds and the time they were last checked. - - - Status of a graph-level restart in progress. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 5aa9bab147ba..f25d7f32f9c8 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,11 +10,11 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 3 Kubernetes API packages with 171 typed CRD and config sections. +Dynamo publishes 3 Kubernetes API packages with 169 typed CRD and config sections. 69 types -73 types +71 types 29 types @@ -1103,12 +1103,6 @@ Placement groups DGD-level scheduler placement signals (score, reporting state, See [PlacementStatus](#placementstatus). **Validation:** Optional: \{\} - - -LPX contains the status of the graph's LPX workload, when present. -See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). -**Validation:** Optional: \{\} - @@ -2928,21 +2922,6 @@ See [KvTransferPolicy](#v1beta1-kvtransferpolicy). - -DynamoGraphDeploymentLPXStatus contains the public status of a graph's LPX workload. -Both served DGD versions use this type, so conversion preserves its live fields natively. - - -**Kind:** `type` -**Appears in:** [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus), [DynamoGraphDeploymentStatus](#v1beta1-dynamographdeploymentstatus) - - -modelDownload contains the model download status for remote builds. -See [ModelDownloadStatus](#modeldownloadstatus). -**Validation:** Optional: \{\} - - - DynamoGraphDeploymentRequest is the Schema for the dynamographdeploymentrequests API. It provides a simplified, SLA-driven interface for deploying inference models on Dynamo. @@ -3320,12 +3299,6 @@ placement groups DGD-level scheduler placement signals (score, reporting state, See [PlacementStatus](#v1beta1-placementstatus). **Validation:** Optional: \{\} - - -lpx contains the status of the graph's LPX workload, when present. -See [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus). -**Validation:** Optional: \{\} - @@ -3709,25 +3682,6 @@ PVCMountPath is the mount path for the PVC inside the container. - -ModelDownloadStatus contains the status of remote LPU model downloads. - - -**Kind:** `type` -**Appears in:** [DynamoGraphDeploymentLPXStatus](#dynamographdeploymentlpxstatus) - - -builds is the sorted set of resolved remote LPU build URLs whose artifacts were successfully downloaded into model-storage. -**Validation:** Optional: \{\} - - - -lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. -See [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta). -**Validation:** Optional: \{\} - - - ModelReference identifies a model served by a component. When specified, a headless service is created for endpoint discovery. diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index e9cff13ccfcb..37aeed510e22 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -49,7 +49,7 @@ # PodSnapshotContentSource/Spec/Status, PodSnapshotReference), which are # owned by github.com/ai-dynamo/snapshot. "nvidia.com/v1alpha1": 69, - "nvidia.com/v1beta1": 73, + "nvidia.com/v1beta1": 71, "operator.config.dynamo.nvidia.com/v1alpha1": 29, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( @@ -206,24 +206,6 @@ def test_resource_types_link_to_real_types_on_the_same_page( ) -def test_cross_package_status_links_resolve_to_v1beta1_types( - reference: kubernetes_api_discovery.KubernetesReference, -) -> None: - """v1alpha1 may intentionally reuse status types owned by v1beta1.""" - by_package = {package.name: package for package in reference.packages} - alpha = by_package["nvidia.com/v1alpha1"] - beta = by_package["nvidia.com/v1beta1"] - alpha_status = next( - type_ for type_ in alpha.types if type_.name == "DynamoGraphDeploymentStatus" - ) - lpx_field = next(field for field in alpha_status.fields if field.name == "lpx") - beta_lpx_status = next( - type_ for type_ in beta.types if type_.name == "DynamoGraphDeploymentLPXStatus" - ) - - assert f"](#{beta_lpx_status.anchor})" in lpx_field.type - - def test_v1beta1_shared_type_names_use_deduplicated_anchors( reference: kubernetes_api_discovery.KubernetesReference, ) -> None: @@ -442,7 +424,6 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( ( ("LPXConfig", "DynamoComponentDeploymentSharedSpec"), ("SchedulingSpec", "DynamoGraphDeploymentSpec"), - ("DynamoGraphDeploymentLPXStatus", "DynamoGraphDeploymentStatus"), ), ) def test_shared_lpx_type_links_across_api_versions( From 465ba3d7899f880aa54c94d468bae5de2b83218a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:27:45 +0300 Subject: [PATCH 117/196] Update deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go Co-authored-by: Dr. Stefan Schimanski Signed-off-by: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> --- deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 9e08f77f3321..e024290b39dc 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -178,6 +178,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. + // LPX components each require an agent role. A DGD may contain independent // LPX components, each with its own conductor role, or a shared draft and // target pair with a conductor role only on the target. Every LPX role From 9d0130d3d94a7b113e18c1fbd59a83804e85e6f3 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 15:59:18 +0000 Subject: [PATCH 118/196] fix(operator): preserve complete roles documentation Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go | 2 +- .../config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml | 1 + .../config/crd/bases/nvidia.com_dynamographdeployments.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index e024290b39dc..78645e5eea29 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -178,7 +178,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. - + // // LPX components each require an agent role. A DGD may contain independent // LPX components, each with its own conductor role, or a shared draft and // target pair with a conductor role only on the target. Every LPX role diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index ca6f43856ca5..22d1eb271d3f 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -28656,6 +28656,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 5109f2b8cbe0..ff7601493005 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -29277,6 +29277,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role From d044e923b37f2315e4ce1ab186cbcc95b8bfbf12 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 15:49:58 +0000 Subject: [PATCH 119/196] fix(operator): scope LPX scheduling deadlines to components Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 3 +- .../dynamographdeployment_conversion.go | 2 - .../v1alpha1/dynamographdeployment_types.go | 5 - .../api/v1alpha1/lpx_conversion_test.go | 41 +++++-- .../api/v1alpha1/zz_generated.deepcopy.go | 7 +- .../v1beta1/dynamographdeployment_types.go | 4 - deploy/operator/api/v1beta1/lpx_types.go | 7 +- .../api/v1beta1/zz_generated.deepcopy.go | 12 +-- .../nvidia.com_dynamographdeployments.yaml | 52 ++++----- .../internal/controller/lpx/controller.go | 16 ++- .../controller/lpx/controller_test.go | 102 ++++++++++++++---- .../lpx/pipeline_request_deadline.go | 22 ++-- .../lpx/pipeline_request_deadline_test.go | 7 +- deploy/operator/internal/dynamo/graph_lpx.go | 2 - .../internal/dynamo/graph_lpx_test.go | 4 +- .../internal/dynamo/lpx/components_test.go | 1 - .../internal/dynamo/lpx/model_projection.go | 5 + .../dynamo/lpx/resolve_workload_test.go | 2 +- ...hdeployment_lpx_validation_envtest_test.go | 30 ++++++ .../additional-resources/api-reference-k8s.md | 8 +- .../dynamo-component-deployment.mdx | 20 +++- .../dynamo-graph-deployment.mdx | 16 --- .../kubernetes-api/full-api-reference.mdx | 22 ++-- .../scripts/tests/test_gen_kubernetes_api.py | 1 - 24 files changed, 253 insertions(+), 138 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index b4a6abf7718d..9c37dd302894 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -62,6 +62,7 @@ DynamoComponentDeploymentSpec.experimental.grove.forceScalingGroup DynamoComponentDeploymentSpec.frontendSidecar DynamoComponentDeploymentSpec.globalDynamoNamespace DynamoComponentDeploymentSpec.lpx.buildId +DynamoComponentDeploymentSpec.lpx.scheduling.attemptDeadlineSeconds DynamoComponentDeploymentSpec.minAvailable DynamoComponentDeploymentSpec.modelRef.name DynamoComponentDeploymentSpec.modelRef.revision @@ -135,6 +136,7 @@ DynamoGraphDeploymentSpec.components.experimental.grove.forceScalingGroup DynamoGraphDeploymentSpec.components.frontendSidecar DynamoGraphDeploymentSpec.components.globalDynamoNamespace DynamoGraphDeploymentSpec.components.lpx.buildId +DynamoGraphDeploymentSpec.components.lpx.scheduling.attemptDeadlineSeconds DynamoGraphDeploymentSpec.components.minAvailable DynamoGraphDeploymentSpec.components.modelRef.name DynamoGraphDeploymentSpec.components.modelRef.revision @@ -170,7 +172,6 @@ DynamoGraphDeploymentSpec.providerOverride.value DynamoGraphDeploymentSpec.restart.id DynamoGraphDeploymentSpec.restart.strategy.order DynamoGraphDeploymentSpec.restart.strategy.type -DynamoGraphDeploymentSpec.scheduling.attemptDeadlineSeconds DynamoGraphDeploymentSpec.topologyConstraint.clusterTopologyName DynamoGraphDeploymentSpec.topologyConstraint.packDomain DynamoGraphDeploymentStatus.checkpoints.checkpointID diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 72d16b89ea95..527433f9fabf 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -107,7 +107,6 @@ func (src *DynamoGraphDeployment) ConvertTo(dstRaw conversion.Hub) error { // v1beta1. func ConvertFromDynamoGraphDeploymentSpec(src *DynamoGraphDeploymentSpec, dst *v1beta1.DynamoGraphDeploymentSpec, restored *v1beta1.DynamoGraphDeploymentSpec, save *DynamoGraphDeploymentSpec, ctx DynamoGraphDeploymentConversionContext) error { // Convert fields represented by both versions from the live source. - dst.Scheduling = src.Scheduling dst.Annotations = src.Annotations dst.Labels = src.Labels dst.PriorityClassName = src.PriorityClassName @@ -421,7 +420,6 @@ func (dst *DynamoGraphDeployment) ConvertFrom(srcRaw conversion.Hub) error { // v1alpha1. func ConvertToDynamoGraphDeploymentSpec(src *v1beta1.DynamoGraphDeploymentSpec, dst *DynamoGraphDeploymentSpec, restored *DynamoGraphDeploymentSpec, save *v1beta1.DynamoGraphDeploymentSpec, ctx DynamoGraphDeploymentConversionContext) error { // Convert fields represented by both versions from the live source. - dst.Scheduling = src.Scheduling dst.Annotations = src.Annotations dst.Labels = src.Labels dst.PriorityClassName = src.PriorityClassName diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 96bbb0dc6bbd..4a43198dc78e 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" ) @@ -93,10 +92,6 @@ type DynamoGraphDeploymentSpec struct { // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` - // Scheduling configures attempts for LPX components. Omission means no deadline. - // +optional - Scheduling *v1beta1.SchedulingSpec `json:"scheduling,omitempty"` - // Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). // Service-level annotations take precedence over these values. // +optional diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/lpx_conversion_test.go index aa50044ec326..c4d17633032f 100644 --- a/deploy/operator/api/v1alpha1/lpx_conversion_test.go +++ b/deploy/operator/api/v1alpha1/lpx_conversion_test.go @@ -22,6 +22,9 @@ const editedThroughAlphaBuildID = "model/edited-through-alpha" func testCanonicalLPXConfig() *v1beta1.LPXConfig { return &v1beta1.LPXConfig{ BuildID: "model/build", + Scheduling: &v1beta1.SchedulingSpec{ + AttemptDeadlineSeconds: ptr.To[int64](900), + }, } } @@ -96,9 +99,6 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "lpx", Namespace: "ns"}, Spec: v1beta1.DynamoGraphDeploymentSpec{ - Scheduling: &v1beta1.SchedulingSpec{ - AttemptDeadlineSeconds: ptr.To[int64](900), - }, Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ ComponentName: "lpx", ComponentType: v1beta1.ComponentTypeLPX, @@ -137,13 +137,11 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { if diff := cmp.Diff(wantAlphaRoles, service.Roles); diff != "" { t.Fatalf("LPX role order or optional fields changed in alpha (-want +got):\n%s", diff) } - if diff := cmp.Diff(src.Spec.Scheduling, alpha.Spec.Scheduling); diff != "" { - t.Fatalf("LPX scheduling conversion mismatch (-want +got):\n%s", diff) - } t.Log("Edit the alpha configuration and round-trip the complete LPX payload") service.LPX = service.LPX.DeepCopy() service.LPX.BuildID = editedThroughAlphaBuildID + *service.LPX.Scheduling.AttemptDeadlineSeconds = 600 service.Roles = []ComponentRoleSpec{ *service.Roles[1].DeepCopy(), {Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: ptr.To(testLPXRoleTemplate("conductor"))}, @@ -167,7 +165,7 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { if diff := cmp.Diff(original, src); diff != "" { t.Fatalf("alpha edit mutated the source LPX deployment (-want +got):\n%s", diff) } - if diff := cmp.Diff(src.Spec.Scheduling, got.Spec.Scheduling); diff != "" { + if diff := cmp.Diff(&v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](600)}, component.LPX.Scheduling); diff != "" { t.Fatalf("LPX scheduling round-trip mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(src.Status, got.Status); diff != "" { @@ -180,3 +178,32 @@ func TestDynamoGraphDeploymentLPXRoundTrip(t *testing.T) { t.Fatalf("canonical LPX status must not use sparse conversion annotations: %v", got.Annotations) } } + +func TestDynamoGraphDeploymentLPXComponentSchedulingRoundTrip(t *testing.T) { + t.Log("Give target and draft different deadlines and preserve unlimited component settings") + src := &v1beta1.DynamoGraphDeployment{} + for _, component := range []struct { + name string + scheduling *v1beta1.SchedulingSpec + }{ + {name: "draft", scheduling: &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](300)}}, + {name: "empty", scheduling: &v1beta1.SchedulingSpec{}}, + {name: "target", scheduling: &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](900)}}, + {name: "unlimited"}, + } { + src.Spec.Components = append(src.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: component.name, + ComponentType: v1beta1.ComponentTypeLPX, + LPX: &v1beta1.LPXConfig{ + BuildID: "model/" + component.name, + Scheduling: component.scheduling, + }, + }) + } + + t.Log("Round-trip each component's deadline without imposing a shared value or default") + got := roundTripFromV1beta1(t, src) + if diff := cmp.Diff(src.Spec, got.Spec); diff != "" { + t.Fatalf("LPX component scheduling round-trip mismatch (-want +got):\n%s", diff) + } +} diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index cbd9c9e36408..a81ec03a4e6c 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -460,7 +460,7 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(v1beta1.LPXConfig) - **out = **in + (*in).DeepCopyInto(*out) } if in.Checkpoint != nil { in, out := &in.Checkpoint, &out.Checkpoint @@ -857,11 +857,6 @@ func (in *DynamoGraphDeploymentSpec) DeepCopyInto(out *DynamoGraphDeploymentSpec *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } - if in.Scheduling != nil { - in, out := &in.Scheduling, &out.Scheduling - *out = new(v1beta1.SchedulingSpec) - (*in).DeepCopyInto(*out) - } if in.Annotations != nil { in, out := &in.Annotations, &out.Annotations *out = make(map[string]string, len(*in)) diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 43fc91743b1a..583742860002 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -33,10 +33,6 @@ type DynamoGraphDeploymentSpec struct { // +optional ProviderOverride *ProviderOverride `json:"providerOverride,omitempty"` - // scheduling configures attempts for LPX components. Omission means no deadline. - // +optional - Scheduling *SchedulingSpec `json:"scheduling,omitempty"` - // annotations to propagate to all child resources (PCS, DCD, Deployments, // and pod templates). Component-level (`podTemplate`) values take precedence // on conflict. diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index 136c10095714..9b32f934369f 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -5,11 +5,16 @@ package v1beta1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// LPXConfig identifies the component's compiled model. +// LPXConfig identifies the component's compiled model and configures scheduling. type LPXConfig struct { // buildId references the immutable model build. // +kubebuilder:validation:MinLength=1 BuildID string `json:"buildId"` + + // scheduling configures this component's LPX scheduling attempts. + // Omission means no deadline. + // +optional + Scheduling *SchedulingSpec `json:"scheduling,omitempty"` } // SchedulingSpec configures LPX scheduling attempts. diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index ad13def5d83e..f97822641d3a 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -369,7 +369,7 @@ func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponent if in.LPX != nil { in, out := &in.LPX, &out.LPX *out = new(LPXConfig) - **out = **in + (*in).DeepCopyInto(*out) } if in.FrontendSidecar != nil { in, out := &in.FrontendSidecar, &out.FrontendSidecar @@ -783,11 +783,6 @@ func (in *DynamoGraphDeploymentSpec) DeepCopyInto(out *DynamoGraphDeploymentSpec *out = new(ProviderOverride) (*in).DeepCopyInto(*out) } - if in.Scheduling != nil { - in, out := &in.Scheduling, &out.Scheduling - *out = new(SchedulingSpec) - (*in).DeepCopyInto(*out) - } if in.Annotations != nil { in, out := &in.Annotations, &out.Annotations *out = make(map[string]string, len(*in)) @@ -1135,6 +1130,11 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXConfig) DeepCopyInto(out *LPXConfig) { *out = *in + if in.Scheduling != nil { + in, out := &in.Scheduling, &out.Scheduling + *out = new(SchedulingSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXConfig. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index ff7601493005..47575aa631bd 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -398,18 +398,6 @@ spec: required: - id type: object - scheduling: - description: Scheduling configures attempts for LPX components. Omission means no deadline. - properties: - attemptDeadlineSeconds: - description: |- - attemptDeadlineSeconds limits how long each LPR may remain pending. - Omission means unlimited; the value is not a solver budget. - format: int64 - maximum: 9223372036 - minimum: 1 - type: integer - type: object services: additionalProperties: properties: @@ -11529,6 +11517,20 @@ spec: description: buildId references the immutable model build. minLength: 1 type: string + scheduling: + description: |- + scheduling configures this component's LPX scheduling attempts. + Omission means no deadline. + properties: + attemptDeadlineSeconds: + description: |- + attemptDeadlineSeconds limits how long each LPR may remain pending. + Omission means unlimited; the value is not a solver budget. + format: int64 + maximum: 9223372036 + minimum: 1 + type: integer + type: object required: - buildId type: object @@ -21024,6 +21026,20 @@ spec: description: buildId references the immutable model build. minLength: 1 type: string + scheduling: + description: |- + scheduling configures this component's LPX scheduling attempts. + Omission means no deadline. + properties: + attemptDeadlineSeconds: + description: |- + attemptDeadlineSeconds limits how long each LPR may remain pending. + Omission means unlimited; the value is not a solver budget. + format: int64 + maximum: 9223372036 + minimum: 1 + type: integer + type: object required: - buildId type: object @@ -37845,18 +37861,6 @@ spec: required: - id type: object - scheduling: - description: scheduling configures attempts for LPX components. Omission means no deadline. - properties: - attemptDeadlineSeconds: - description: |- - attemptDeadlineSeconds limits how long each LPR may remain pending. - Omission means unlimited; the value is not a solver budget. - format: int64 - maximum: 9223372036 - minimum: 1 - type: integer - type: object topologyConstraint: description: |- topologyConstraint is the deployment-level topology constraint. When diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 2f08aa91d698..2d6bcd4ed20a 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -223,6 +223,8 @@ func (r *graphReconciler) reconcileWorkloads( desiredRequests = make(map[string]*lpxv1alpha1.LPUPipelineRequest) explicitReplicas = make(map[string]*int32, len(plans)) missingRequests []*lpxv1alpha1.LPUPipelineRequest + expiredRequests []*lpxv1alpha1.LPUPipelineRequest + deadlineAt time.Time ) // Resolve each workload in the same order used for request publication. @@ -249,10 +251,20 @@ func (r *graphReconciler) reconcileWorkloads( maps.Copy(desiredRequests, desired) missingRequests = append(missingRequests, missing...) + + // Expanded models use their source component's policy, not the conductor's. + secondsByModel := make(map[string]*int64) + for _, projection := range workload.ModelProjections() { + if scheduling := dgd.GetComponentByName(projection.ComponentName()).LPX.Scheduling; scheduling != nil { + secondsByModel[projection.Model()] = scheduling.AttemptDeadlineSeconds + } + } + expired, next := pipelineRequestDeadlines(desired, secondsByModel, deadlineAt) + expiredRequests = append(expiredRequests, expired...) + deadlineAt = next } - // One graph-wide deadline observation preserves the earliest wakeup on every return. - expiredRequests, deadlineAt := pipelineRequestDeadlines(desiredRequests, pipelineRequestDeadlineSeconds(dgd)) + // Preserve the earliest deadline across workloads on every subsequent return. defer func() { result = requeueForPipelineRequestDeadline(deadlineAt, result, err) }() diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index c4370ef49214..7752d329c92f 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -335,7 +335,7 @@ func TestLPXWorkloadErrorDoesNotAcknowledgeGeneration(t *testing.T) { child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) child.Generation, child.Status.ObservedGeneration = 2, 1 if tc.deadline { - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](60)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](60)} } r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) @@ -406,11 +406,67 @@ func TestLPXWorkloadErrorDoesNotAcknowledgeGeneration(t *testing.T) { } } +func TestLPXSharedWorkloadComponentDeadlines(t *testing.T) { + for _, tc := range []struct { + name string + target, draft *int64 + wantSeconds int64 + wantFailed bool + }{ + {name: "target wakes first", target: ptr.To(int64(120)), draft: ptr.To(int64(300)), wantSeconds: 120}, + {name: "expanded drafts wake first", target: ptr.To(int64(300)), draft: ptr.To(int64(120)), wantSeconds: 120}, + {name: "unlimited target", draft: ptr.To(int64(120)), wantSeconds: 120}, + {name: "unlimited drafts", target: ptr.To(int64(120)), wantSeconds: 120}, + {name: "all unlimited"}, + {name: "only target expired", target: ptr.To(int64(30)), draft: ptr.To(int64(300)), wantFailed: true}, + {name: "only drafts expired", target: ptr.To(int64(300)), draft: ptr.To(int64(30)), wantFailed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Publish target and two draft requests with the same scheduling start") + child, dgd, registry := newLPXSpecDecodeTestDGD(t) + r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) + objects := lpxMaterializedObjects(t, r, child, dgd, selected) + createLPXTestObjects(t, t.Context(), r.Client, objects...) + publishSelectedLPXForTest(t, t.Context(), r, child, selected) + pcs := findLPXTestPodCliqueSet(t, objects) + started := time.Now().UTC().Truncate(time.Second).Add(-time.Minute) + requests, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.Len(t, requests, 3) + for _, request := range requests { + request.Status = newTestPipelineRequest(child, pcs, request.Name, started, lpxv1alpha1.RequestPhasePending).Status + require.NoError(t, r.Update(t.Context(), request)) + } + + t.Log("Edit component policies without replacing requests or resetting their clocks") + dgd.GetComponentByName("lpx").LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: tc.target} + dgd.GetComponentByName("draft").LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: tc.draft} + dgd.Spec.Components[0], dgd.Spec.Components[1] = dgd.Spec.Components[1], dgd.Spec.Components[0] + updateTestDGD(t, r, child, dgd) + key := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)} + result, err := r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + require.Equal(t, tc.wantFailed, meta.IsStatusConditionTrue(child.Status.Conditions, schedulingFailedCondition)) + if tc.wantFailed { + require.Positive(t, result.RequeueAfter) + } else if tc.wantSeconds != 0 { + require.WithinDuration(t, started.Add(time.Duration(tc.wantSeconds)*time.Second), time.Now().Add(result.RequeueAfter), time.Second) + } else { + require.Zero(t, result.RequeueAfter) + } + after, err := r.getPipelineRequests(t.Context(), pcs) + require.NoError(t, err) + require.True(t, apiequality.Semantic.DeepEqual(requests, after), "deadline edits must preserve requests and their scheduling clocks") + }) + } +} + func TestPipelineRequestDeadlineContinuesDuringRequestDeletion(t *testing.T) { t.Log("Publish two pending replicas and expire the tail first") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) @@ -447,7 +503,7 @@ func TestPipelineRequestDeadlineFailureMustPersistBeforeCleanup(t *testing.T) { t.Log("Publish an expired request without a persisted deadline failure") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -526,7 +582,7 @@ func TestLPXDeadlineWaitsForScalingGroup(t *testing.T) { t.Log("Publish an expired request and hide its scaling group from the cache") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, desired) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -578,7 +634,7 @@ func TestPipelineRequestDeadlineHoleWaitsForSchedulingChange(t *testing.T) { t.Log("Expire an interior workload while a higher ordinal is still Bound") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, desired := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, desired) @@ -738,7 +794,7 @@ func TestLPXEditBeforeDeadlineFailureDoesNotAuthorizeRetry(t *testing.T) { t.Log("Publish a request, then advance the deployment before that request expires") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -915,7 +971,7 @@ func TestLPXFailedScaleOutPreservesServingWorkloads(t *testing.T) { t.Log("Observe a completed workload and a newer pending request in their shared PCS") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) @@ -1025,7 +1081,7 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { t.Log("Publish four replicas with an expired interior request and a finalizer on the healthy tail") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(4)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) @@ -1124,7 +1180,7 @@ func TestLPXExternalScaleInAfterSchedulingFailure(t *testing.T) { t.Log("Publish four externally managed replicas with one expired request") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = nil r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) selected.plan.Replicas = 4 @@ -1329,7 +1385,7 @@ func TestLPXMissingRequestGetsANewIndependentDeadline(t *testing.T) { t.Log("Publish two requests and remove one without a terminal deadline decision") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) @@ -1663,7 +1719,7 @@ func TestLPXReconcileWaitsForMatchingDGDRevision(t *testing.T) { func TestLPXRequestListFailureUsesControllerBackoff(t *testing.T) { t.Log("A request-list failure prevents deriving any active deadline") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) createLPXTestObjects(t, t.Context(), r.Client, lpxMaterializedObjects(t, r, child, dgd, desired)...) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(child), child)) @@ -2066,7 +2122,7 @@ func TestLPXTerminatingGroveResourcesBlockPublication(t *testing.T) { t.Log("Publish a workload whose PCS and scaling group are both terminating") ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -2395,7 +2451,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) createLPXTestObjects(t, ctx, r.Client, lpxMaterializedObjects(t, r, child, dgd, selected)...) key := client.ObjectKeyFromObject(child) @@ -2834,7 +2890,7 @@ func observedLPXTestPodCliqueSet( func TestLPXEditBetweenFailureAndCleanupPreservesRetry(t *testing.T) { t.Log("Persist a deadline failure before the user edits the input to retry") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, t.Context(), r.Client, objects...) @@ -2851,7 +2907,7 @@ func TestLPXEditBetweenFailureAndCleanupPreservesRetry(t *testing.T) { require.True(t, isSchedulingFailedConditionCurrent(child)) t.Log("The edited generation uses the old failure to retire the already-expired cycle") - dgd.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To[int64](31) + dgd.Spec.Components[0].LPX.Scheduling.AttemptDeadlineSeconds = ptr.To[int64](31) updateTestDGD(t, r, child, dgd) _, err = r.Reconcile(t.Context(), req) require.NoError(t, err) @@ -2874,7 +2930,7 @@ func TestLPXCommittedDeadlineCleanupPreservesExternalCapacity(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) dgd.Spec.Components[0].Replicas = nil - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To[int64](30)} r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) createLPXTestObjects(t, ctx, r.Client, objects...) @@ -3225,16 +3281,19 @@ func TestIndependentLPXDeadlineCleanup(t *testing.T) { for _, tc := range []struct { name string firstExpired int64 + secondDeadline *int64 secondExternal bool secondHealthy bool wantFirst int32 wantSecond int32 wantRequests int }{ - {name: "interior failure leaves the other engine's suffix removable", firstExpired: 0, wantFirst: 3, wantSecond: 1, wantRequests: 4}, - {name: "healthy neighbor survives failure cleanup", firstExpired: 2, secondHealthy: true, wantFirst: 2, wantSecond: 2, wantRequests: 4}, - {name: "independent expired suffixes", firstExpired: 2, wantFirst: 2, wantSecond: 1, wantRequests: 3}, - {name: "external engine retains capacity ownership", firstExpired: 2, secondExternal: true, wantFirst: 2, wantSecond: 2, wantRequests: 3}, + {name: "interior failure leaves the other engine's suffix removable", firstExpired: 0, secondDeadline: ptr.To(int64(60)), wantFirst: 3, wantSecond: 1, wantRequests: 4}, + {name: "healthy neighbor survives failure cleanup", firstExpired: 2, secondHealthy: true, secondDeadline: ptr.To(int64(60)), wantFirst: 2, wantSecond: 2, wantRequests: 4}, + {name: "independent expired suffixes", firstExpired: 2, secondDeadline: ptr.To(int64(60)), wantFirst: 2, wantSecond: 1, wantRequests: 3}, + {name: "external engine retains capacity ownership", firstExpired: 2, secondExternal: true, secondDeadline: ptr.To(int64(60)), wantFirst: 2, wantSecond: 2, wantRequests: 3}, + {name: "pending neighbor has a longer deadline", firstExpired: 2, secondDeadline: ptr.To(int64(300)), wantFirst: 2, wantSecond: 2, wantRequests: 4}, + {name: "pending neighbor has no deadline", firstExpired: 2, wantFirst: 2, wantSecond: 2, wantRequests: 4}, } { t.Run(tc.name, func(t *testing.T) { t.Log("Publish two workloads with different replica axes and active scheduling clocks") @@ -3242,11 +3301,12 @@ func TestIndependentLPXDeadlineCleanup(t *testing.T) { dgd.Spec.Components[0].Replicas = ptr.To(int32(3)) second := dgd.Spec.Components[0].DeepCopy() second.ComponentName, second.Replicas = "timeout-engine", ptr.To(int32(2)) + second.LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: tc.secondDeadline} if tc.secondExternal { second.Replicas = nil } dgd.Spec.Components = append(dgd.Spec.Components, *second) - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(60))} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(60))} r := newLPXTestReconciler(t, registry, child, dgd) workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index b5f66a9105ca..b408d389e0f2 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -14,7 +14,6 @@ import ( "time" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" @@ -23,24 +22,17 @@ import ( // Keep transient errors from delaying an active scheduling deadline indefinitely. const pipelineRequestDeadlineRetryInterval = 5 * time.Second -// pipelineRequestDeadlineSeconds reads the optional scheduling timeout from a non-nil DGD. -func pipelineRequestDeadlineSeconds(dgd *v1beta1.DynamoGraphDeployment) *int64 { - if dgd.Spec.Scheduling == nil { - return nil - } - return dgd.Spec.Scheduling.AttemptDeadlineSeconds -} - -// pipelineRequestDeadlines examines already-selected, owned requests without I/O. +// pipelineRequestDeadlines examines one workload's selected, owned requests without I/O. +// secondsByModel supplies each model's component deadline; nil means unlimited. +// next carries the earliest deadline from any previously examined workloads. // Creation time bounds the initial wait for the scheduler. On reused requests, // the scheduler-supplied start takes precedence for later scheduling cycles. -func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineRequest, seconds *int64) (expired []*lpxv1alpha1.LPUPipelineRequest, next time.Time) { - if seconds == nil { - return nil, time.Time{} - } +func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineRequest, secondsByModel map[string]*int64, next time.Time) ([]*lpxv1alpha1.LPUPipelineRequest, time.Time) { + var expired []*lpxv1alpha1.LPUPipelineRequest now := time.Now() for _, request := range requests { - if !request.DeletionTimestamp.IsZero() || isPipelineRequestDeadlineExempt(request) { + seconds := secondsByModel[request.Annotations[pipelineRequestModelAnnotation]] + if seconds == nil || !request.DeletionTimestamp.IsZero() || isPipelineRequestDeadlineExempt(request) { continue } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 2f5796789063..3254f3b39f89 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -42,6 +42,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { lastPlanRevision int64 wantExpired []string wantWake time.Time + previousWake time.Time }{ {name: "disabled deadline", starts: []*metav1.Time{oldStart}}, {name: "no requests", seconds: ptr.To(int64(30))}, @@ -55,6 +56,9 @@ func TestPipelineRequestDeadlines(t *testing.T) { {name: "independent starts", starts: []*metav1.Time{laterStart, newStart}, seconds: ptr.To(int64(30)), wantWake: newStart.Add(30 * time.Second)}, {name: "expired and active cycles", starts: []*metav1.Time{oldStart, newStart}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}, wantWake: newStart.Add(30 * time.Second)}, {name: "surviving request starts again", starts: []*metav1.Time{newStart}, seconds: ptr.To(int64(30)), lastPlanRevision: 1, wantWake: newStart.Add(30 * time.Second)}, + {name: "preserve earlier workload deadline", starts: []*metav1.Time{newStart}, seconds: ptr.To(int64(30)), previousWake: now, wantWake: now}, + {name: "replace later workload deadline", starts: []*metav1.Time{newStart}, seconds: ptr.To(int64(30)), previousWake: laterStart.Time, wantWake: newStart.Add(30 * time.Second)}, + {name: "unlimited workload preserves prior deadline", starts: []*metav1.Time{oldStart}, previousWake: now, wantWake: now}, } { t.Run(tc.name, func(t *testing.T) { t.Log("Use creation time until the scheduler supplies a scheduling-cycle start") @@ -67,6 +71,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { SchedulingStartedAt: start, LastPlanRevision: tc.lastPlanRevision, }, } + request.Annotations = map[string]string{pipelineRequestModelAnnotation: "default"} if tc.created != nil { request.CreationTimestamp = *tc.created } @@ -80,7 +85,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { } t.Log("Return only expired requests and the earliest still-active deadline") - expired, wake := pipelineRequestDeadlines(requests, tc.seconds) + expired, wake := pipelineRequestDeadlines(requests, map[string]*int64{"default": tc.seconds}, tc.previousWake) var names []string for _, request := range expired { names = append(names, request.Name) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index b02af4de1a28..fc38fe918df6 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -128,7 +128,6 @@ var ErrLPXSourcePending = errors.New("LPXGraphDeployment is waiting for the curr // settings cannot silently fall out of the revision. type lpxInputRevisionPayload struct { Components []v1beta1.DynamoComponentDeploymentSharedSpec - Scheduling *v1beta1.SchedulingSpec Labels map[string]string Annotations map[string]string PropagatedAnnotations map[string]string @@ -179,7 +178,6 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin } input := lpxInputRevisionPayload{ Components: canonical, - Scheduling: dgd.Spec.Scheduling, Labels: dgd.Spec.Labels, Annotations: dgd.Spec.Annotations, PropagatedAnnotations: annotations, diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 3caf1554b1f4..7db395e94db0 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -245,7 +245,7 @@ func TestLPXInputRevision(t *testing.T) { require.Equal(t, pcsName, PCSNameForLPX(deployment)) t.Log("Each LPX component or shared-input change invalidates the revision; ordinary edits do not") - source.Spec.Scheduling = &v1beta1.SchedulingSpec{} + source.GetComponentByName("decode").LPX.Scheduling = &v1beta1.SchedulingSpec{} want, err = LPXInputRevision(source, "") require.NoError(t, err) for _, test := range []struct { @@ -298,7 +298,7 @@ func TestLPXInputRevision(t *testing.T) { d.GetComponentByName("decode").ComponentRole(v1beta1.ComponentRoleLPXConductor).PodTemplate.Annotations = map[string]string{"role": "new"} }}, {"scheduling/deadline", true, func(d *v1beta1.DynamoGraphDeployment) { - d.Spec.Scheduling.AttemptDeadlineSeconds = ptr.To(int64(60)) + d.GetComponentByName("decode").LPX.Scheduling.AttemptDeadlineSeconds = ptr.To(int64(60)) }}, {"shared/labels", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Labels = map[string]string{"workload": "new"} }}, {"shared/annotations", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Annotations = map[string]string{"workload": "new"} }}, diff --git a/deploy/operator/internal/dynamo/lpx/components_test.go b/deploy/operator/internal/dynamo/lpx/components_test.go index 79cc5aae1701..970737f597f3 100644 --- a/deploy/operator/internal/dynamo/lpx/components_test.go +++ b/deploy/operator/internal/dynamo/lpx/components_test.go @@ -54,7 +54,6 @@ func TestLPXSchedulerSelectionUsesComponents(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Annotations: test.annotations}, Spec: dynamov1beta1.DynamoGraphDeploymentSpec{ Components: test.components, - Scheduling: &dynamov1beta1.SchedulingSpec{}, }, } diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index 38932090422a..b11285b7371d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -79,6 +79,11 @@ func (p *ModelProjection) Model() string { return p.model } +// ComponentName returns the DGD component that supplies this model. +func (p *ModelProjection) ComponentName() string { + return p.stage +} + // RequestSpec returns a fresh node-local request for one Grove scaling-group // replica. The receiver and plan must be non-nil; neither input is mutated. func (p *ModelProjection) RequestSpec( diff --git a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go index 845954dcf800..5bb9cf9bdc62 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go @@ -136,7 +136,7 @@ func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { require.NotEqual(t, plan.Agents[0].CliqueName, replica.Agents[0].CliqueName) t.Log("A scheduling deadline and annotation do not change hybrid launch") - dgd.Spec.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} + dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} dgd.Annotations = map[string]string{commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend} scheduled, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index a7562b307728..ab330de244b7 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -96,6 +96,36 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { dgd.Spec.Components[0].Replicas = k8sptr.To(int32(10)) }), }, + // Scheduling deadlines belong to each component, including shared draft and target pairs. + { + name: "shared LPX components admit distinct scheduling deadlines at both bounds", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, nil) + dgd.Spec.Components[0].LPX.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(1))} + dgd.Spec.Components[1].LPX.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(9223372036))} + }), + }, + { + name: "shared LPX components admit an omitted scheduling deadline beside a finite deadline", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + setBetaLPXSpecDec(dgd, nil) + dgd.Spec.Components[1].LPX.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(120))} + }), + }, + { + name: "LPX component scheduling deadline rejects zero", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].LPX.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(0))} + }), + wantSchemaErr: "spec.components[0].lpx.scheduling.attemptDeadlineSeconds: Invalid value: 0: spec.components[0].lpx.scheduling.attemptDeadlineSeconds in body should be greater than or equal to 1", + }, + { + name: "LPX component scheduling deadline rejects duration overflow", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].LPX.Scheduling = &nvidiacomv1beta1.SchedulingSpec{AttemptDeadlineSeconds: k8sptr.To(int64(9223372037))} + }), + wantSchemaErr: "spec.components[0].lpx.scheduling.attemptDeadlineSeconds: Invalid value: 9223372037: spec.components[0].lpx.scheduling.attemptDeadlineSeconds in body should be less than or equal to 9223372036", + }, { name: "LPX autoscaling is rejected by the schema", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index b4f539b1d270..0d2cc05ea514 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -607,7 +607,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | ProviderOverride configures the root resource generated by the selected
graph-level workload provider. For Grove it uses apiVersion
`grove.io/v1alpha1`, target `PodCliqueSet`, and may set only
`spec.template.topologyConstraint`. It cannot select or change the provider. | | Optional: \{\}
| -| `scheduling` _[SchedulingSpec](#schedulingspec)_ | Scheduling configures attempts for LPX components. Omission means no deadline. | | Optional: \{\}
| | `annotations` _object (keys:string, values:string)_ | Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates).
Service-level annotations take precedence over these values. | | Optional: \{\}
| | `labels` _object (keys:string, values:string)_ | Labels to propagate to all child resources (PCS, DCD, Deployments, and pod templates).
Service-level labels take precedence over these values. | | Optional: \{\}
| | `priorityClassName` _string_ | PriorityClassName is the name of the PriorityClass to use for Grove PodCliqueSets.
Requires the Grove pathway. | | Optional: \{\}
| @@ -2169,7 +2168,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the root resource generated by the selected
graph-level workload provider. For Grove it uses apiVersion
`grove.io/v1alpha1`, target `PodCliqueSet`, and may set only
`spec.template.topologyConstraint`. It cannot select or change the provider. | | Optional: \{\}
| -| `scheduling` _[SchedulingSpec](#schedulingspec)_ | scheduling configures attempts for LPX components. Omission means no deadline. | | Optional: \{\}
| | `annotations` _object (keys:string, values:string)_ | annotations to propagate to all child resources (PCS, DCD, Deployments,
and pod templates). Component-level (`podTemplate`) values take precedence
on conflict. | | Optional: \{\}
| | `labels` _object (keys:string, values:string)_ | labels to propagate to all child resources. Same precedence rules as `annotations`. | | Optional: \{\}
| | `priorityClassName` _string_ | priorityClassName is the name of the PriorityClass to use for Grove PodCliqueSets.
Requires the Grove pathway. | | Optional: \{\}
| @@ -2488,7 +2486,7 @@ _Appears in:_ -LPXConfig identifies the component's compiled model. +LPXConfig identifies the component's compiled model and configures scheduling. @@ -2499,6 +2497,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `buildId` _string_ | buildId references the immutable model build. | | MinLength: 1
| +| `scheduling` _[SchedulingSpec](#schedulingspec)_ | scheduling configures this component's LPX scheduling attempts.
Omission means no deadline. | | Optional: \{\}
| #### MockerSpec @@ -2925,8 +2924,7 @@ SchedulingSpec configures LPX scheduling attempts. _Appears in:_ -- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec) -- [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) +- [LPXConfig](#lpxconfig) | Field | Description | Default | Validation | | --- | --- | --- | --- | diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 799b269ba98a..1be594409f6c 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -63,11 +63,29 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c - DGD-only compiled-model configuration. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). + DGD-only compiled-model and scheduling configuration. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). When the operator's `lpx.modelRegistryURL` is configured, `buildId` must be relative to that registry; absolute paths and URLs require an empty registry URL. + + + Configures this LPX component's scheduling attempts. Components may use different deadlines, including when they share a workload. + + + + + Best-effort deadline for each of this component's LPU pipeline requests to leave Pending, including waiting for the scheduler to start. The timer uses a nonzero `status.schedulingStartedAt` when available; otherwise it uses `metadata.creationTimestamp`. A scheduler-provided timestamp for a later scheduling cycle takes precedence over the request's original creation time. Valid values are `1` through `9,223,372,036`; omission means unlimited. This field is not a solver budget or an overall deployment readiness timeout. + + Current-generation `Bound`, `NoFit`, and `Unsupported` receipts are exempt. `Degraded`, `Releasing`, and `Released` receipts are also exempt when their committed execution was accepted for the current request generation. `NoFit` can therefore leave the deployment Pending indefinitely, and a `Bound` request can still wait indefinitely for runtime readiness. + + Deadline handling runs after download checks, compiler-registry resolution, and desired-state validation. An outage or invalid desired configuration can delay failure reporting and cleanup indefinitely, even after the deadline has elapsed. The timer does not bound asynchronous cleanup or finalization. + + Expiry records `LPXSchedulingDeadlineExceeded` before cleanup. Cleanup removes complete engine replicas only from a trailing suffix: an expired interior replica blocks deadline cleanup to preserve healthy higher ordinals. Explicit `replicas` authorizes lowering the Grove replica count to the surviving prefix; externally managed capacity remains unchanged. Grove scaling and owner garbage collection handle pod cleanup; the LPX graph controller does not delete pods directly. A later LPX input revision authorizes republication after cleanup. An existing failure remains cleanup authority for cycles that started before it, so retiring those cycles does not consume the retry edit. + + + + Declares the Dynamo runtime version in the component's `main` image, taking precedence over the image tag. Use canonical `MAJOR.MINOR.PATCH`, for example `"1.4.0"`, with each part between `0` and `9999` and no leading zeros, `v` prefix, prerelease suffix, or build metadata. This field does not change the image. See [Runtime Version Compatibility](#runtime-version-compatibility) for admission requirements and rollout behavior. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index f8222ef56732..98efa2e2046d 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -85,22 +85,6 @@ spec: Name of the `PriorityClass` to use for Grove `PodCliqueSet`s. Requires the Grove pathway. See [Multinode Orchestration](../../kubernetes/installation/multinode-orchestration.md). - - Configures scheduling attempts for `type: lpx` components. Omit this object for attempts without a deadline. - - - - - Best-effort deadline for each LPU pipeline request to leave Pending, including waiting for the scheduler to start. The timer uses a nonzero `status.schedulingStartedAt` when available; otherwise it uses `metadata.creationTimestamp`. A scheduler-provided timestamp for a later scheduling cycle takes precedence over the request's original creation time. Valid values are `1` through `9,223,372,036`; omission means unlimited. This field is not a solver budget or an overall deployment readiness timeout. - - Current-generation `Bound`, `NoFit`, and `Unsupported` receipts are exempt. `Degraded`, `Releasing`, and `Released` receipts are also exempt when their committed execution was accepted for the current request generation. `NoFit` can therefore leave the deployment Pending indefinitely, and a `Bound` request can still wait indefinitely for runtime readiness. - - Deadline handling runs after download checks, compiler-registry resolution, and desired-state validation. An outage or invalid desired configuration can delay failure reporting and cleanup indefinitely, even after the deadline has elapsed. The timer does not bound asynchronous cleanup or finalization. - - Expiry records `LPXSchedulingDeadlineExceeded` before cleanup. Cleanup removes complete engine replicas only from a trailing suffix: an expired interior replica blocks deadline cleanup to preserve healthy higher ordinals. Explicit `replicas` authorizes lowering the Grove replica count to the surviving prefix; externally managed capacity remains unchanged. Grove scaling and owner garbage collection handle pod cleanup; the LPX graph controller does not delete pods directly. A later LPX input revision authorizes republication after cleanup. An existing failure remains cleanup authority for cycles that started before it, so retiring those cycles does not consume the retry edit. - - - Restart policy for the graph. Must be unset on creation; set or change `restart.id` on an existing DGD to trigger a restart. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index f25d7f32f9c8..b9227108ffc1 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -988,12 +988,6 @@ See [ProviderOverride](#provideroverride). **Validation:** Optional: \{\} - -Scheduling configures attempts for LPX components. Omission means no deadline. -See [SchedulingSpec](#schedulingspec). -**Validation:** Optional: \{\} - - Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). Service-level annotations take precedence over these values. **Validation:** Optional: \{\} @@ -3188,12 +3182,6 @@ See [ProviderOverride](#v1beta1-provideroverride). **Validation:** Optional: \{\} - -scheduling configures attempts for LPX components. Omission means no deadline. -See [SchedulingSpec](#schedulingspec). -**Validation:** Optional: \{\} - - annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). Component-level (`podTemplate`) values take precedence on conflict. **Validation:** Optional: \{\} @@ -3634,7 +3622,7 @@ preferredWeight is required and used only when enforcement is "preferred". Highe -LPXConfig identifies the component's compiled model. +LPXConfig identifies the component's compiled model and configures scheduling. **Kind:** `type` @@ -3644,6 +3632,12 @@ LPXConfig identifies the component's compiled model. buildId references the immutable model build. **Validation:** MinLength: 1 + + +scheduling configures this component's LPX scheduling attempts. Omission means no deadline. +See [SchedulingSpec](#schedulingspec). +**Validation:** Optional: \{\} + @@ -4082,7 +4076,7 @@ SchedulingSpec configures LPX scheduling attempts. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentSpec](#dynamographdeploymentspec), [DynamoGraphDeploymentSpec](#v1beta1-dynamographdeploymentspec) +**Appears in:** [LPXConfig](#lpxconfig) attemptDeadlineSeconds limits how long each LPR may remain pending. Omission means unlimited; the value is not a solver budget. diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index 37aeed510e22..31bb8ac76946 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -423,7 +423,6 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( "name,parent", ( ("LPXConfig", "DynamoComponentDeploymentSharedSpec"), - ("SchedulingSpec", "DynamoGraphDeploymentSpec"), ), ) def test_shared_lpx_type_links_across_api_versions( From 63031acf081a551495beb66d769b22c8d539325d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 16:43:11 +0000 Subject: [PATCH 120/196] style(docs): format LPX API reference test Signed-off-by: Andrew Paprotsky --- docs/fern/scripts/tests/test_gen_kubernetes_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index 31bb8ac76946..9c87c630c1dd 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -421,9 +421,7 @@ def test_raw_reference_omits_dgd_only_fields_from_standalone_dcd_docs( @pytest.mark.parametrize( "name,parent", - ( - ("LPXConfig", "DynamoComponentDeploymentSharedSpec"), - ), + (("LPXConfig", "DynamoComponentDeploymentSharedSpec"),), ) def test_shared_lpx_type_links_across_api_versions( source_text: str, From ea355e382d0b8a671be2a3f6a735041d17e4390b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 16:48:52 +0000 Subject: [PATCH 121/196] fix(operator): keep component readiness on LPX deployments Signed-off-by: Andrew Paprotsky --- .../conversion_field_coverage_test.go | 2 - ...namocomponentdeployment_conversion_test.go | 1 - .../dynamographdeployment_conversion.go | 2 - .../dynamographdeployment_conversion_test.go | 1 - .../v1alpha1/dynamographdeployment_types.go | 5 -- .../api/v1alpha1/lpxgraphdeployment_types.go | 13 +++- .../api/v1alpha1/zz_generated.deepcopy.go | 25 ++++++- deploy/operator/api/v1beta1/common.go | 5 -- ...nvidia.com_dynamocomponentdeployments.yaml | 10 --- .../nvidia.com_dynamographdeployments.yaml | 10 --- .../bases/nvidia.com_lpxgraphdeployments.yaml | 66 ++++++++++++++++-- deploy/operator/docs/crd-ref-docs-config.yaml | 1 + .../controller/dgd_grove_lpx_restart.go | 16 +++-- .../controller/dgd_grove_lpx_restart_test.go | 39 ++++++++++- .../controller/dgd_grove_lpx_status.go | 2 +- .../controller/dgd_grove_lpx_status_test.go | 20 ++++-- .../dgd_lpx_handoff_envtest_test.go | 9 ++- .../controller/dgd_lpx_handoff_test.go | 49 +++++++++---- .../dynamocomponentdeployment_controller.go | 2 - ...namocomponentdeployment_controller_test.go | 3 - .../dynamographdeployment_controller_test.go | 5 -- .../internal/controller/lpx/controller.go | 22 ++++-- .../controller/lpx/controller_test.go | 69 ++++++++++++++++--- .../internal/controller/lpx/status.go | 41 ++++++++--- .../internal/controller/lpx/status_test.go | 32 +++++++-- deploy/operator/internal/dynamo/grove.go | 1 - deploy/operator/internal/dynamo/grove_lpx.go | 5 -- deploy/operator/internal/dynamo/grove_test.go | 4 -- .../additional-resources/api-reference-k8s.md | 2 - .../kubernetes-api/full-api-reference.mdx | 10 --- 30 files changed, 335 insertions(+), 137 deletions(-) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 9c37dd302894..b7559150f633 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -89,7 +89,6 @@ DynamoComponentDeploymentStatus.component.componentKind DynamoComponentDeploymentStatus.component.componentNames DynamoComponentDeploymentStatus.component.gpusPerEngine DynamoComponentDeploymentStatus.component.gpusPerReplica -DynamoComponentDeploymentStatus.component.ready DynamoComponentDeploymentStatus.component.readyReplicas DynamoComponentDeploymentStatus.component.replicas DynamoComponentDeploymentStatus.component.runtimeNamespace @@ -183,7 +182,6 @@ DynamoGraphDeploymentStatus.components.componentKind DynamoGraphDeploymentStatus.components.componentNames DynamoGraphDeploymentStatus.components.gpusPerEngine DynamoGraphDeploymentStatus.components.gpusPerReplica -DynamoGraphDeploymentStatus.components.ready DynamoGraphDeploymentStatus.components.readyReplicas DynamoGraphDeploymentStatus.components.replicas DynamoGraphDeploymentStatus.components.runtimeNamespace diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go index ef7cb855e980..7c43dc144d1e 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_conversion_test.go @@ -804,7 +804,6 @@ func TestDCD_RoundTrip_Status(t *testing.T) { ComponentNames: []string{"dcd-0"}, GPUsPerEngine: ptr.To(int64(2)), GPUsPerReplica: ptr.To(int64(3)), - Ready: true, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 527433f9fabf..4a46a4b110c2 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -810,7 +810,6 @@ func ConvertFromServiceReplicaStatus(src *ServiceReplicaStatus, dst *v1beta1.Com ComponentKind: v1beta1.ComponentKind(src.ComponentKind), ComponentNames: componentNamesToHub(src), RuntimeNamespace: src.RuntimeNamespace, - Ready: src.Ready, Replicas: src.Replicas, UpdatedReplicas: src.UpdatedReplicas, } @@ -840,7 +839,6 @@ func ConvertToServiceReplicaStatus(src *v1beta1.ComponentReplicaStatus, dst *Ser ComponentKind: ComponentKind(src.ComponentKind), ComponentNames: componentNames, RuntimeNamespace: src.RuntimeNamespace, - Ready: src.Ready, Replicas: src.Replicas, UpdatedReplicas: src.UpdatedReplicas, } diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index 5749987055f5..79a22f92399d 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -978,7 +978,6 @@ func TestDGD_RoundTrip_Status(t *testing.T) { RuntimeNamespace: "ns-status-worker-abc123", GPUsPerEngine: ptr.To(int64(2)), GPUsPerReplica: ptr.To(int64(3)), - Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 4a43198dc78e..162cff121dfc 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -324,11 +324,6 @@ type ServiceReplicaStatus struct { // +kubebuilder:validation:Minimum=0 GPUsPerReplica *int64 `json:"gpusPerReplica,omitempty"` - // Ready is the binary determination of whether the correct number of replicas - // are scheduled and available. - // +optional - Ready bool `json:"ready"` - // Replicas is the total number of non-terminated replicas. // Required for all component kinds. // +kubebuilder:validation:Minimum=0 diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index 70462ceaac42..716d5599e8f9 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -33,6 +33,17 @@ type LPXGraphDeploymentSpec struct { InputRevision string `json:"inputRevision"` } +// LPXComponentStatus reports logical replicas and readiness for one LPX component. +type LPXComponentStatus struct { + v1beta1.ComponentReplicaStatus `json:",inline"` + + // conditions reports component readiness for the current child generation. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + // LPXGraphDeploymentStatus is the LPX controller's durable lifecycle state. // Dynamo consumes readiness only for the current generation and input revision. type LPXGraphDeploymentStatus struct { @@ -49,7 +60,7 @@ type LPXGraphDeploymentStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` // components reports logical replicas by authored component name, never Agent Pods. // +optional - Components map[string]v1beta1.ComponentReplicaStatus `json:"components,omitempty"` + Components map[string]LPXComponentStatus `json:"components,omitempty"` // modelDownload retains the existing remote-build download progress. // +optional ModelDownload *v1beta1.ModelDownloadStatus `json:"modelDownload,omitempty"` diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index a81ec03a4e6c..874dd93654ab 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1358,6 +1358,29 @@ func (in *KvTransferPolicy) DeepCopy() *KvTransferPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LPXComponentStatus) DeepCopyInto(out *LPXComponentStatus) { + *out = *in + in.ComponentReplicaStatus.DeepCopyInto(&out.ComponentReplicaStatus) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LPXComponentStatus. +func (in *LPXComponentStatus) DeepCopy() *LPXComponentStatus { + if in == nil { + return nil + } + out := new(LPXComponentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LPXGraphDeployment) DeepCopyInto(out *LPXGraphDeployment) { *out = *in @@ -1444,7 +1467,7 @@ func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) } if in.Components != nil { in, out := &in.Components, &out.Components - *out = make(map[string]v1beta1.ComponentReplicaStatus, len(*in)) + *out = make(map[string]LPXComponentStatus, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index a592501cd9df..c6c7e733539d 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -842,11 +842,6 @@ type ComponentReplicaStatus struct { // +kubebuilder:validation:Minimum=0 GPUsPerReplica *int64 `json:"gpusPerReplica,omitempty"` - // ready is the binary determination of whether the correct number of replicas - // are scheduled and available. - // +optional - Ready bool `json:"ready"` - // replicas is the total number of non-terminated replicas. // +kubebuilder:validation:Minimum=0 Replicas int32 `json:"replicas"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 22d1eb271d3f..c279bc5ee44a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -19849,11 +19849,6 @@ spec: format: int64 minimum: 0 type: integer - ready: - description: |- - Ready is the binary determination of whether the correct number of replicas - are scheduled and available. - type: boolean readyReplicas: description: |- ReadyReplicas is the number of ready replicas. @@ -36939,11 +36934,6 @@ spec: format: int64 minimum: 0 type: integer - ready: - description: |- - ready is the binary determination of whether the correct number of replicas - are scheduled and available. - type: boolean readyReplicas: description: |- readyReplicas is the number of ready replicas. Populated for diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 47575aa631bd..9a593f30b87c 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20387,11 +20387,6 @@ spec: format: int64 minimum: 0 type: integer - ready: - description: |- - Ready is the binary determination of whether the correct number of replicas - are scheduled and available. - type: boolean readyReplicas: description: |- ReadyReplicas is the number of ready replicas. @@ -37965,11 +37960,6 @@ spec: format: int64 minimum: 0 type: integer - ready: - description: |- - ready is the binary determination of whether the correct number of replicas - are scheduled and available. - type: boolean readyReplicas: description: |- readyReplicas is the number of ready replicas. Populated for diff --git a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml index 6430f0cd5c2e..c3e699e0d4b1 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_lpxgraphdeployments.yaml @@ -74,7 +74,7 @@ spec: properties: components: additionalProperties: - description: ComponentReplicaStatus contains replica information for a single component. + description: LPXComponentStatus reports logical replicas and readiness for one LPX component. properties: availableReplicas: description: |- @@ -102,6 +102,65 @@ spec: items: type: string type: array + conditions: + description: conditions reports component readiness for the current child generation. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map gpusPerEngine: description: |- gpusPerEngine is the number of GPUs assigned to one inference engine in a @@ -122,11 +181,6 @@ spec: format: int64 minimum: 0 type: integer - ready: - description: |- - ready is the binary determination of whether the correct number of replicas - are scheduled and available. - type: boolean readyReplicas: description: |- readyReplicas is the number of ready replicas. Populated for diff --git a/deploy/operator/docs/crd-ref-docs-config.yaml b/deploy/operator/docs/crd-ref-docs-config.yaml index a39bfd6eec9b..121efa5353b5 100644 --- a/deploy/operator/docs/crd-ref-docs-config.yaml +++ b/deploy/operator/docs/crd-ref-docs-config.yaml @@ -37,6 +37,7 @@ processor: - "ConversionContext$" # LPXGraphDeployment is an operator-generated private child, not a public API. - "LPXGraphDeployment.*$" + - "LPXComponentStatus$" - "DynamoComponentDeploymentStatus$" - "BaseStatus$" - "ExcludedNamespacesChecker" diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index 9b265ed45dc4..b10a62f9ce37 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -28,12 +28,16 @@ func (r *lpxRestartProgressResolver) Resolve( source *v1beta1.DynamoGraphDeployment, inProgress []string, ) []string { - childComponents := r.observeRestart(ctx, source) + child := r.observeRestart(ctx, source) remaining := make([]string, 0, len(inProgress)) for _, name := range inProgress { - if !childComponents[name].Ready { - remaining = append(remaining, name) + if child != nil { + ready := meta.FindStatusCondition(child.Status.Components[name].Conditions, v1alpha1.LPXReadyCondition) + if ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == child.Generation { + continue + } } + remaining = append(remaining, name) } return remaining } @@ -84,12 +88,12 @@ func resolveCompositeGroveRestartProgress( return remaining } -// observeRestart returns component status from one current, ready restart observation. +// observeRestart returns the child for one current, ready restart observation. // A failed read or incomplete child leaves every requested member pending. func (r *lpxRestartProgressResolver) observeRestart( ctx context.Context, source *v1beta1.DynamoGraphDeployment, -) map[string]v1beta1.ComponentReplicaStatus { +) *v1alpha1.LPXGraphDeployment { child := &v1alpha1.LPXGraphDeployment{} if err := r.reader.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil || child.Status.ObservedGeneration != child.Generation || !child.DeletionTimestamp.IsZero() || @@ -110,5 +114,5 @@ func (r *lpxRestartProgressResolver) observeRestart( if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { return nil } - return child.Status.Components + return child } diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go index 3fdad84bb518..2a1b2e391c9c 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -28,7 +28,12 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { require.NoError(t, err) require.Equal(t, "restart-1", child.Annotations[dynamo.LPXRestartAnnotation]) child.Status.ObservedGeneration = child.Generation - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Ready: true}} + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "lpx": {Conditions: []metav1.Condition{{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, + ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, + }}}, + } meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, @@ -62,6 +67,38 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { ObservedGeneration: updated.Generation, Reason: v1alpha1.LPXReadyReasonReady, }) require.NoError(t, kube.Status().Update(ctx, updated)) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) + + t.Log("Keep the restart pending while component readiness is absent, unknown, false, or stale") + for _, scenario := range []struct { + status metav1.ConditionStatus + generation int64 + }{ + {status: "", generation: updated.Generation}, + {status: metav1.ConditionUnknown, generation: updated.Generation}, + {status: metav1.ConditionFalse, generation: updated.Generation}, + {status: metav1.ConditionTrue, generation: updated.Generation - 1}, + } { + component := v1alpha1.LPXComponentStatus{} + if scenario.status != "" { + component.Conditions = []metav1.Condition{{ + Type: v1alpha1.LPXReadyCondition, Status: scenario.status, + ObservedGeneration: scenario.generation, Reason: v1alpha1.LPXReadyReasonPending, + }} + } + updated.Status.Components["lpx"] = component + require.NoError(t, kube.Status().Update(ctx, updated)) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"}), "%+v", scenario) + } + + t.Log("Complete after the component reports Ready for the current child generation") + component := updated.Status.Components["lpx"] + meta.SetStatusCondition(&component.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, + ObservedGeneration: updated.Generation, Reason: v1alpha1.LPXReadyReasonReady, + }) + updated.Status.Components["lpx"] = component + require.NoError(t, kube.Status().Update(ctx, updated)) require.Empty(t, resolver.Resolve(ctx, dgd, []string{"lpx"})) restart = newDGDRestartReconciler().Resolve(ctx, dgd, &dgd.Status, resolver.Resolve) require.Equal(t, v1beta1.RestartPhaseCompleted, restart.Status.Phase) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 54266180004d..b973af15d9a7 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -38,7 +38,7 @@ func mergeLPXChildStatus( if observed { for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { - ordinary.ComponentStatus[component.ComponentName] = *observedStatus.DeepCopy() + ordinary.ComponentStatus[component.ComponentName] = *observedStatus.ComponentReplicaStatus.DeepCopy() } } } diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go index 38928cb0301c..a8584cce1116 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go @@ -10,6 +10,7 @@ import ( v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -39,11 +40,18 @@ func TestLPXStatusUsesOneCurrentChild(t *testing.T) { ModelDownload: &v1beta1.ModelDownloadStatus{ Builds: []string{"gs://models/first", "gs://models/second"}, LastCheckedAt: &checkedAt, }, - Components: map[string]v1beta1.ComponentReplicaStatus{}, + Components: map[string]v1alpha1.LPXComponentStatus{}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: 2}}, } for i, component := range source.Spec.Components { - child.Status.Components[component.ComponentName] = v1beta1.ComponentReplicaStatus{Ready: true, Replicas: int32(i + 1)} + replicas := int32(i + 1) + child.Status.Components[component.ComponentName] = v1alpha1.LPXComponentStatus{ + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{ + Replicas: replicas, UpdatedReplicas: replicas, ReadyReplicas: ptr.To(replicas), + AvailableReplicas: ptr.To(replicas), ScheduledReplicas: ptr.To(replicas), + }, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, + } } t.Log("Gate the graph on the shared child's current lifecycle") @@ -66,12 +74,14 @@ func TestLPXStatusUsesOneCurrentChild(t *testing.T) { result := mergeLPXChildStatus(source, child, ReconcileResult{State: v1beta1.DGDStateSuccessful}) require.Equal(t, want, result.State) - t.Log("Publish per-component readiness only for a current child") + t.Log("Publish per-component replica diagnostics only for a current child") if scenario == "ready" || scenario == "failure" || scenario == "no-download" { - require.Equal(t, before.Status.Components, result.ComponentStatus) + require.Len(t, result.ComponentStatus, len(before.Status.Components)) + for name, component := range before.Status.Components { + require.Equal(t, component.ComponentReplicaStatus, result.ComponentStatus[name]) + } } else { for _, component := range result.ComponentStatus { - require.False(t, component.Ready) require.Zero(t, component.Replicas) } } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 73066597428e..4ba411ea75b3 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -297,7 +297,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { source.Status.State = result.State require.NoError(t, env.Client().Status().Update(t.Context(), source)) require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) - require.False(t, source.Status.Components["lpx"].Ready) + require.Zero(t, source.Status.Components["lpx"].Replicas) t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") handoff := &dgdLPXHandoff{client: env.Client()} @@ -316,6 +316,12 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, LastTransitionTime: metav1.Now(), Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup}, + Conditions: []metav1.Condition{child.Status.Conditions[0]}, + }, + } failureTime := metav1.NewTime(time.Now().Add(-time.Minute).UTC().Truncate(time.Second)) meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ Type: "SchedulingFailed", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, @@ -323,6 +329,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { }) require.NoError(t, env.Client().Status().Update(t.Context(), child)) require.NoError(t, env.Client().Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, child.Status.Conditions[:1], child.Status.Components["lpx"].Conditions) require.Len(t, child.Status.Conditions, 2) schedulingFailed := meta.FindStatusCondition(child.Status.Conditions, "SchedulingFailed") require.NotNil(t, schedulingFailed) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 92acfd5a0fcb..cfec78519d7d 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -204,17 +204,19 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { t.Log("Project exactly one complete LPX engine alongside an ordinary component") child, source, _ := newLPXHandoffFixture(t, "node-local-v2-hybrid") child.Status.ObservedGeneration = child.Generation - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": { - Replicas: 3, AvailableReplicas: ptr.To(int32(3)), Ready: true, - GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{"lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{ + Replicas: 3, AvailableReplicas: ptr.To(int32(3)), + GPUsPerEngine: ptr.To(int64(8)), GPUsPerReplica: ptr.To(int64(8)), + }, }} source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) - result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4)), Ready: true}}} + result := ReconcileResult{State: v1beta1.DGDStateSuccessful, ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{"prefill": {Replicas: 4, AvailableReplicas: ptr.To(int32(4))}}} result = mergeLPXChildStatus(source, child, result) require.Equal(t, v1beta1.DGDStateSuccessful, result.State) require.Len(t, result.ComponentStatus, 2) - require.Equal(t, child.Status.Components["lpx"], result.ComponentStatus["lpx"]) + require.Equal(t, child.Status.Components["lpx"].ComponentReplicaStatus, result.ComponentStatus["lpx"]) t.Log("Reject missing or stale children and stale Ready conditions") for _, failure := range []string{"missing-child", "generation", "condition"} { @@ -233,14 +235,14 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { t.Log("A partially ready child retains logical replica counts and GPU capacity in public status") partial := child.Status.Components["lpx"] - partial.AvailableReplicas, partial.Ready = ptr.To(int32(2)), false + partial.AvailableReplicas = ptr.To(int32(2)) child.Status.Components["lpx"] = partial result.Reason = "CheckpointReady" meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonPending, Message: "Waiting for removed LPX requests"}) result = mergeLPXChildStatus(source, child, result) require.Equal(t, v1beta1.DGDStatePending, result.State) require.Equal(t, Reason(v1alpha1.LPXReadyReasonPending), result.Reason) - require.Equal(t, partial, result.ComponentStatus["lpx"]) + require.Equal(t, partial.ComponentReplicaStatus, result.ComponentStatus["lpx"]) source.Status.Components = result.ComponentStatus request := &v1beta1.DynamoGraphDeploymentRequest{} require.True(t, updateDeploymentInfo(request, source)) @@ -275,7 +277,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Log("Only a current child failure may bypass the complete-observation gate") child, source, _ := newLPXHandoffFixture(t, "node-local-v2-lpu-only") - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{"lpx": {ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 1}}} child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} result := ReconcileResult{State: v1beta1.DGDStateSuccessful} status := v1beta1.DynamoGraphDeploymentStatus{} @@ -355,7 +357,10 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { require.Equal(t, "restart-1", updated.Annotations[dynamo.LPXRestartAnnotation]) require.Equal(t, []string{"lpx"}, progress(t.Context(), source, []string{"lpx"})) updated.Status.ObservedGeneration = updated.Generation - updated.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Ready: true, Replicas: 1, AvailableReplicas: ptr.To(int32(1))}} + updated.Status.Components = map[string]v1alpha1.LPXComponentStatus{"lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 1, AvailableReplicas: ptr.To(int32(1))}, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: updated.Generation}}, + }} meta.SetStatusCondition(&updated.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: updated.Generation, Reason: "Ready"}) require.NoError(t, kube.Status().Update(t.Context(), updated)) @@ -398,9 +403,13 @@ func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend:0"}}}}, }) child.Status.ObservedGeneration = child.Generation - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ - "lpx": {Replicas: 1, AvailableReplicas: ptr.To(int32(1)), Ready: true}, - "draft": {Replicas: 2, AvailableReplicas: ptr.To(int32(2)), Ready: true}, + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 1, AvailableReplicas: ptr.To(int32(1))}, + }, + "draft": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 2, AvailableReplicas: ptr.To(int32(2))}, + }, } meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) require.NoError(t, kube.Status().Update(t.Context(), child)) @@ -498,8 +507,15 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { require.Equal(t, "pair-restart", child.Annotations[dynamo.LPXRestartAnnotation]) require.NotEmpty(t, progress(t.Context(), source, source.Status.Restart.InProgress)) child.Status.ObservedGeneration = child.Generation - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ - "lpx": {Ready: true, Replicas: 1}, "draft": {Ready: true, Replicas: 2}, + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 1}, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, + }, + "draft": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 2}, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, + }, } meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready"}) require.NoError(t, kube.Status().Update(t.Context(), child)) @@ -538,7 +554,10 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, ) ordinaryDGD = projectOrdinaryGroveDeployment(source) - child.Status.Components["draft"] = v1beta1.ComponentReplicaStatus{Ready: true, Replicas: 2} + child.Status.Components["draft"] = v1alpha1.LPXComponentStatus{ + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 2}, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, + } require.NoError(t, kube.Status().Update(t.Context(), child)) requested := []string{"draft", "frontend", "lpx", "prefill", "removed"} beforeSource := source.DeepCopy() diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go index 69db5a2965c7..52d1f9002849 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go @@ -346,7 +346,6 @@ func (r *DynamoComponentDeploymentReconciler) reconcileDeploymentResources(ctx c ComponentKind: nvidiacomv1beta1.ComponentKindDeployment, ComponentNames: []string{deployment.Name}, RuntimeNamespace: dynamo.GetDCDRuntimeNamespace(dynamoComponentDeployment), - Ready: ready, Replicas: deployment.Status.Replicas, UpdatedReplicas: deployment.Status.UpdatedReplicas, ReadyReplicas: &deployment.Status.ReadyReplicas, @@ -561,7 +560,6 @@ func getLeaderWorkerSetReplicasStatus(leaderWorkerSet *leaderworkersetv1.LeaderW return nvidiacomv1beta1.ComponentReplicaStatus{ ComponentKind: nvidiacomv1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{leaderWorkerSet.Name}, - Ready: IsLeaderWorkerSetReady(leaderWorkerSet), Replicas: leaderWorkerSet.Status.Replicas, UpdatedReplicas: leaderWorkerSet.Status.UpdatedReplicas, ReadyReplicas: &leaderWorkerSet.Status.ReadyReplicas, diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go index 86cfaf1ff4f0..31e7dec4744d 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go @@ -2211,7 +2211,6 @@ func Test_reconcileLeaderWorkerSetResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{"test-component-0"}, - Ready: true, ReadyReplicas: ptr.To(int32(1)), UpdatedReplicas: 1, Replicas: 1, @@ -2290,7 +2289,6 @@ func Test_reconcileLeaderWorkerSetResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindLeaderWorkerSet, ComponentNames: []string{"test-component-0"}, - Ready: true, ReadyReplicas: ptr.To(int32(3)), UpdatedReplicas: 3, Replicas: 3, @@ -2587,7 +2585,6 @@ func Test_reconcileDeploymentResources(t *testing.T) { serviceReplicaStatus: &v1beta1.ComponentReplicaStatus{ ComponentKind: v1beta1.ComponentKindDeployment, ComponentNames: []string{"test-component"}, - Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 33f78bd0330c..6ffcbf78e69c 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -865,7 +865,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, - Ready: true, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), @@ -933,7 +932,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, - Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -1019,7 +1017,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "decode": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-decode"}, - Ready: true, Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: ptr.To(int32(1)), @@ -1029,7 +1026,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, - Ready: true, Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: ptr.To(int32(1)), @@ -1103,7 +1099,6 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, - Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 2d6bcd4ed20a..6f94d847738e 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -117,7 +117,7 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu }(deployment.Status.DeepCopy()) for name, component := range deployment.Status.Components { - component.Ready = false + component.Conditions = nil deployment.Status.Components[name] = component } @@ -423,7 +423,7 @@ func (r *graphReconciler) reconcileReadiness( requests map[string]*lpxv1alpha1.LPUPipelineRequest, ) ctrl.Result { readiness := dynamo.GroveReadiness{Ready: true} - deployment.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) + deployment.Status.Components = make(map[string]v1alpha1.LPXComponentStatus) componentGroups := lpx.ComponentGroups(dgd) @@ -432,14 +432,28 @@ func (r *graphReconciler) reconcileReadiness( observed := dynamo.EvaluateLPXGroveReadiness(ctx, dgd, groupName, componentGroups[groupName], pcs, pcsgs[plan.LPXScalingGroup], pclqs) - maps.Copy(deployment.Status.Components, observed.ComponentStatuses) + state := v1beta1.DGDStatePending + if observed.Ready { + state = v1beta1.DGDStateSuccessful + } + conditions := []metav1.Condition{readyCondition(deployment.Generation, state, observed.Message)} + groupRequests := make(map[string]*lpxv1alpha1.LPUPipelineRequest) + for name, request := range requests { + if request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name == plan.LPXScalingGroup { + groupRequests[name] = request + } + } + setPipelineRequestReadyCondition(&conditions, deployment.Generation, groupRequests) + for name, status := range observed.ComponentStatuses { + deployment.Status.Components[name] = v1alpha1.LPXComponentStatus{ComponentReplicaStatus: status, Conditions: conditions} + } if !observed.Ready && readiness.Ready { readiness = observed } } - if !setPipelineRequestReadyCondition(deployment, requests) { + if !setPipelineRequestReadyCondition(&deployment.Status.Conditions, deployment.Generation, requests) { return ctrl.Result{} } diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 7752d329c92f..960505766155 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -134,8 +134,8 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) - require.False(t, child.Status.Components["lpx"].Ready) - require.True(t, child.Status.Components[second.ComponentName].Ready) + require.True(t, meta.IsStatusConditionFalse(child.Status.Components["lpx"].Conditions, v1alpha1.LPXReadyCondition)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Components[second.ComponentName].Conditions, v1alpha1.LPXReadyCondition)) t.Log("Preserve PCS identity and spec, both groups, all Agents, and the other workload's workers") for _, original := range objects { @@ -191,8 +191,8 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) - require.False(t, child.Status.Components["lpx"].Ready) - require.True(t, child.Status.Components[second.ComponentName].Ready) + require.True(t, meta.IsStatusConditionFalse(child.Status.Components["lpx"].Conditions, v1alpha1.LPXReadyCondition)) + require.True(t, meta.IsStatusConditionTrue(child.Status.Components[second.ComponentName].Conditions, v1alpha1.LPXReadyCondition)) observedAgent := &grovev1alpha1.PodClique{} require.NoError(t, r.Get(t.Context(), agentKey, observedAgent)) require.Equal(t, agent, observedAgent, "reconciliation must preserve user-managed Agent capacity") @@ -1918,7 +1918,7 @@ func TestLPXScalingWaitsForObservedCapacity(t *testing.T) { require.Equal(t, tc.wantGroupUpdates, groupUpdates) require.Equal(t, 2, cliqueUpdates) require.NoError(t, r.Get(ctx, key.NamespacedName, child)) - require.False(t, child.Status.Components["lpx"].Ready) + require.True(t, meta.IsStatusConditionFalse(child.Status.Components["lpx"].Conditions, v1alpha1.LPXReadyCondition)) require.Equal(t, ptr.To(int32(0)), child.Status.Components["lpx"].AvailableReplicas) t.Log("Report readiness only after Grove observes the new generation and all requested replicas") @@ -2238,8 +2238,6 @@ func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { require.Equal(t, ptr.To(int32(1)), draft.ReadyReplicas) require.Equal(t, ptr.To(int32(0)), target.AvailableReplicas) } - require.Equal(t, readiness.Ready, draft.Ready) - require.Equal(t, readiness.Ready, target.Ready) }) } } @@ -3063,6 +3061,10 @@ func TestLPXWaitsForAllScalingGroups(t *testing.T) { second := dgd.Spec.Components[0].DeepCopy() second.ComponentName, second.Replicas = "external", nil dgd.Spec.Components = append(dgd.Spec.Components, *second) + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "lpx": {Conditions: []metav1.Condition{readyCondition(child.Generation, v1beta1.DGDStateSuccessful, "Previous observation")}}, + "external": {Conditions: []metav1.Condition{readyCondition(child.Generation, v1beta1.DGDStateSuccessful, "Previous observation")}}, + } r := newLPXTestReconciler(t, registry, child, dgd) workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) @@ -3106,6 +3108,15 @@ func TestLPXWaitsForAllScalingGroups(t *testing.T) { require.NotNil(t, ready) require.Equal(t, v1alpha1.LPXReadyReasonPending, ready.Reason) require.Equal(t, "Waiting for all LPX scaling groups", ready.Message) + require.Len(t, child.Status.Components, 2) + for _, component := range child.Status.Components { + condition := meta.FindStatusCondition(component.Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionUnknown, condition.Status) + require.Equal(t, "NotObserved", condition.Reason) + require.Equal(t, "Component readiness has not been observed", condition.Message) + require.Equal(t, child.Generation, condition.ObservedGeneration) + } require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) require.Equal(t, before, pcs) requests, err := r.getPipelineRequests(t.Context(), pcs) @@ -3214,7 +3225,10 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) for groupName, plan := range plans { status := child.Status.Components[workloads[groupName].ServingComponentName()] - require.True(t, status.Ready) + require.True(t, meta.IsStatusConditionTrue(status.Conditions, v1alpha1.LPXReadyCondition)) + ready := meta.FindStatusCondition(status.Conditions, v1alpha1.LPXReadyCondition) + require.Equal(t, v1alpha1.LPXReadyReasonReady, ready.Reason) + require.Equal(t, child.Generation, ready.ObservedGeneration) require.Equal(t, plan.Replicas, status.Replicas) require.Equal(t, []string{plan.LPXScalingGroup}, status.ComponentNames) service := &corev1.Service{} @@ -3223,6 +3237,41 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { require.Equal(t, consts.KubeLabelValueTrue, service.Spec.Selector[dynamo.LPXServingLabel]) } + t.Log("Scheduler progress and failure affect only their own workload, preserving its healthy neighbor") + healthy := child.Status.Components["lpx"] + bound := meta.FindStatusCondition(child.Status.Components["second-engine"].Conditions, v1alpha1.LPXReadyCondition).DeepCopy() + var secondRequest *lpxv1alpha1.LPUPipelineRequest + for _, request := range requests { + if target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef; target.Name == secondPlan.LPXScalingGroup && target.ReplicaIndex == 0 { + secondRequest = request + } + } + require.NotNil(t, secondRequest) + for _, observation := range []struct { + phase lpxv1alpha1.RequestPhase + status metav1.ConditionStatus + reason string + message string + }{ + {lpxv1alpha1.RequestPhasePending, metav1.ConditionFalse, v1alpha1.LPXReadyReasonPending, "LPX scheduler is waiting to plan the current request"}, + {lpxv1alpha1.RequestPhaseUnsupported, metav1.ConditionFalse, v1alpha1.LPXReadyReasonFailed, "LPX scheduler cannot support the current request"}, + {lpxv1alpha1.RequestPhaseBound, metav1.ConditionTrue, v1alpha1.LPXReadyReasonReady, bound.Message}, + } { + secondRequest.Status.Phase = observation.phase + require.NoError(t, r.Update(t.Context(), secondRequest)) + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) + condition := meta.FindStatusCondition(child.Status.Components["second-engine"].Conditions, v1alpha1.LPXReadyCondition) + require.NotNil(t, condition) + require.Equal(t, observation.status, condition.Status) + require.Equal(t, observation.reason, condition.Reason) + require.Equal(t, observation.message, condition.Message) + require.Equal(t, child.Generation, condition.ObservedGeneration) + require.Equal(t, healthy, child.Status.Components["lpx"]) + require.Equal(t, observation.status, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Status) + } + t.Log("A missing role in the second workload leaves the first workload's readiness intact") clique := &grovev1alpha1.PodClique{} require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: secondPlan.ConductorClique}, clique)) @@ -3230,8 +3279,8 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) - require.True(t, child.Status.Components["lpx"].Ready) - require.False(t, child.Status.Components["second-engine"].Ready) + require.True(t, meta.IsStatusConditionTrue(child.Status.Components["lpx"].Conditions, v1alpha1.LPXReadyCondition)) + require.True(t, meta.IsStatusConditionFalse(child.Status.Components["second-engine"].Conditions, v1alpha1.LPXReadyCondition)) require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) t.Log("Scale down the explicit workload without replacing the PCS or its neighbor's requests") diff --git a/deploy/operator/internal/controller/lpx/status.go b/deploy/operator/internal/controller/lpx/status.go index d1e4ddfb252c..17d40cf1829b 100644 --- a/deploy/operator/internal/controller/lpx/status.go +++ b/deploy/operator/internal/controller/lpx/status.go @@ -28,6 +28,21 @@ const ( // updateStatus persists the updated status on the deployment. func (r *graphReconciler) updateStatus(ctx context.Context, deployment *v1alpha1.LPXGraphDeployment, previous *v1alpha1.LPXGraphDeploymentStatus) error { + for name, component := range deployment.Status.Components { + ready := meta.FindStatusCondition(component.Conditions, v1alpha1.LPXReadyCondition) + if ready == nil { + ready = &metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionUnknown, + ObservedGeneration: deployment.Generation, Reason: "NotObserved", + Message: "Component readiness has not been observed", + } + } + // Compare the final observation to persisted history, not intermediate states. + component.Conditions = slices.Clone(previous.Components[name].Conditions) + meta.SetStatusCondition(&component.Conditions, *ready) + deployment.Status.Components[name] = component + } + // Persist the complete observation without overwriting per-workload readiness. if apiequality.Semantic.DeepEqual(previous, &deployment.Status) { return nil @@ -42,6 +57,10 @@ func (r *graphReconciler) updateStatus(ctx context.Context, deployment *v1alpha1 // setReadyCondition updates the deployment's Ready condition. func setReadyCondition(deployment *v1alpha1.LPXGraphDeployment, state v1beta1.DGDState, message string) { + meta.SetStatusCondition(&deployment.Status.Conditions, readyCondition(deployment.Generation, state, message)) +} + +func readyCondition(generation int64, state v1beta1.DGDState, message string) metav1.Condition { status, reason := metav1.ConditionFalse, v1alpha1.LPXReadyReasonPending switch state { case v1beta1.DGDStateSuccessful: @@ -49,16 +68,16 @@ func setReadyCondition(deployment *v1alpha1.LPXGraphDeployment, state v1beta1.DG case v1beta1.DGDStateFailed: reason = v1alpha1.LPXReadyReasonFailed } - meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ - Type: v1alpha1.LPXReadyCondition, Status: status, ObservedGeneration: deployment.Generation, + return metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: status, ObservedGeneration: generation, Reason: reason, Message: truncateConditionMessage(message), - }) + } } // setPipelineRequestReadyCondition reports the first current failure, otherwise the first // non-Bound request, and returns whether every desired request is Bound. -// deployment is non-nil; absent requests and stale receipts are still pending. -func setPipelineRequestReadyCondition(deployment *v1alpha1.LPXGraphDeployment, requests map[string]*lpxv1alpha1.LPUPipelineRequest) bool { +// Absent receipts and stale receipts are still pending. +func setPipelineRequestReadyCondition(conditions *[]metav1.Condition, generation int64, requests map[string]*lpxv1alpha1.LPUPipelineRequest) bool { // Failures outrank pending receipts; stable name order keeps diagnostics consistent. allBound := true var pending *lpxv1alpha1.LPUPipelineRequestStatus @@ -80,7 +99,7 @@ func setPipelineRequestReadyCondition(deployment *v1alpha1.LPXGraphDeployment, r lpxv1alpha1.RequestPhaseReleasing, lpxv1alpha1.RequestPhaseReleased: // Keep the first pending receipt while checking the remaining requests for failure. default: - setPipelineRequestPhaseCondition(deployment, status) + setPipelineRequestPhaseCondition(conditions, generation, status) return false } } @@ -92,17 +111,17 @@ func setPipelineRequestReadyCondition(deployment *v1alpha1.LPXGraphDeployment, r } if !allBound { - setPipelineRequestPhaseCondition(deployment, pending) + setPipelineRequestPhaseCondition(conditions, generation, pending) } return allBound } // setPipelineRequestPhaseCondition formats a non-Bound receipt without copying it. -// deployment is non-nil; nil status means no current scheduler receipt exists. -func setPipelineRequestPhaseCondition(deployment *v1alpha1.LPXGraphDeployment, status *lpxv1alpha1.LPUPipelineRequestStatus) { +// Nil status means no current scheduler receipt exists. +func setPipelineRequestPhaseCondition(conditions *[]metav1.Condition, generation int64, status *lpxv1alpha1.LPUPipelineRequestStatus) { // A missing or stale receipt cannot establish scheduler progress. if status == nil { - setReadyCondition(deployment, v1beta1.DGDStatePending, "LPUPipelineRequest is published") + meta.SetStatusCondition(conditions, readyCondition(generation, v1beta1.DGDStatePending, "LPUPipelineRequest is published")) return } @@ -137,7 +156,7 @@ func setPipelineRequestPhaseCondition(deployment *v1alpha1.LPXGraphDeployment, s summary = fmt.Sprintf("LPX scheduler reported unknown phase %q", status.Phase) } - setReadyCondition(deployment, state, pipelineRequestDiagnosticMessage(summary, status.Diagnostics)) + meta.SetStatusCondition(conditions, readyCondition(generation, state, pipelineRequestDiagnosticMessage(summary, status.Diagnostics))) } // isSchedulingFailedConditionCurrent reports whether the current generation must remain diff --git a/deploy/operator/internal/controller/lpx/status_test.go b/deploy/operator/internal/controller/lpx/status_test.go index 9d10f4becb16..6c5e100e436e 100644 --- a/deploy/operator/internal/controller/lpx/status_test.go +++ b/deploy/operator/internal/controller/lpx/status_test.go @@ -20,6 +20,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/stretchr/testify/require" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -112,8 +113,8 @@ func TestLPXStatusSkipsUnchangedObservations(t *testing.T) { for index := range child.Status.Conditions { child.Status.Conditions[index].LastTransitionTime = transition } - child.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ - "engine": {Ready: state == v1beta1.DGDStateSuccessful}, + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ + "engine": {Conditions: []metav1.Condition{child.Status.Conditions[0]}}, } r := &graphReconciler{Client: newLPXTestClient(t, child)} previous := child.Status.DeepCopy() @@ -133,6 +134,10 @@ func TestLPXStatusSkipsUnchangedObservations(t *testing.T) { } setReadyCondition(child, state, "not externally visible") setReadyCondition(child, state, "current observation") + component := child.Status.Components["engine"] + component.Conditions = []metav1.Condition{readyCondition(child.Generation, different, "intermediate observation")} + meta.SetStatusCondition(&component.Conditions, readyCondition(child.Generation, state, "current observation")) + child.Status.Components["engine"] = component require.Zero(t, writes) require.NoError(t, r.updateStatus(t.Context(), child, previous)) require.Zero(t, writes) @@ -144,7 +149,20 @@ func TestLPXStatusSkipsUnchangedObservations(t *testing.T) { require.NoError(t, r.updateStatus(t.Context(), child, previous)) require.Equal(t, 1, writes) require.True(t, meta.FindStatusCondition(child.Status.Conditions, "Ready").LastTransitionTime.After(transition.Time)) - require.Equal(t, state == v1beta1.DGDStateSuccessful, child.Status.Components["engine"].Ready) + require.True(t, apiequality.Semantic.DeepEqual(previous.Components["engine"].Conditions, child.Status.Components["engine"].Conditions)) + + t.Log("Observing the same component state for a new generation preserves its transition time") + previous = child.Status.DeepCopy() + child.Generation++ + require.NoError(t, r.Update(t.Context(), child)) + component = child.Status.Components["engine"] + component.Conditions = []metav1.Condition{readyCondition(child.Generation, state, "current observation")} + child.Status.Components["engine"] = component + require.NoError(t, r.updateStatus(t.Context(), child, previous)) + require.Equal(t, 2, writes) + ready := meta.FindStatusCondition(child.Status.Components["engine"].Conditions, v1alpha1.LPXReadyCondition) + require.Equal(t, child.Generation, ready.ObservedGeneration) + require.True(t, transition.Equal(&ready.LastTransitionTime)) }) } } @@ -170,13 +188,13 @@ func TestPipelineRequestReadyConditionGolden(t *testing.T) { request := &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{Generation: 7}, Status: status.DeepCopy()} before := request.DeepCopy() requests := map[string]*lpxv1alpha1.LPUPipelineRequest{"observed": request} - require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + require.False(t, setPipelineRequestReadyCondition(&deployment.Status.Conditions, deployment.Generation, requests)) require.Len(t, deployment.Status.Conditions, 1) conditions[string(status.Phase)] = *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) t.Log("A failure outranks missing receipts and repeated map observations stay stable") requests["first-open"] = &lpxv1alpha1.LPUPipelineRequest{} - require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + require.False(t, setPipelineRequestReadyCondition(&deployment.Status.Conditions, deployment.Generation, requests)) wantReason := v1alpha1.LPXReadyReasonPending if status.Phase == lpxv1alpha1.RequestPhaseUnsupported || status.Phase == "future" { wantReason = v1alpha1.LPXReadyReasonFailed @@ -184,7 +202,7 @@ func TestPipelineRequestReadyConditionGolden(t *testing.T) { condition := *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition) require.Equal(t, wantReason, condition.Reason) for range 10 { - require.False(t, setPipelineRequestReadyCondition(deployment, requests)) + require.False(t, setPipelineRequestReadyCondition(&deployment.Status.Conditions, deployment.Generation, requests)) require.Equal(t, condition, *meta.FindStatusCondition(deployment.Status.Conditions, v1alpha1.LPXReadyCondition)) } require.Equal(t, before, request) @@ -221,7 +239,7 @@ func TestPipelineRequestReadyConditionRequiresCurrentReceipts(t *testing.T) { Diagnostics: []lpxv1alpha1.StatusDiagnostic{{Detail: "Old diagnostic"}}, }, } - require.Equal(t, tc.allBound, setPipelineRequestReadyCondition(child, map[string]*lpxv1alpha1.LPUPipelineRequest{"request": request})) + require.Equal(t, tc.allBound, setPipelineRequestReadyCondition(&child.Status.Conditions, child.Generation, map[string]*lpxv1alpha1.LPUPipelineRequest{"request": request})) if tc.allBound { require.Equal(t, before, &child.Status) } else { diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index 4ead8af4b3df..e5f48f54cbe8 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -311,7 +311,6 @@ func evaluateGroveComponents( componentReadiness := componentReadinesses[component.ComponentName] componentStatus := componentReadiness.status componentStatus.RuntimeNamespace = namespacePlan.runtimeNamespace(dgd, component) - componentStatus.Ready = componentReadiness.ready componentStatuses[component.ComponentName] = componentStatus } diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index dc7a9628f5a8..384e327dcf3d 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -39,15 +39,10 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD } verifiedAvailable := int32(0) result := func(ready bool, classification, message string) GroveReadiness { - status.Ready = ready if status.AvailableReplicas != nil { status.AvailableReplicas = ptr.To(min(*status.AvailableReplicas, verifiedAvailable)) } statuses[component.ComponentName] = status - for name, member := range statuses { - member.Ready = ready - statuses[name] = member - } return GroveReadiness{Ready: ready, Classification: classification, Message: message, ComponentStatuses: statuses} } pending := func(message string) GroveReadiness { diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index 3b2e8935e23b..86a082a0028c 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -1186,7 +1186,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { "decode": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-decode"}, - Ready: true, Replicas: 2, UpdatedReplicas: 2, AvailableReplicas: ptr.To(int32(2)), @@ -1195,7 +1194,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, - Ready: true, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: ptr.To(int32(3)), @@ -1346,7 +1344,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { "frontend": { ComponentKind: v1beta1.ComponentKindPodClique, ComponentNames: []string{"test-dgd-0-frontend"}, - Ready: true, Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -1363,7 +1360,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { "prefill": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-prefill"}, - Ready: true, Replicas: 2, UpdatedReplicas: 2, AvailableReplicas: ptr.To(int32(2)), diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 0d2cc05ea514..db6c9717cba1 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -1394,7 +1394,6 @@ _Appears in:_ | `runtimeNamespace` _string_ | RuntimeNamespace is the effective Dynamo runtime namespace for this
component. Worker components may include a generation suffix; non-workers and
Grove-backed workers use the base namespace. During rolling updates, worker
status keeps the old active revision namespace until cutover completes. | | Optional: \{\}
| | `gpusPerEngine` _integer_ | GPUsPerEngine is the number of GPUs assigned to one inference engine in a
service replica, across all of its nodes. Independent auxiliary GPU
allocations are excluded. A present zero means the engine itself has no
GPUs; consult GPUsPerReplica for auxiliary allocations. | | Minimum: 0
Optional: \{\}
| | `gpusPerReplica` _integer_ | GPUsPerReplica is the unique GPU allocation added when this service scales
by one replica, across all nodes, application and initialization phases,
and provider-owned Pods. Scalar GPUs use the Kubernetes effective Pod
scheduling footprint; shared DRA claims are counted once. A present zero
records a successful non-GPU resolution; omission means no current shape
is available. | | Minimum: 0
Optional: \{\}
| -| `ready` _boolean_ | Ready is the binary determination of whether the correct number of replicas
are scheduled and available. | | Optional: \{\}
| | `replicas` _integer_ | Replicas is the total number of non-terminated replicas.
Required for all component kinds. | | Minimum: 0
| | `updatedReplicas` _integer_ | UpdatedReplicas is the number of replicas at the current/desired revision.
Required for all component kinds. | | Minimum: 0
| | `readyReplicas` _integer_ | ReadyReplicas is the number of ready replicas.
Populated for PodClique, Deployment, and LeaderWorkerSet.
Not available for PodCliqueScalingGroup.
When nil, the field is omitted from the API response. | | Minimum: 0
Optional: \{\}
| @@ -1706,7 +1705,6 @@ _Appears in:_ | `runtimeNamespace` _string_ | runtimeNamespace is the effective Dynamo runtime namespace for this
component. Worker components may include a generation suffix; non-workers
use the base namespace. During rolling updates, worker status keeps the old
active revision namespace until cutover completes. | | Optional: \{\}
| | `gpusPerEngine` _integer_ | gpusPerEngine is the number of GPUs assigned to one inference engine in a
component replica, across all of its nodes. Independent auxiliary GPU
allocations are excluded. A present zero means the engine itself has no
GPUs; consult gpusPerReplica for auxiliary allocations. | | Minimum: 0
Optional: \{\}
| | `gpusPerReplica` _integer_ | gpusPerReplica is the unique GPU allocation added when this component
scales by one replica, across all nodes, application and initialization
phases, and provider-owned Pods. Scalar GPUs use the Kubernetes effective
Pod scheduling footprint; shared DRA claims are counted once. A present
zero records a successful non-GPU resolution; omission means no current
shape is available. | | Minimum: 0
Optional: \{\}
| -| `ready` _boolean_ | ready is the binary determination of whether the correct number of replicas
are scheduled and available. | | Optional: \{\}
| | `replicas` _integer_ | replicas is the total number of non-terminated replicas. | | Minimum: 0
| | `updatedReplicas` _integer_ | updatedReplicas is the number of replicas at the current/desired revision. | | Minimum: 0
| | `readyReplicas` _integer_ | readyReplicas is the number of ready replicas. Populated for
`PodClique`, `Deployment`, and `LeaderWorkerSet`; not available for
`PodCliqueScalingGroup`. | | Minimum: 0
Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index b9227108ffc1..ca1137376595 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -2022,11 +2022,6 @@ GPUsPerReplica is the unique GPU allocation added when this service scales by on **Validation:** Minimum: 0 Optional: \{\} - -Ready is the binary determination of whether the correct number of replicas are scheduled and available. -**Validation:** Optional: \{\} - - Replicas is the total number of non-terminated replicas. Required for all component kinds. **Validation:** Minimum: 0 @@ -2391,11 +2386,6 @@ gpusPerReplica is the unique GPU allocation added when this component scales by **Validation:** Minimum: 0 Optional: \{\} - -ready is the binary determination of whether the correct number of replicas are scheduled and available. -**Validation:** Optional: \{\} - - replicas is the total number of non-terminated replicas. **Validation:** Minimum: 0 From 811f209dc400c5b25be6dcfbd175989e1950d6f1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 17:24:25 +0000 Subject: [PATCH 122/196] refactor(operator): tidy LPX API helpers and documentation Signed-off-by: Andrew Paprotsky --- .../api/v1alpha1/dynamocomponentdeployment_types.go | 5 +++++ ...o => dynamographdeployment_lpx_conversion_test.go} | 0 deploy/operator/api/v1alpha1/lpx_types.go | 11 ----------- .../api/v1beta1/dynamocomponentdeployment_types.go | 6 +++--- .../crd/bases/nvidia.com_dynamographdeployments.yaml | 6 +++--- .../additional-resources/api-reference-k8s.md | 6 +++--- .../reference/kubernetes-api/full-api-reference.mdx | 6 +++--- 7 files changed, 17 insertions(+), 23 deletions(-) rename deploy/operator/api/v1alpha1/{lpx_conversion_test.go => dynamographdeployment_lpx_conversion_test.go} (100%) delete mode 100644 deploy/operator/api/v1alpha1/lpx_types.go diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index b8ad5e4587c2..29cfac036ae7 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -403,6 +403,11 @@ func (s *DynamoComponentDeployment) GetNumberOfNodes() int32 { return s.Spec.GetNumberOfNodes() } +// IsLPX reports whether this shared spec uses the LPX integration. +func (s *DynamoComponentDeploymentSharedSpec) IsLPX() bool { + return s.ComponentType == commonconsts.ComponentTypeLPX +} + func (s *DynamoComponentDeploymentSharedSpec) IsMultinode() bool { return s.GetNumberOfNodes() > 1 } diff --git a/deploy/operator/api/v1alpha1/lpx_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_lpx_conversion_test.go similarity index 100% rename from deploy/operator/api/v1alpha1/lpx_conversion_test.go rename to deploy/operator/api/v1alpha1/dynamographdeployment_lpx_conversion_test.go diff --git a/deploy/operator/api/v1alpha1/lpx_types.go b/deploy/operator/api/v1alpha1/lpx_types.go deleted file mode 100644 index a174bab26f1a..000000000000 --- a/deploy/operator/api/v1alpha1/lpx_types.go +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - -// IsLPX reports whether this shared spec uses the LPX integration. -func (s *DynamoComponentDeploymentSharedSpec) IsLPX() bool { - return s.ComponentType == commonconsts.ComponentTypeLPX -} diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 78645e5eea29..d5376cddfc03 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -66,9 +66,9 @@ type DynamoComponentDeploymentSpec struct { // volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are // replaced with a single `podTemplate` field holding a native // `corev1.PodTemplateSpec`. The operator injects its defaults into the -// container named `"main"` and merges user overrides. Users can add sidecars, -// init containers, and pod-level configuration directly in `podTemplate` -// without any `extraPodSpec`-style escape hatch. +// container named `"main"` and merges user overrides using strategic-merge-by-name +// semantics. Users can add sidecars, init containers, and pod-level configuration +// directly in `podTemplate` without any `extraPodSpec`-style escape hatch. // +kubebuilder:validation:XValidation:rule="!has(self.eppConfig) || (has(self.type) && self.type == 'epp')",message="eppConfig may only be set when type is epp" // +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 9a593f30b87c..52c42ec7f8ba 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20526,9 +20526,9 @@ spec: volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the - container named `"main"` and merges user overrides. Users can add sidecars, - init containers, and pod-level configuration directly in `podTemplate` - without any `extraPodSpec`-style escape hatch. + container named `"main"` and merges user overrides using strategic-merge-by-name + semantics. Users can add sidecars, init containers, and pod-level configuration + directly in `podTemplate` without any `extraPodSpec`-style escape hatch. properties: compilationCache: description: |- diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index db6c9717cba1..7da61c10feb8 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -1878,9 +1878,9 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides. Users can add sidecars, -init containers, and pod-level configuration directly in `podTemplate` -without any `extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides using strategic-merge-by-name +semantics. Users can add sidecars, init containers, and pod-level configuration +directly in `podTemplate` without any `extraPodSpec`-style escape hatch. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index ca1137376595..cc8741d80d36 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -2609,9 +2609,9 @@ v1alpha1 (resources, envs, envFromSecret, livenessProbe, readinessProbe, volumeMounts, annotations, labels, extraPodMetadata, extraPodSpec) are replaced with a single `podTemplate` field holding a native `corev1.PodTemplateSpec`. The operator injects its defaults into the -container named `"main"` and merges user overrides. Users can add sidecars, -init containers, and pod-level configuration directly in `podTemplate` -without any `extraPodSpec`-style escape hatch. +container named `"main"` and merges user overrides using strategic-merge-by-name +semantics. Users can add sidecars, init containers, and pod-level configuration +directly in `podTemplate` without any `extraPodSpec`-style escape hatch. **Kind:** `type` From dbc464e01c8769b373516fd5c8b37874b53b1f4b Mon Sep 17 00:00:00 2001 From: Steven Davidovitz <13248+steved@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:01:45 -0700 Subject: [PATCH 123/196] fix(operator): persist Cyborg replica counts in PCS templates (#555) Signed-off-by: Steven Davidovitz --- .../controller/lpx/controller_test.go | 74 ++++++++++++++++--- .../internal/controller/lpx/render_test.go | 6 +- deploy/operator/internal/dynamo/lpx/render.go | 5 -- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 960505766155..dd9daca12577 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -78,7 +78,7 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { require.NoError(t, err) } - t.Log("Accept scheduler requests and observe initial Cyborg capacity independently of the seed") + t.Log("Accept scheduler requests and observe each workload's initial Cyborg capacity") requests, err := r.getPipelineRequests(t.Context(), pcs) require.NoError(t, err) require.Len(t, requests, 4) @@ -130,6 +130,10 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) + t.Log("Observe the PCS template before scaling existing workers") + _, err = r.Reconcile(t.Context(), key) + require.NoError(t, err) + t.Log("Observe the capacity write before calculating each workload's readiness") _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) @@ -137,8 +141,25 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { require.True(t, meta.IsStatusConditionFalse(child.Status.Components["lpx"].Conditions, v1alpha1.LPXReadyCondition)) require.True(t, meta.IsStatusConditionTrue(child.Status.Components[second.ComponentName].Conditions, v1alpha1.LPXReadyCondition)) - t.Log("Preserve PCS identity and spec, both groups, all Agents, and the other workload's workers") + t.Log("Persist the Cyborg template count without changing PCS identity or the other templates") + expectedPCS := pcs.DeepCopy() + for _, template := range expectedPCS.Spec.Template.Cliques { + if template.Name == plans["lpx"].CyborgTemplate { + template.Spec.Replicas = ptr.Deref(replicas, 1) + } + } + actualPCS := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), actualPCS)) + require.Equal(t, pcs.UID, actualPCS.UID) + require.True(t, actualPCS.DeletionTimestamp.IsZero()) + require.Equal(t, pcs.Annotations[lpx.WorkloadDigestAnnotation], actualPCS.Annotations[lpx.WorkloadDigestAnnotation]) + require.Equal(t, expectedPCS.Spec, actualPCS.Spec) + + t.Log("Preserve both groups, all Agents, the other workload's workers and scheduler requests") for _, original := range objects { + if _, ok := original.(*grovev1alpha1.PodCliqueSet); ok { + continue + } if clique, ok := original.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker && clique.Labels[lpx.StageLabel] == "lpx" { continue } @@ -153,7 +174,7 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { require.Equal(t, requestUIDs[request.Name], request.UID) } - t.Log("Observe the requested live capacity and prove Ready without changing the template seed") + t.Log("Observe the requested live capacity and prove Ready without repeating scale writes") var workers []*grovev1alpha1.PodClique for ordinal := range plans["lpx"].Replicas { clique := &grovev1alpha1.PodClique{} @@ -1822,7 +1843,6 @@ func TestLPXScalingWaitsForObservedCapacity(t *testing.T) { ctx := t.Context() child, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) dgd.GetComponentByName("lpx").Replicas = ptr.To(int32(2)) - dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(3)) r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, selected) pcsg := getResource[*grovev1alpha1.PodCliqueScalingGroup](t, objects, selected.plan.LPXScalingGroup) @@ -1844,6 +1864,16 @@ func TestLPXScalingWaitsForObservedCapacity(t *testing.T) { request.Status = newTestPipelineRequest(child, pcs, request.Name, time.Now(), lpxv1alpha1.RequestPhaseBound).Status require.NoError(t, r.Update(ctx, request)) } + + t.Log("Request three Cyborg replicas while the PCS template and live cliques still have one") + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) + dgd.GetComponentByName("lpx").ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(3)) + require.NoError(t, r.Update(ctx, dgd)) + require.NoError(t, r.Get(ctx, key.NamespacedName, child)) + child.Spec.InputRevision, err = dynamo.LPXInputRevision(dgd, "") + require.NoError(t, err) + child.Generation++ + require.NoError(t, r.Update(ctx, child)) pcsg.Spec.Replicas = tc.groupReplicas pcsg.Status.Replicas, pcsg.Status.UpdatedReplicas = tc.groupReplicas, tc.groupReplicas pcsg.Status.AvailableReplicas, pcsg.Status.ScheduledReplicas = tc.groupReplicas, tc.groupReplicas @@ -1893,20 +1923,44 @@ func TestLPXScalingWaitsForObservedCapacity(t *testing.T) { }, }) - t.Log("A capacity write stops reconciliation before the cached ready 1/1 state can be reported") + if tc.groupReplicas > 2 { + t.Log("Scale down the group before updating its worker template") + _, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.Equal(t, 1, groupUpdates) + require.Zero(t, cliqueUpdates) + } + + t.Log("Persist the PCS worker template before scaling existing cliques") + cliqueLists = 0 result, err := r.Reconcile(ctx, key) require.NoError(t, err) - require.Zero(t, result, "capacity observations resume reconciliation through watches") - require.Equal(t, tc.wantGroupUpdates, groupUpdates) + require.Zero(t, result, "PCS observations resume reconciliation through watches") + require.Zero(t, cliqueUpdates) require.Equal(t, 1, cliqueLists) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(pcs), pcs)) + cyborgIndex := slices.IndexFunc(pcs.Spec.Template.Cliques, func(clique *grovev1alpha1.PodCliqueTemplateSpec) bool { + return clique.Name == selected.plan.CyborgTemplate + }) + require.NotEqual(t, -1, cyborgIndex) + require.EqualValues(t, 3, pcs.Spec.Template.Cliques[cyborgIndex].Spec.Replicas) require.NoError(t, r.Get(ctx, key.NamespacedName, child)) require.Equal(t, v1alpha1.LPXReadyReasonPending, meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition).Reason) - if tc.wantGroupUpdates > 0 { - t.Log("Observe the group scale before updating its workers") - require.Zero(t, cliqueUpdates) + if tc.groupReplicas < 2 { + t.Log("Scale up the group and wait to observe it before updating its workers") _, err = r.Reconcile(ctx, key) require.NoError(t, err) + require.Equal(t, 1, groupUpdates) + require.Zero(t, cliqueUpdates) } + + t.Log("Scaling the workers stops reconciliation before the cached ready 1/1 state can be reported") + cliqueLists = 0 + result, err = r.Reconcile(ctx, key) + require.NoError(t, err) + require.Zero(t, result, "capacity observations resume reconciliation through watches") + require.Equal(t, 1, cliqueLists) + require.Equal(t, tc.wantGroupUpdates, groupUpdates) require.Equal(t, 2, cliqueUpdates) require.NoError(t, r.Get(ctx, key.NamespacedName, child)) require.False(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index c6680b3a0965..18d4148143a5 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -472,7 +472,7 @@ func TestLPXRenderingMetadata(t *testing.T) { } } -func TestLPXReplicaChangesPreserveRenderedTemplates(t *testing.T) { +func TestLPXReplicaChangesUpdateCyborgTemplate(t *testing.T) { t.Log("Render omitted hybrid capacity for two I/O endpoints with two clients each") root := t.TempDir() const buildID = "split-io" @@ -501,13 +501,15 @@ func TestLPXReplicaChangesPreserveRenderedTemplates(t *testing.T) { require.EqualValues(t, 4, cyborg.Spec.Replicas) require.Equal(t, ptr.To(int32(1)), cyborg.Spec.MinAvailable) - t.Log("Changing backbone and Cyborg capacity preserves templates, configuration and the graph digest") + t.Log("Persist explicit Cyborg capacity while preserving the group seed, configuration and graph digest") dgd.Spec.Components[0].Replicas = ptr.To(int32(12)) dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor).Replicas = ptr.To(int32(8)) workloads, plans, err = r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) after, afterResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) require.NoError(t, err) + require.EqualValues(t, 8, after.Spec.Template.Cliques[cyborgIndex].Spec.Replicas) + before.Spec.Template.Cliques[cyborgIndex].Spec.Replicas = 8 require.Equal(t, before, after) require.Equal(t, beforeResources, afterResources) } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 3df7a3d38055..52a36b2e6fa5 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -226,11 +226,6 @@ func RenderNodeLocal( return nil, err } - minimumReplicas, err := workload.MinimumCyborgReplicas() - if err != nil { - return nil, err - } - cyborg.Spec.Replicas = minimumReplicas cyborg.Spec.MinAvailable = ptr.To(int32(1)) } From f7262ab16c91c89522830ad1703addd2b623998a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:50:48 +0300 Subject: [PATCH 124/196] refactor(operator): move LPX constants into the API (#557) Signed-off-by: Andrew Paprotsky --- .../dynamocomponentdeployment_types.go | 2 +- .../api/v1alpha1/lpxgraphdeployment_types.go | 30 ++++++++++++++++++- .../api/v1alpha1/zz_generated.deepcopy.go | 26 +++++++++++++++- deploy/operator/api/v1beta1/lpx_types.go | 14 --------- .../api/v1beta1/zz_generated.deepcopy.go | 24 --------------- deploy/operator/internal/consts/lpx.go | 15 ---------- .../controller/dgd_grove_lpx_status_test.go | 2 +- .../dgd_lpx_handoff_envtest_test.go | 12 ++++---- .../controller/dgd_lpx_handoff_test.go | 2 +- .../controller/lpx/controller_test.go | 6 ++-- .../internal/controller/lpx/modeldownload.go | 4 +-- .../controller/lpx/modeldownload_test.go | 4 +-- .../controller/lpx/pipeline_request_test.go | 7 ++--- .../internal/controller/lpx/render_test.go | 8 ++--- .../internal/controller/lpx/testdata/dgd.yaml | 2 -- deploy/operator/internal/dynamo/graph_lpx.go | 4 +-- .../internal/dynamo/graph_lpx_test.go | 4 +-- .../dynamo/lpx/compiler_fixture_test.go | 3 -- .../internal/dynamo/lpx/cyborg_render.go | 3 +- .../internal/dynamo/lpx/lpu_configmap.go | 5 ++-- .../operator/internal/dynamo/lpx/pod_roles.go | 6 ++-- deploy/operator/internal/dynamo/lpx/render.go | 7 +++-- .../internal/dynamo/lpx/render_test.go | 3 +- .../dynamo/lpx/resolve_workload_test.go | 4 +-- .../lpu-cyborg-specdec.input.yaml | 2 -- .../lpu-gpu-specdecode.input.yaml | 2 -- .../node-local-v2-hybrid.input.yaml | 2 -- .../node-local-v2-lpu-only.input.yaml | 2 -- .../node-local-v2-specdecode.input.yaml | 2 -- .../node-local-v3-hx-hybrid.input.yaml | 2 -- .../node-local-v3-hx-lpu-only.input.yaml | 2 -- .../node-local-v3-hx-specdecode.input.yaml | 2 -- .../from_dgd_yaml/single_v2.input.yaml | 2 -- .../validation/dynamographdeployment.go | 3 +- ...hdeployment_lpx_validation_envtest_test.go | 10 ++----- .../additional-resources/api-reference-k8s.md | 4 +-- 36 files changed, 101 insertions(+), 131 deletions(-) delete mode 100644 deploy/operator/internal/consts/lpx.go diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 29cfac036ae7..81fa9b0fae07 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -405,7 +405,7 @@ func (s *DynamoComponentDeployment) GetNumberOfNodes() int32 { // IsLPX reports whether this shared spec uses the LPX integration. func (s *DynamoComponentDeploymentSharedSpec) IsLPX() bool { - return s.ComponentType == commonconsts.ComponentTypeLPX + return s.ComponentType == string(v1beta1.ComponentTypeLPX) } func (s *DynamoComponentDeploymentSharedSpec) IsMultinode() bool { diff --git a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go index 716d5599e8f9..1b86f72abac9 100644 --- a/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go +++ b/deploy/operator/api/v1alpha1/lpxgraphdeployment_types.go @@ -9,6 +9,22 @@ import ( ) const ( + // LPXSchedulerName selects Grove's LPX backend in generated role PodSpecs. + // The operator sets schedulerName to this value for every LPX role, including + // Cyborg. Non-LPX component Pod templates may not select this scheduler. + LPXSchedulerName = "lpx-scheduler" + + // ModelStorageVolumeName identifies the model-storage volume and its mount in + // each LPX role's main container. The operator uses the mountPath from the + // role's podTemplate as the root for compiled model artifacts. + ModelStorageVolumeName = "model-storage" + + // AnnotationExtraResourcesHash records an opaque content hash of the role's + // generated immutable runtime ConfigMap on PodClique templates + // (PodCliqueSet.spec.template.cliques[].annotations). Grove propagates it to + // PodClique and Pod metadata. + AnnotationExtraResourcesHash = "nvidia.com/extra-resources-hash" + // LPXReadyCondition reports if LPXGraphDeployment object is ready. // Note: This condition type is defined to ensure consistent naming of conditions across objects. // Please use object specific variants of this condition which provides more details for each context where @@ -63,7 +79,19 @@ type LPXGraphDeploymentStatus struct { Components map[string]LPXComponentStatus `json:"components,omitempty"` // modelDownload retains the existing remote-build download progress. // +optional - ModelDownload *v1beta1.ModelDownloadStatus `json:"modelDownload,omitempty"` + ModelDownload *ModelDownloadStatus `json:"modelDownload,omitempty"` +} + +// ModelDownloadStatus contains the status of remote LPU model downloads. +type ModelDownloadStatus struct { + // builds is the sorted set of resolved remote LPU build URLs whose artifacts + // were successfully downloaded into model-storage. + // +optional + Builds []string `json:"builds,omitempty"` + + // lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. + // +optional + LastCheckedAt *metav1.Time `json:"lastCheckedAt,omitempty"` } // +kubebuilder:object:root=true diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 874dd93654ab..4fa389406b93 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1474,7 +1474,7 @@ func (in *LPXGraphDeploymentStatus) DeepCopyInto(out *LPXGraphDeploymentStatus) } if in.ModelDownload != nil { in, out := &in.ModelDownload, &out.ModelDownload - *out = new(v1beta1.ModelDownloadStatus) + *out = new(ModelDownloadStatus) (*in).DeepCopyInto(*out) } } @@ -1504,6 +1504,30 @@ func (in *LabelItemSchema) DeepCopy() *LabelItemSchema { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelDownloadStatus) DeepCopyInto(out *ModelDownloadStatus) { + *out = *in + if in.Builds != nil { + in, out := &in.Builds, &out.Builds + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LastCheckedAt != nil { + in, out := &in.LastCheckedAt, &out.LastCheckedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelDownloadStatus. +func (in *ModelDownloadStatus) DeepCopy() *ModelDownloadStatus { + if in == nil { + return nil + } + out := new(ModelDownloadStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelReference) DeepCopyInto(out *ModelReference) { *out = *in diff --git a/deploy/operator/api/v1beta1/lpx_types.go b/deploy/operator/api/v1beta1/lpx_types.go index 9b32f934369f..61494f9f985a 100644 --- a/deploy/operator/api/v1beta1/lpx_types.go +++ b/deploy/operator/api/v1beta1/lpx_types.go @@ -3,8 +3,6 @@ package v1beta1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - // LPXConfig identifies the component's compiled model and configures scheduling. type LPXConfig struct { // buildId references the immutable model build. @@ -27,18 +25,6 @@ type SchedulingSpec struct { AttemptDeadlineSeconds *int64 `json:"attemptDeadlineSeconds,omitempty"` } -// ModelDownloadStatus contains the status of remote LPU model downloads. -type ModelDownloadStatus struct { - // builds is the sorted set of resolved remote LPU build URLs whose artifacts - // were successfully downloaded into model-storage. - // +optional - Builds []string `json:"builds,omitempty"` - - // lastCheckedAt is the last time all remote LPU builds were checked with ModelExpress. - // +optional - LastCheckedAt *metav1.Time `json:"lastCheckedAt,omitempty"` -} - // HasLPXComponent returns true if any component uses the LPX integration. func (s *DynamoGraphDeployment) HasLPXComponent() bool { for i := range s.Spec.Components { diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index f97822641d3a..fe86c0e94ceb 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -1177,30 +1177,6 @@ func (in *ModelCacheSpec) DeepCopy() *ModelCacheSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ModelDownloadStatus) DeepCopyInto(out *ModelDownloadStatus) { - *out = *in - if in.Builds != nil { - in, out := &in.Builds, &out.Builds - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.LastCheckedAt != nil { - in, out := &in.LastCheckedAt, &out.LastCheckedAt - *out = (*in).DeepCopy() - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelDownloadStatus. -func (in *ModelDownloadStatus) DeepCopy() *ModelDownloadStatus { - if in == nil { - return nil - } - out := new(ModelDownloadStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelReference) DeepCopyInto(out *ModelReference) { *out = *in diff --git a/deploy/operator/internal/consts/lpx.go b/deploy/operator/internal/consts/lpx.go deleted file mode 100644 index 49a6f3698411..000000000000 --- a/deploy/operator/internal/consts/lpx.go +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package consts - -const ( - KubeAnnotationLPXSchedulerBackend = "scheduling.lpu.nvidia.com/scheduler-backend" - KubeAnnotationLPXExecutionBackend = "scheduling.lpu.nvidia.com/execution-backend" - LPXSchedulerBackend = "lpx-scheduler" - ComponentTypeLPX = "lpx" - ModelStorageVolumeName = "model-storage" - - // AnnotationExtraResourcesHash stores the generated LPX configuration hash. - AnnotationExtraResourcesHash = "nvidia.com/extra-resources-hash" -) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go index a8584cce1116..75e278470df4 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status_test.go @@ -37,7 +37,7 @@ func TestLPXStatusUsesOneCurrentChild(t *testing.T) { checkedAt := metav1.Unix(1, 0) child.Status = v1alpha1.LPXGraphDeploymentStatus{ ObservedGeneration: 2, - ModelDownload: &v1beta1.ModelDownloadStatus{ + ModelDownload: &v1alpha1.ModelDownloadStatus{ Builds: []string{"gs://models/first", "gs://models/second"}, LastCheckedAt: &checkedAt, }, Components: map[string]v1alpha1.LPXComponentStatus{}, diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 4ba411ea75b3..c23ed9dfb210 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -162,9 +162,9 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "example/conductor:1.4.0", Command: []string{"/custom-conductor"}, Args: []string{"--workers", "$(LPX_ALLOCATION)"}, - VolumeMounts: []corev1.VolumeMount{{Name: consts.ModelStorageVolumeName, MountPath: "/models"}}, + VolumeMounts: []corev1.VolumeMount{{Name: v1alpha1.ModelStorageVolumeName, MountPath: "/models"}}, }}, - Volumes: []corev1.Volume{{Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + Volumes: []corev1.Volume{{Name: v1alpha1.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, }}, }, { Name: v1beta1.ComponentRoleLPXAgent, @@ -172,7 +172,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { Containers: []corev1.Container{{ Name: consts.MainContainerName, Image: "example/lpu-runtime:1.4.0", VolumeMounts: []corev1.VolumeMount{ - {Name: consts.ModelStorageVolumeName, MountPath: "/models"}, + {Name: v1alpha1.ModelStorageVolumeName, MountPath: "/models"}, {Name: "config", MountPath: "/configs"}, {Name: "host-dev", MountPath: "/dev"}, {Name: "host-sys", MountPath: "/sys"}, @@ -182,7 +182,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { }, }}, Volumes: []corev1.Volume{ - {Name: consts.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: v1alpha1.ModelStorageVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "host-dev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}}, {Name: "host-sys", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys"}}}, {Name: "hugepages", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumHugePages}}}, @@ -272,7 +272,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { if !assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) { return } - configMap.Name = dynamolpx.LPUConfigMapName(pcs.Name, pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash]) + configMap.Name = dynamolpx.LPUConfigMapName(pcs.Name, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]) for _, resource := range []client.Object{pcs, configMap, service} { if assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(resource), resource)) { assert.Equal(c, metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK), metav1.GetControllerOf(resource)) @@ -313,7 +313,7 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { t.Log("Persist the child's download payload and project its actionable failure") child.Status.ObservedGeneration = child.Generation - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} + child.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, ObservedGeneration: child.Generation, LastTransitionTime: metav1.Now(), Reason: v1alpha1.LPXReadyReasonFailed, Message: "Check the namespace quota"}} child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index cfec78519d7d..8e8c0723012c 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -42,7 +42,7 @@ func newLPXHandoffSource(t *testing.T, fixture string) *v1beta1.DynamoGraphDeplo t.Helper() data, err := os.ReadFile(filepath.Join("../dynamo/lpx/testdata/from_dgd_yaml", fixture+".input.yaml")) require.NoError(t, err) - source := &v1beta1.DynamoGraphDeployment{} + source := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} require.NoError(t, yaml.Unmarshal(data, source)) source.GetComponentByName("lpu").ComponentName = "lpx" source.Generation = 3 diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index dd9daca12577..b4c40f80ff63 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1386,7 +1386,7 @@ func TestLPXMaterializationUsesDGDAndChildIdentity(t *testing.T) { root := dynamo.PCSNameForLPX(child) for _, object := range []client.Object{ &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: root, Namespace: child.Namespace}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: lpx.LPUConfigMapName(root, pcs.Spec.Template.Cliques[0].Annotations[consts.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: lpx.LPUConfigMapName(root, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-serve", Namespace: child.Namespace}}, } { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) @@ -2419,7 +2419,7 @@ func TestLPXPublishedWorkloadCleanupAfterFailure(t *testing.T) { conductor := dgd.Spec.Components[0].ComponentRole(v1beta1.ComponentRoleLPXConductor) for index := range conductor.PodTemplate.Spec.Containers[0].VolumeMounts { mount := &conductor.PodTemplate.Spec.Containers[0].VolumeMounts[index] - if mount.Name == consts.ModelStorageVolumeName { + if mount.Name == v1alpha1.ModelStorageVolumeName { mount.MountPath = "/different-model-storage" } } @@ -2664,7 +2664,7 @@ func loadTestDGD(t testing.TB, pipeline lpx.Pipeline, buildID string) *v1beta1.D t.Helper() payload, err := os.ReadFile("testdata/dgd.yaml") require.NoError(t, err) - dgd := &v1beta1.DynamoGraphDeployment{} + dgd := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} require.NoError(t, yaml.UnmarshalStrict(payload, dgd)) dgd.Spec.Components[0].LPX.BuildID = buildID diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index f26dd6d8b54e..ded524968570 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -87,7 +87,7 @@ func (r *graphReconciler) reconcileModelDownloads( return ctrl.Result{}, nil } - deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: downloaded} + deployment.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{Builds: downloaded} if err != nil { return ctrl.Result{}, err } @@ -100,7 +100,7 @@ func (r *graphReconciler) reconcileModelDownloads( lastCheckedAt = new(metav1.Now()) } - deployment.Status.ModelDownload = &v1beta1.ModelDownloadStatus{Builds: downloaded, LastCheckedAt: lastCheckedAt} + deployment.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{Builds: downloaded, LastCheckedAt: lastCheckedAt} return ctrl.Result{}, nil } diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index fc2743570ea6..bb239d04d23d 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -170,7 +170,7 @@ func TestRunningLPXModelDownloadRefresh(t *testing.T) { t.Log("Refresh a running workload while preserving its downloaded builds") dgd := newModelDownloadDGD(modelDownloadTestBuildID) child := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}} - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ + child.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{ Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: &metav1.Time{Time: tt.checkedAt}, } child.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}} @@ -249,7 +249,7 @@ func TestModelDownloadCacheAcrossRevisions(t *testing.T) { setReadyCondition(child, v1beta1.DGDStateSuccessful, "Previous input is ready") } checkedAt := metav1.NewTime(time.Now().Add(-tc.cacheAge)) - child.Status.ModelDownload = &v1beta1.ModelDownloadStatus{ + child.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{ Builds: []string{modelDownloadTestBuildID}, LastCheckedAt: &checkedAt, } registry := newModelDownloadRegistry(t, nil, map[string]error{tc.build: fmt.Errorf("ModelExpress unavailable")}) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index fb8b39f0d432..3f961861ef5c 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -12,7 +12,6 @@ import ( "testing" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -73,11 +72,9 @@ func TestGetPipelineRequests(t *testing.T) { } func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { - t.Log("Publish the implicit hybrid runtime without interpreting stale selector annotations") + t.Log("Publish the implicit hybrid runtime") ctx := t.Context() deployment, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) - dgd.Annotations[consts.KubeAnnotationLPXSchedulerBackend] = "unknown-scheduler" - dgd.Annotations[consts.KubeAnnotationLPXExecutionBackend] = "unknown-execution" reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) require.Empty(t, desired.plan.ConductorTemplate) require.Empty(t, desired.plan.ConductorClique) @@ -91,7 +88,7 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { require.NotContains(t, pcsg.Spec.CliqueNames, "") cyborg := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.CyborgClique) require.NotContains(t, cyborg.Spec.StartsAfter, "") - require.Equal(t, lpx.SchedulerName, cyborg.Spec.PodSpec.SchedulerName) + require.Equal(t, v1alpha1.LPXSchedulerName, cyborg.Spec.PodSpec.SchedulerName) createLPXTestObjects(t, ctx, reconciler.Client, objects...) condition := publishSelectedLPXForTest(t, ctx, reconciler, deployment, desired) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 18d4148143a5..b91859ad9587 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -95,7 +95,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { got, extraResources, err := r.renderPodCliqueSet(t.Context(), child, &dynamoDeployment, workloads, plans) require.NoError(t, err) for _, clique := range got.Spec.Template.Cliques { - require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.Equal(t, v1alpha1.LPXSchedulerName, clique.Spec.PodSpec.SchedulerName) component := dynamoDeployment.GetComponentByName(clique.Labels[consts.KubeLabelDynamoComponent]) require.NotNil(t, component) require.True(t, component.IsLPX()) @@ -114,7 +114,7 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { require.NoError(t, err) require.NotEqual(t, normal.Name, got.Name) for _, clique := range normal.Spec.Template.Cliques { - require.NotEqual(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.NotEqual(t, v1alpha1.LPXSchedulerName, clique.Spec.PodSpec.SchedulerName) require.False(t, dynamoDeployment.GetComponentByName(clique.Labels[consts.KubeLabelDynamoComponent]).IsLPX()) } podCliqueSets := []*grovev1alpha1.PodCliqueSet{got} @@ -627,7 +627,7 @@ func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { if clique.Name == plan.ConductorTemplate { require.Equal(t, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}, clique.Spec.PodSpec.Containers[0].Env[0]) require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, env) - hash = clique.Annotations[consts.AnnotationExtraResourcesHash] + hash = clique.Annotations[v1alpha1.AnnotationExtraResourcesHash] } } require.NotEmpty(t, hash) @@ -1105,7 +1105,7 @@ func TestHybridWorkloadUsesLPXSchedulerWithKaiEnabled(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, rendered.Cliques) for _, clique := range rendered.Cliques { - require.Equal(t, lpx.SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.Equal(t, v1alpha1.LPXSchedulerName, clique.Spec.PodSpec.SchedulerName) require.NotContains(t, clique.Labels, consts.KubeLabelKaiSchedulerQueue, "clique %s", clique.Name) } } diff --git a/deploy/operator/internal/controller/lpx/testdata/dgd.yaml b/deploy/operator/internal/controller/lpx/testdata/dgd.yaml index ae4ce8de2f75..6fac6ea3cbfa 100644 --- a/deploy/operator/internal/controller/lpx/testdata/dgd.yaml +++ b/deploy/operator/internal/controller/lpx/testdata/dgd.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: source-uid generation: 3 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: lpx diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index fc38fe918df6..15d8f9bbd7c2 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -170,7 +170,7 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin }) annotations := lpxSchedulingMetadata(dgd.Annotations) - for _, key := range append(slices.Clone(dgdPropagatedAnnotationKeys), commonconsts.KubeAnnotationLPXSchedulerBackend, commonconsts.KubeAnnotationWorkloadProvider, + for _, key := range append(slices.Clone(dgdPropagatedAnnotationKeys), commonconsts.KubeAnnotationWorkloadProvider, commonconsts.KubeAnnotationGroveUpdateStrategy, commonconsts.KubeAnnotationKaiSchedulerQueue, commonconsts.KubeAnnotationVolcanoQueue) { if value, exists := dgd.Annotations[key]; exists { annotations[key] = value @@ -324,7 +324,7 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn } // Select LPX before applying Grove defaults; the PCS already owns the KAI queue. - gpuTemplate.Spec.SchedulerName = dynamolpx.SchedulerName + gpuTemplate.Spec.SchedulerName = v1alpha1.LPXSchedulerName clique, err := buildCliqueFromTemplate(gpu, *gpuTemplate) if err != nil { return nil, fmt.Errorf("rendering %s.conductor: %w", component.ComponentName, err) diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 7db395e94db0..f05ce009728f 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -25,7 +25,7 @@ import ( ) func TestBackendDetectionSkipsNonWorkers(t *testing.T) { - for _, componentType := range []string{commonconsts.ComponentTypeLPX, commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypePlanner} { + for _, componentType := range []string{string(v1beta1.ComponentTypeLPX), commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypePlanner} { t.Run(componentType, func(t *testing.T) { t.Log("Ignore ambiguous GPU commands and explicit backends for non-workers") backend, err := determineBackendFramework(componentType, nil, @@ -346,7 +346,7 @@ func TestLPXInputRevision(t *testing.T) { for _, key := range []string{ commonconsts.KubeAnnotationEnableMetrics, commonconsts.KubeAnnotationDynamoDiscoveryBackend, commonconsts.KubeAnnotationDynamoKubeDiscoveryMode, commonconsts.KubeAnnotationDynamoOperatorOriginVersion, - commonconsts.KubeAnnotationVLLMDistributedExecutorBackend, commonconsts.KubeAnnotationLPXSchedulerBackend, + commonconsts.KubeAnnotationVLLMDistributedExecutorBackend, commonconsts.KubeAnnotationWorkloadProvider, commonconsts.KubeAnnotationGroveUpdateStrategy, commonconsts.KubeAnnotationKaiSchedulerQueue, commonconsts.KubeAnnotationVolcanoQueue, "kai.scheduler/topology", "priorityClassName", "project", "user", diff --git a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go index 868297720a85..4bf2183c7f91 100644 --- a/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/compiler_fixture_test.go @@ -51,9 +51,6 @@ func newSelectedTestDGD(t *testing.T, name string, components ...v1beta1.DynamoC return &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "test", UID: "uid", Generation: 1, - Annotations: map[string]string{ - commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend, - }, }, Spec: v1beta1.DynamoGraphDeploymentSpec{ Components: components, diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 33fd5e404100..4622928f2b2a 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -8,6 +8,7 @@ package lpx import ( "fmt" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" @@ -51,7 +52,7 @@ func configureHybridCyborg( if err := withLPUConfigVolume(&cyborg.Spec.PodSpec, cyborgConfigMap.Name, true); err != nil { return err } - cyborg.Annotations[commonconsts.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) + cyborg.Annotations[v1alpha1.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) } cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) return nil diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 36e9de19edd4..03ef626e06af 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controllercommon "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" @@ -67,12 +68,12 @@ func LPUConfigMapHash(configMap *corev1.ConfigMap) string { func lpuModelStoragePath(spec corev1.PodSpec) (string, error) { container := common.FindContainerByName(spec.Containers, commonconsts.MainContainerName) mountIndex := slices.IndexFunc(container.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == commonconsts.ModelStorageVolumeName + return mount.Name == v1alpha1.ModelStorageVolumeName }) if mountIndex < 0 { return "", fmt.Errorf( "selected LPX main container requires model storage volume mount %q", - commonconsts.ModelStorageVolumeName, + v1alpha1.ModelStorageVolumeName, ) } mount := container.VolumeMounts[mountIndex] diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index 2efae24653a5..253c5ece64af 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -8,6 +8,7 @@ package lpx import ( "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" @@ -15,9 +16,6 @@ import ( ) const ( - // SchedulerName selects Grove's LPX backend for the complete PodCliqueSet. - SchedulerName = "lpx-scheduler" - v2LPUResourceName corev1.ResourceName = "lpu.nvidia.com/lpu" v3LPUResourceName corev1.ResourceName = "nvidia.com/lpu" @@ -32,7 +30,7 @@ func configureAgentScheduling( ) { // Preserve Agent placement while replacing authored LPU resources with the selected device. stripLPUResources(agent) - agent.SchedulerName = SchedulerName + agent.SchedulerName = v1alpha1.LPXSchedulerName container := common.FindContainerByName(agent.Containers, commonconsts.MainContainerName) // Model projection has already restricted the target family to XT or HX. name, amount := v2LPUResourceName, resource.MustParse("8") diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 52a36b2e6fa5..240766c411fe 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -11,6 +11,7 @@ import ( "slices" "strings" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" @@ -126,7 +127,7 @@ func RenderNodeLocal( return nil, err } annotations := roleAnnotations(conductorTemplate.Annotations, lpxv1alpha1.PodRoleConductor, workloadDigest) - annotations[commonconsts.AnnotationExtraResourcesHash] = configHash + annotations[v1alpha1.AnnotationExtraResourcesHash] = configHash conductor = &grovev1alpha1.PodCliqueTemplateSpec{ Name: conductorTemplateName, Labels: maps.Clone(conductorTemplate.Labels), @@ -177,7 +178,7 @@ func RenderNodeLocal( } annotations := roleAnnotations(template.Annotations, lpxv1alpha1.PodRoleAgent, projection.Digest().String()) - annotations[commonconsts.AnnotationExtraResourcesHash] = configHash + annotations[v1alpha1.AnnotationExtraResourcesHash] = configHash annotations[lpxv1alpha1.PodModelAnnotation] = projection.model annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation] = projection.CompilerSnapshotDigest() annotations[WorkloadModeAnnotation] = string(projection.schedulerWorkloadMode()) @@ -244,7 +245,7 @@ func RenderNodeLocal( // Keep every LPX role in one backend gang, including the KAI fallback roles. for _, clique := range rendered.Cliques { - clique.Spec.PodSpec.SchedulerName = SchedulerName + clique.Spec.PodSpec.SchedulerName = v1alpha1.LPXSchedulerName } rendered.Resources = extraResources diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 5343001ff7bf..45c511952985 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" manifestcapnp "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" @@ -150,7 +151,7 @@ func TestRenderResolvesAuthoredMetadataAndMounts(t *testing.T) { require.Same(t, pcs, rendered) require.Equal(t, projection.Digest().String(), rendered.Spec.Template.PodCliqueScalingGroupConfigs[0].Annotations[WorkloadDigestAnnotation]) for _, clique := range rendered.Spec.Template.Cliques { - require.Equal(t, SchedulerName, clique.Spec.PodSpec.SchedulerName) + require.Equal(t, v1alpha1.LPXSchedulerName, clique.Spec.PodSpec.SchedulerName) if clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleAgent { require.Equal(t, projection.CompilerSnapshotDigest(), clique.Annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation]) } else { diff --git a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go index 5bb9cf9bdc62..2aaa37221099 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go @@ -23,7 +23,6 @@ import ( func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { t.Log("Create one LPX component beside an unrelated conventional decode") dgd := newSelectedTestDGD(t, "graph", testLPXComponent("LPX", "build", v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXAgent, PodTemplate: testLPXPodTemplate("lpu-runtime")}, v1beta1.ComponentRoleSpec{Name: v1beta1.ComponentRoleLPXConductor, PodTemplate: testLPXPodTemplate("conductor-runtime")})) - dgd.Annotations = nil dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ ComponentName: "ordinary-decode", ComponentType: v1beta1.ComponentTypeDecode, Replicas: ptr.To(int32(0)), @@ -135,9 +134,8 @@ func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { replica := plan.ForReplica(1) require.NotEqual(t, plan.Agents[0].CliqueName, replica.Agents[0].CliqueName) - t.Log("A scheduling deadline and annotation do not change hybrid launch") + t.Log("A scheduling deadline does not change hybrid launch") dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} - dgd.Annotations = map[string]string{commonconsts.KubeAnnotationLPXSchedulerBackend: commonconsts.LPXSchedulerBackend} scheduled, err := ResolveWorkload(t.Context(), dgd, singleGroupComponents(t, dgd), source) require.NoError(t, err) require.Equal(t, xt, scheduled) diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml index 8ed2e304d688..e1b8f241549d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 22222222-2222-4222-8222-222222222222 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: draft diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml index d5d75f59ff13..8b3ce326b949 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 22222222-2222-4222-8222-222222222222 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: draft diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml index 923756e3c6fb..784b3b7b0a35 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 22222222-2222-4222-8222-222222222222 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: lpu diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml index 44d93e6fbd6f..608b3a06fe80 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 11111111-1111-4111-8111-111111111111 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: lpu diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml index 001fea7c74e7..d9c00bf95e40 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 33333333-3333-4333-8333-333333333333 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: draft diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml index b44a806bbef4..7d5969e99169 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 44444444-4444-4444-8444-444444444444 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: lpu diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml index bd4307b8c6f1..d9d6c84db762 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 33333333-3333-4333-8333-333333333333 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: lpu diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml index 44a18ac8cea2..ca21b712b41e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.input.yaml @@ -8,8 +8,6 @@ metadata: namespace: test uid: 44444444-4444-4444-8444-444444444444 generation: 1 - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler spec: components: - name: draft diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml index 61df6a27fc03..27ded1d3d454 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.input.yaml @@ -4,8 +4,6 @@ apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - annotations: - scheduling.lpu.nvidia.com/scheduler-backend: lpx-scheduler generation: 1 name: dgd-single-v2 namespace: inference-engine diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 08fc5d69b67b..93f6da4b3e47 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -24,6 +24,7 @@ import ( "strings" semver "github.com/Masterminds/semver/v3" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" @@ -381,7 +382,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( // Ordinary components cannot select the scheduler reserved for LPX-managed Pods. if !component.ManagedByExternalController() && component.PodTemplate != nil && - component.PodTemplate.Spec.SchedulerName == consts.LPXSchedulerBackend { + component.PodTemplate.Spec.SchedulerName == nvidiacomv1alpha1.LPXSchedulerName { allErrs = append(allErrs, field.Forbidden( componentPath.Child("podTemplate", "spec", "schedulerName"), "LPX schedulerName is controller-owned; declare an LPX component instead", diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index ab330de244b7..c1c70d9518f0 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -616,10 +616,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { { name: "selected v1beta1 LPX derives SpecDecode from two models", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { - dgd.Annotations = map[string]string{ - consts.KubeAnnotationLPXSchedulerBackend: "ignored-scheduler", - consts.KubeAnnotationLPXExecutionBackend: "ignored-execution", - } component := &dgd.Spec.Components[0] component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ @@ -775,7 +771,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { { name: "ordinary graph rejects manual LPX scheduler selection", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { - betaWorkerComponent(dgd).PodTemplate.Spec.SchedulerName = consts.LPXSchedulerBackend + betaWorkerComponent(dgd).PodTemplate.Spec.SchedulerName = nvidiacomv1alpha1.LPXSchedulerName }), wantWebhookErrs: []string{"spec.components[1].podTemplate.spec.schedulerName: Forbidden: LPX schedulerName is controller-owned; declare an LPX component instead"}, }, @@ -783,7 +779,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { name: "mixed graph rejects manual LPX scheduler selection on an ordinary component", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { worker := *betaWorkerComponent(betaDGDForAdmission(nil)) - worker.PodTemplate.Spec.SchedulerName = consts.LPXSchedulerBackend + worker.PodTemplate.Spec.SchedulerName = nvidiacomv1alpha1.LPXSchedulerName dgd.Spec.Components = append(dgd.Spec.Components, worker) }), wantWebhookErrs: []string{"spec.components[1].podTemplate.spec.schedulerName: Forbidden: LPX schedulerName is controller-owned; declare an LPX component instead"}, @@ -835,7 +831,7 @@ func alphaLPXDGDForAdmission( dgd := alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services = map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "lpx": { - ComponentType: consts.ComponentTypeLPX, + ComponentType: string(nvidiacomv1beta1.ComponentTypeLPX), Replicas: k8sptr.To(int32(1)), LPX: &nvidiacomv1beta1.LPXConfig{BuildID: "test/build"}, Roles: []nvidiacomv1alpha1.ComponentRoleSpec{ diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 7da61c10feb8..5a848c99d7d2 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -961,6 +961,8 @@ _Appears in:_ + + #### ModelReference @@ -2532,8 +2534,6 @@ _Appears in:_ | `pvcMountPath` _string_ | PVCMountPath is the mount path for the PVC inside the container. | /opt/model-cache | Optional: \{\}
| - - #### ModelReference From 92389605cac50e8c9f8ecad32a1fa9d558e80d29 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 20:09:42 +0000 Subject: [PATCH 125/196] refactor(operator): remove redundant LPX checkpoint guard Admission rejects enabled checkpoints on LPX components, and GetCheckpoint returns nil for absent or disabled configurations. Rely on the existing nil check and remove the test block that bypassed admission with an invalid LPX checkpoint. Signed-off-by: Andrew Paprotsky --- .../controller/dgd_checkpoints_reconciler.go | 2 +- .../controller/dgd_checkpoints_reconciler_test.go | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go b/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go index 15aa2d29ebf4..8f25b14e0139 100644 --- a/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go +++ b/deploy/operator/internal/controller/dgd_checkpoints_reconciler.go @@ -99,7 +99,7 @@ func (r *dgdCheckpointsReconciler) Reconcile( component := &dgd.Spec.Components[i] componentName := component.ComponentName checkpointConfig := dynamo.GetCheckpoint(component) - if checkpointConfig == nil || component.IsLPX() { + if checkpointConfig == nil { continue } if !r.runtimeConfig.Gate.Enabled(features.Checkpoint) { diff --git a/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go b/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go index d987c1fbcc38..be2c2a5cf559 100644 --- a/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_checkpoints_reconciler_test.go @@ -921,21 +921,6 @@ func TestDGDCheckpointsReconciler_ExplicitRestoreDoesNotDependOnActiveWorkerHash assert.True(t, info.Ready) require.NotNil(t, info.NativeSnapshot) assert.Equal(t, referenced.UID, info.NativeSnapshot.UID) - - t.Log("Verify automatic and referenced LPX checkpoint settings do not block the following worker") - dgd.Spec.Components = append([]v1beta1.DynamoComponentDeploymentSharedSpec{{ - ComponentName: "lpx", - ComponentType: v1beta1.ComponentTypeLPX, - Experimental: &v1beta1.ExperimentalSpec{Checkpoint: &v1beta1.ComponentCheckpointConfig{ - Enabled: true, - }}, - }}, dgd.Spec.Components...) - for _, checkpointRef := range []*string{nil, ptr.To("lpx-snapshot")} { - dgd.Spec.Components[0].Experimental.Checkpoint.CheckpointRef = checkpointRef - withLPX, err := newTestDGDCheckpointsReconciler(reconciler).Reconcile(ctx, dgd) - assert.NoError(t, err) - assert.Equal(t, result, withLPX) - } } func TestDGDCheckpointsReconciler_ExplicitRestoreIsPortableAcrossDGDIdentity(t *testing.T) { From 12110e0033006732ec6bc415967674b3da8e4aa8 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Wed, 23 Sep 2026 23:48:33 +0300 Subject: [PATCH 126/196] fix(operator): clarify component provider rejection (#558) Document why the component pathway retains its guard despite admission validation. Use a component-agnostic Ready reason and update both existing expectations without changing rejection behavior. Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_component_program.go | 4 ++-- .../operator/internal/controller/dgd_workload_program_test.go | 4 ++-- .../controller/dynamographdeployment_controller_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_component_program.go b/deploy/operator/internal/controller/dgd_component_program.go index 354e432381eb..59d0e9902bd9 100644 --- a/deploy/operator/internal/controller/dgd_component_program.go +++ b/deploy/operator/internal/controller/dgd_component_program.go @@ -71,7 +71,7 @@ func (p *componentProgram) Reconcile( programResult = newWorkloadProgramResult(req.DGD) clearComponentGPUShapes(programResult.Status.Components) - // Reject components that the component pathway cannot materialize. + // Admission prevents this combination; guard previously stored or admission-bypassed objects. for i := range req.DGD.Spec.Components { component := &req.DGD.Spec.Components[i] if !component.ManagedByExternalController() { @@ -82,7 +82,7 @@ func (p *componentProgram) Reconcile( component.ComponentName, component.ComponentType, ) - programResult.Fail(req.DGD.Generation, "LPXRejected", err) + programResult.Fail(req.DGD.Generation, "UnsupportedComponent", err) return programResult, reconcile.TerminalError(err) } diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index b1425ff2f542..e956a9650378 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -134,11 +134,11 @@ func TestComponentProgramRejectsExternallyManagedComponents(t *testing.T) { result, err := (&componentProgram{}).Reconcile(t.Context(), workloadProgramRequest{DGD: dgd}) require.ErrorIs(t, err, reconcile.TerminalError(nil)) - t.Log("Verify the program returns the public LPX rejection without a terminal-error prefix") + t.Log("Verify the program returns the component rejection without a terminal-error prefix") ready := meta.FindStatusCondition(result.Status.Conditions, "Ready") require.NotNil(t, ready) assert.Equal(t, metav1.ConditionFalse, ready.Status) - assert.Equal(t, "LPXRejected", ready.Reason) + assert.Equal(t, "UnsupportedComponent", ready.Reason) assert.Equal(t, `component "serving" of type "lpx" requires the Grove workload provider`, ready.Message) } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 6ffcbf78e69c..54538d991ded 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -183,7 +183,7 @@ func TestDynamoGraphDeploymentReconcilePersistsComponentProgramLPXRejection(t *t ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready") require.NotNil(t, ready) require.Equal(t, metav1.ConditionFalse, ready.Status) - require.Equal(t, "LPXRejected", ready.Reason) + require.Equal(t, "UnsupportedComponent", ready.Reason) require.Equal(t, `component "serving" of type "lpx" requires the Grove workload provider`, ready.Message) } From 2837cefa45894ba4258090856d68554580e31e3b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 24 Sep 2026 00:05:22 +0300 Subject: [PATCH 127/196] refactor(operator): simplify Grove component projection (#559) Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_projection.go | 23 ------------------- .../internal/controller/dgd_grove_program.go | 18 +++++++++++++-- .../dgd_grove_workloads_reconciler_test.go | 20 ++++++++-------- .../controller/dgd_lpx_handoff_test.go | 14 +++++------ .../dgd_worker_rollout_reconciler_test.go | 2 +- .../controller/dgd_workload_program_test.go | 4 ++-- .../dynamographdeployment_controller_test.go | 10 ++++---- .../internal/controller/upgrade_test.go | 4 ++-- 8 files changed, 43 insertions(+), 52 deletions(-) delete mode 100644 deploy/operator/internal/controller/dgd_grove_lpx_projection.go diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_projection.go b/deploy/operator/internal/controller/dgd_grove_lpx_projection.go deleted file mode 100644 index befb794b536a..000000000000 --- a/deploy/operator/internal/controller/dgd_grove_lpx_projection.go +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "slices" - - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" -) - -// projectOrdinaryGroveDeployment returns the component subset materialized by -// the DGD-owned Grove PodCliqueSet. The source is not mutated. -func projectOrdinaryGroveDeployment(source *v1beta1.DynamoGraphDeployment) *v1beta1.DynamoGraphDeployment { - ordinary := source.DeepCopy() - ordinary.Spec.Components = slices.DeleteFunc( - ordinary.Spec.Components, - func(component v1beta1.DynamoComponentDeploymentSharedSpec) bool { - return component.ManagedByExternalController() - }, - ) - return ordinary -} diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 97b83f24c3fe..af742b0fb302 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -20,6 +20,7 @@ package controller import ( "context" "fmt" + "slices" "time" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -106,7 +107,7 @@ func (p *groveProgram) Reconcile( programResult.Fail(req.DGD.Generation, reason, retErr) } if ordinaryDGD == nil { - ordinaryDGD = projectOrdinaryGroveDeployment(req.DGD) + ordinaryDGD = projectWithoutExternallyManagedComponents(req.DGD) } p.topology.Reconcile(ctx, ordinaryDGD, &programResult) }() @@ -143,7 +144,7 @@ func (p *groveProgram) Reconcile( if err != nil { return programResult, err } - ordinaryDGD = projectOrdinaryGroveDeployment(req.DGD) + ordinaryDGD = projectWithoutExternallyManagedComponents(req.DGD) previousRestart := programResult.Status.Restart restart := p.restart.Resolve( @@ -209,3 +210,16 @@ func (p *groveProgram) Reconcile( programResult.applyReconcileResult(req.DGD.Generation, result) return programResult, nil } + +// projectWithoutExternallyManagedComponents copies source without externally managed components. +// The source must be non-nil and is not mutated. +func projectWithoutExternallyManagedComponents(source *nvidiacomv1beta1.DynamoGraphDeployment) *nvidiacomv1beta1.DynamoGraphDeployment { + projected := source.DeepCopy() + projected.Spec.Components = slices.DeleteFunc( + projected.Spec.Components, + func(component nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component.ManagedByExternalController() + }, + ) + return projected +} diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 013fffea51f3..885a82a650e0 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -120,7 +120,7 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { result, err := reconciler.newGroveProgram().workloads.Reconcile( context.Background(), dgd, - projectOrdinaryGroveDeployment(dgd), + projectWithoutExternallyManagedComponents(dgd), nil, nil, ) @@ -141,7 +141,7 @@ func TestGroveWorkloadsReconcilerUsesStableReadinessWithoutOrdinaryPodCliqueSet( const dgdName = "graph" source := newLPXHandoffSource(t, "node-local-v2-lpu-only") source.Name, source.Namespace, source.UID = dgdName, corev1.NamespaceDefault, "dgd-uid" - ordinary := projectOrdinaryGroveDeployment(source) + ordinary := projectWithoutExternallyManagedComponents(source) owned := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), Namespace: source.Namespace, @@ -337,7 +337,7 @@ func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFai t.Log("Reconcile the full workload transition") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectWithoutExternallyManagedComponents(observedDGD), nil, nil) t.Log("Verify the failed PCS sync leaves the persisted DGD hash unchanged") require.Error(t, err) @@ -424,7 +424,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Persist the PCS suffix without projecting the DGD hash from the write receipt") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectWithoutExternallyManagedComponents(observedDGD), nil, nil) require.NoError(t, err) t.Log("Verify the write receipt leaves the parent hash unchanged") @@ -443,7 +443,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Observe the suffix on a later reconcile, then reject the parent projection") freshDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectWithoutExternallyManagedComponents(freshDGD), nil, nil) require.Error(t, err) assert.Equal(t, 1, pcsUpdateCalls) assert.Equal(t, 1, dgdUpdateCalls) @@ -452,7 +452,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( failDGDUpdate = false freshDGD = &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectWithoutExternallyManagedComponents(freshDGD), nil, nil) require.NoError(t, err) t.Log("Verify the retry commits the target hash without rewriting the PCS") @@ -465,7 +465,7 @@ func TestGroveWorkloadsReconciler_RecoversWorkerHashCommitAfterPodCliqueSetSync( t.Log("Verify the completed transition is idempotent") idempotentDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), idempotentDGD)) - _, err = workloads.Reconcile(context.Background(), idempotentDGD, projectOrdinaryGroveDeployment(idempotentDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), idempotentDGD, projectWithoutExternallyManagedComponents(idempotentDGD), nil, nil) require.NoError(t, err) assert.Equal(t, 1, pcsUpdateCalls) assert.Equal(t, 2, dgdUpdateCalls) @@ -749,7 +749,7 @@ func TestGroveWorkloadsReconciler_SkipsHashObservationWhenHashIsCurrent(t *testi t.Log("Reconcile: hash observation block must be skipped entirely when needsCommit is false") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectWithoutExternallyManagedComponents(observedDGD), nil, nil) require.NoError(t, err) assert.Zero(t, dgdUpdateCalls, "DGD must not be updated when the hash annotation is already current") @@ -828,14 +828,14 @@ func TestGroveWorkloadsReconciler_DefersHashCommitUntilPCSWriteObserved(t *testi t.Log("First reconcile writes the PCS; commit must be deferred") observedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), observedDGD)) - _, err = workloads.Reconcile(context.Background(), observedDGD, projectOrdinaryGroveDeployment(observedDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), observedDGD, projectWithoutExternallyManagedComponents(observedDGD), nil, nil) require.NoError(t, err) assert.Zero(t, dgdUpdateCalls, "hash annotation must not be committed on the reconcile that writes the PCS") t.Log("Second reconcile is a no-op PCS sync; commit must proceed") freshDGD := &nvidiacomv1beta1.DynamoGraphDeployment{} require.NoError(t, kubeClient.Get(context.Background(), client.ObjectKeyFromObject(dgd), freshDGD)) - _, err = workloads.Reconcile(context.Background(), freshDGD, projectOrdinaryGroveDeployment(freshDGD), nil, nil) + _, err = workloads.Reconcile(context.Background(), freshDGD, projectWithoutExternallyManagedComponents(freshDGD), nil, nil) require.NoError(t, err) assert.Equal(t, 1, dgdUpdateCalls, "hash annotation must be committed once the PCS write is observed") diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 8e8c0723012c..ee093c15aa4c 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -64,7 +64,7 @@ func newLPXHandoffFixture(t *testing.T, fixture string) (*v1alpha1.LPXGraphDeplo return child, source, kube } -func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { +func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing.T) { for _, test := range []struct { name string longNames bool @@ -94,7 +94,7 @@ func TestOrdinaryGroveProjectionExcludesLPXWithoutMutatingSource(t *testing.T) { }) } before := source.DeepCopy() - ordinary := projectOrdinaryGroveDeployment(source) + ordinary := projectWithoutExternallyManagedComponents(source) require.Nil(t, ordinary.GetComponentByName(lpxName)) require.Len(t, ordinary.Spec.Components, len(source.Spec.Components)-1) @@ -341,7 +341,7 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { program := (&DynamoGraphDeploymentReconciler{ Client: kube, RuntimeConfig: &commoncontroller.RuntimeConfig{}, }).newGroveProgram() - ordinaryDGD := projectOrdinaryGroveDeployment(source) + ordinaryDGD := projectWithoutExternallyManagedComponents(source) progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { return resolveCompositeGroveRestartProgress( ctx, source, ordinaryDGD, inProgress, program.restartProgress, program.lpxRestartProgress, @@ -490,7 +490,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { program := (&DynamoGraphDeploymentReconciler{ Client: observed, RuntimeConfig: &commoncontroller.RuntimeConfig{}, }).newGroveProgram() - ordinaryDGD := projectOrdinaryGroveDeployment(source) + ordinaryDGD := projectWithoutExternallyManagedComponents(source) progress := func(ctx context.Context, source *v1beta1.DynamoGraphDeployment, inProgress []string) []string { return resolveCompositeGroveRestartProgress( ctx, source, ordinaryDGD, inProgress, program.restartProgress, program.lpxRestartProgress, @@ -553,7 +553,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1))}, v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, ) - ordinaryDGD = projectOrdinaryGroveDeployment(source) + ordinaryDGD = projectWithoutExternallyManagedComponents(source) child.Status.Components["draft"] = v1alpha1.LPXComponentStatus{ ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 2}, Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, @@ -734,7 +734,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.NoError(t, err) require.Equal(t, v1beta1.DGDStatePending, result.Status.State) pcs := &grovev1alpha1.PodCliqueSet{} - ordinaryDGD := projectOrdinaryGroveDeployment(source) + ordinaryDGD := projectWithoutExternallyManagedComponents(source) key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinaryDGD.Name, ordinaryDGD.Spec.Components)} require.NoError(t, kube.Get(t.Context(), key, pcs)) require.True(t, metav1.IsControlledBy(pcs, source)) @@ -809,7 +809,7 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { t.Log("Removing the last ordinary component deletes its PCS without disturbing the child") source.Spec.Components = source.Spec.Components[:1] for range 2 { - _, err = program.workloads.Reconcile(t.Context(), source, projectOrdinaryGroveDeployment(source), nil, nil) + _, err = program.workloads.Reconcile(t.Context(), source, projectWithoutExternallyManagedComponents(source), nil, nil) require.NoError(t, err) require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), key, pcs))) } diff --git a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go index 6a2295d11fd9..f62756a1d385 100644 --- a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go @@ -270,7 +270,7 @@ func TestGroveRenderDeploymentWorkerHashSuffix(t *testing.T) { before := dgd.DeepCopy() t.Log("Render the Grove deployment") - rendered, err := groveRenderDeployment(projectOrdinaryGroveDeployment(dgd), dgd, nil, tt.workerHashSuffix) + rendered, err := groveRenderDeployment(projectWithoutExternallyManagedComponents(dgd), dgd, nil, tt.workerHashSuffix) require.NoError(t, err) worker := rendered.GetComponentByName("worker") require.NotNil(t, worker) diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index e956a9650378..63bad19b48c8 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -485,14 +485,14 @@ func TestGroveRendererFailsWhenResolvedDRADependencyDisappears(t *testing.T) { ) t.Log("Verify the renderer initially publishes the full multinode shape") - rendered, err := renderer.Render(t.Context(), dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + rendered, err := renderer.Render(t.Context(), dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerEngine) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerReplica) t.Log("Delete the dependency without changing DGD generation and render again") require.NoError(t, kubeClient.Delete(t.Context(), claimTemplate)) - _, err = renderer.Render(t.Context(), dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + _, err = renderer.Render(t.Context(), dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.ErrorContains(t, err, "ResourceClaimTemplate default/gpu-template") } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 54538d991ded..887b2b009539 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -1161,7 +1161,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { result, err := reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, - projectOrdinaryGroveDeployment(dgd), + projectWithoutExternallyManagedComponents(dgd), nil, nil, ) @@ -1203,7 +1203,7 @@ func TestGroveWorkloadsReconciler_Reconcile(t *testing.T) { result, err = reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, - projectOrdinaryGroveDeployment(dgd), + projectWithoutExternallyManagedComponents(dgd), nil, nil, ) @@ -1300,7 +1300,7 @@ func TestGroveWorkloadsReconciler_UsesPreservedAlphaServiceIngress(t *testing.T) _, err := reconciler.newGroveProgram().workloads.Reconcile( ctx, dgd, - projectOrdinaryGroveDeployment(dgd), + projectWithoutExternallyManagedComponents(dgd), nil, nil, ) @@ -1387,7 +1387,7 @@ func TestGroveWorkloadRendererRenderPreservesLegacyWorkerSelectors(t *testing.T) nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) generatedPCS := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -1685,7 +1685,7 @@ func TestGroveWorkloadRendererRenderKeepsNativeWorkerSelectors(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) renderDGD := renderedPCS.renderDeployment prefill := renderDGD.GetComponentByName("prefill") diff --git a/deploy/operator/internal/controller/upgrade_test.go b/deploy/operator/internal/controller/upgrade_test.go index 5d1844a1e5c7..57001c81dd19 100644 --- a/deploy/operator/internal/controller/upgrade_test.go +++ b/deploy/operator/internal/controller/upgrade_test.go @@ -806,7 +806,7 @@ spec: &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) pcs := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -932,7 +932,7 @@ func TestGroveNativeWorkerIdentityLabelsStayNative(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectOrdinaryGroveDeployment(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) desired := renderedPCS.desired renderDGD := renderedPCS.renderDeployment From 265ce0377029b5fc93fefa5b4a416f5fe23556c8 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 21:15:24 +0000 Subject: [PATCH 128/196] perf(operator): avoid deep-copying Grove projections Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_grove_program.go | 8 ++++---- .../operator/internal/controller/dgd_lpx_handoff_test.go | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index af742b0fb302..f91593233ddb 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -212,14 +212,14 @@ func (p *groveProgram) Reconcile( } // projectWithoutExternallyManagedComponents copies source without externally managed components. -// The source must be non-nil and is not mutated. +// The source must be non-nil; shared nested data must not be mutated by callers. func projectWithoutExternallyManagedComponents(source *nvidiacomv1beta1.DynamoGraphDeployment) *nvidiacomv1beta1.DynamoGraphDeployment { - projected := source.DeepCopy() + projected := *source projected.Spec.Components = slices.DeleteFunc( - projected.Spec.Components, + slices.Clone(projected.Spec.Components), func(component nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) bool { return component.ManagedByExternalController() }, ) - return projected + return &projected } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index ee093c15aa4c..b002dcdcf2f8 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -147,9 +147,8 @@ func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing require.Equal(t, "missing ordinary topology", condition.Message) } - t.Log("Keep the projected copy independently mutable after all observations") + t.Log("Keep the source unchanged after observations and projected component-slice edits") require.Equal(t, before, source) - ordinary.Annotations["projection-only"] = "independent-copy" if len(ordinary.Spec.Components) > 0 { ordinary.Spec.Components[0].ComponentName = "ordinary-copy" } From b201c3dbdc8497d94c941547c1f6b98ccf80f912 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 23 Sep 2026 21:35:44 +0000 Subject: [PATCH 129/196] fix(operator): require parallel LPX restarts Signed-off-by: Andrew Paprotsky --- .../v1alpha1/dynamographdeployment_types.go | 1 + .../v1beta1/dynamographdeployment_types.go | 1 + .../nvidia.com_dynamographdeployments.yaml | 4 + .../controller/dgd_grove_lpx_restart_test.go | 2 +- .../controller/dgd_lpx_handoff_test.go | 95 +++++++++++++++++-- .../internal/controller/lpx/render_test.go | 5 + ...hdeployment_lpx_validation_envtest_test.go | 68 ++++++++++++- .../dynamo-graph-deployment.mdx | 1 + 8 files changed, 168 insertions(+), 9 deletions(-) diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 162cff121dfc..4a0770e2a0ef 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -84,6 +84,7 @@ type PlacementStatus struct { // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. // +kubebuilder:validation:XValidation:rule="oldSelf.hasValue() || !has(self.restart)",message="spec.restart must be unset on create; set spec.restart.id after creation to request a restart",optionalOldSelf=true +// +kubebuilder:validation:XValidation:rule="!has(self.restart) || !has(self.services) || !self.services.exists(name, has(self.services[name].componentType) && self.services[name].componentType == 'lpx') || (has(self.restart.strategy) && has(self.restart.strategy.type) && self.restart.strategy.type == 'Parallel')",message="spec.restart.strategy.type must be Parallel when the graph contains LPX components" type DynamoGraphDeploymentSpec struct { // ProviderOverride configures the root resource generated by the selected // graph-level workload provider. For Grove it uses apiVersion diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 583742860002..aa29154b4b48 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -25,6 +25,7 @@ import ( // DynamoGraphDeploymentSpec defines the desired state of a DynamoGraphDeployment. // +kubebuilder:validation:XValidation:rule="oldSelf.hasValue() || !has(self.restart)",message="spec.restart must be unset on create; set spec.restart.id after creation to request a restart",optionalOldSelf=true +// +kubebuilder:validation:XValidation:rule="!has(self.restart) || !has(self.components) || !self.components.exists(c, has(c.type) && c.type == 'lpx') || (has(self.restart.strategy) && has(self.restart.strategy.type) && self.restart.strategy.type == 'Parallel')",message="spec.restart.strategy.type must be Parallel when the graph contains LPX components" type DynamoGraphDeploymentSpec struct { // providerOverride configures the root resource generated by the selected // graph-level workload provider. For Grove it uses apiVersion diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 52c42ec7f8ba..3689ab4b4258 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20172,6 +20172,8 @@ spec: - message: spec.restart must be unset on create; set spec.restart.id after creation to request a restart optionalOldSelf: true rule: oldSelf.hasValue() || !has(self.restart) + - message: spec.restart.strategy.type must be Parallel when the graph contains LPX components + rule: '!has(self.restart) || !has(self.services) || !self.services.exists(name, has(self.services[name].componentType) && self.services[name].componentType == ''lpx'') || (has(self.restart.strategy) && has(self.restart.strategy.type) && self.restart.strategy.type == ''Parallel'')' status: description: Status reflects the current observed state of this graph deployment. properties: @@ -37884,6 +37886,8 @@ spec: - message: spec.restart must be unset on create; set spec.restart.id after creation to request a restart optionalOldSelf: true rule: oldSelf.hasValue() || !has(self.restart) + - message: spec.restart.strategy.type must be Parallel when the graph contains LPX components + rule: '!has(self.restart) || !has(self.components) || !self.components.exists(c, has(c.type) && c.type == ''lpx'') || (has(self.restart.strategy) && has(self.restart.strategy.type) && self.restart.strategy.type == ''Parallel'')' status: description: status reflects the current observed state of this graph deployment. properties: diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go index 2a1b2e391c9c..86c21506c997 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -15,7 +15,7 @@ import ( ) func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { - for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeSequential, v1beta1.RestartStrategyTypeParallel} { + for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeParallel} { t.Run(string(strategy), func(t *testing.T) { t.Log("Observe a Ready child after delivering the selected LPX restart") ctx := t.Context() diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index b002dcdcf2f8..cae1bdddb261 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -328,7 +328,7 @@ func TestLPXFailureProjectionRequiresCurrentCondition(t *testing.T) { } func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { - for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeSequential, v1beta1.RestartStrategyTypeParallel} { + for _, strategy := range []v1beta1.RestartStrategyType{v1beta1.RestartStrategyTypeParallel} { t.Run(string(strategy), func(t *testing.T) { t.Log("Request an LPX-only restart without requiring an ordinary PCS") _, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") @@ -392,6 +392,92 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { } } +func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testing.T) { + t.Log("Request one parallel restart of two independent LPX workloads and a frontend") + child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + require.NoError(t, rbacv1.AddToScheme(kube.Scheme())) + second := source.Spec.Components[0].DeepCopy() + second.ComponentName = "second" + source.Spec.Components = append(source.Spec.Components, *second, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend"}}}}, + }) + source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) + source.Spec.Restart = &v1beta1.Restart{ID: "parallel-restart", Strategy: &v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeParallel}} + require.NoError(t, kube.Update(t.Context(), source)) + config := &configv1alpha1.OperatorConfiguration{} + config.Namespace.Restricted = source.Namespace + program := (&DynamoGraphDeploymentReconciler{ + Client: kube, Config: config, Recorder: events.NewFakeRecorder(100), + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, + }).newGroveProgram() + + t.Log("Persist selection of all three components before publishing their restart") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, []string{"frontend", "lpx", "second"}, result.Status.Restart.InProgress) + source.Status = result.Status + require.NoError(t, kube.Status().Update(t.Context(), source)) + + t.Log("Deliver the same token to LPX and the ordinary Pod template while every workload is pending") + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.DGDStatePending, result.Status.State) + require.Equal(t, []string{"frontend", "lpx", "second"}, result.Status.Restart.InProgress) + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Equal(t, source.Spec.Restart.ID, child.Annotations[dynamo.LPXRestartAnnotation]) + require.Empty(t, child.Status.Components) + ordinary := projectWithoutExternallyManagedComponents(source) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKey{ + Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), + }, pcs)) + require.Len(t, pcs.Spec.Template.Cliques, 1) + require.Equal(t, source.Spec.Restart.ID, pcs.Spec.Template.Cliques[0].Annotations[consts.RestartAnnotation]) + require.Nil(t, pcs.Status.ObservedGeneration) + source.Status = result.Status + require.NoError(t, kube.Status().Update(t.Context(), source)) + + t.Log("One ready LPX workload cannot finish the restart while the second is pending") + child.Status.ObservedGeneration = child.Generation + child.Status.Components = map[string]v1alpha1.LPXComponentStatus{"lpx": { + ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 1, AvailableReplicas: ptr.To(int32(1))}, + Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, + }} + require.NoError(t, kube.Status().Update(t.Context(), child)) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.RestartPhaseRestarting, result.Status.Restart.Phase) + require.Contains(t, result.Status.Restart.InProgress, "second") + + t.Log("Both ready LPX workloads still wait for the ordinary component") + child.Status.Components["second"] = child.Status.Components["lpx"] + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: "Ready", + }) + require.NoError(t, kube.Status().Update(t.Context(), child)) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.RestartPhaseRestarting, result.Status.Restart.Phase) + require.Equal(t, []string{"frontend"}, result.Status.Restart.InProgress) + source.Status = result.Status + require.NoError(t, kube.Status().Update(t.Context(), source)) + + t.Log("Complete only after Grove observes the frontend's ready replacement") + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) + require.NoError(t, kube.Update(t.Context(), pcs)) + require.NoError(t, kube.Create(t.Context(), &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(ordinary, "frontend"), Namespace: source.Namespace, Generation: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, + })) + result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + require.Equal(t, v1beta1.RestartPhaseCompleted, result.Status.Restart.Phase) + require.Empty(t, result.Status.Restart.InProgress) +} + func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { t.Log("Seed the shared child's current readiness") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-specdecode") @@ -462,8 +548,6 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { name string strategy v1beta1.RestartStrategy }{ - {"draft first", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeSequential, Order: []string{"draft", "lpx"}}}, - {"target first", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeSequential, Order: []string{"lpx", "draft"}}}, {"parallel", v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeParallel}}, } { t.Run(scenario.name, func(t *testing.T) { @@ -499,7 +583,7 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { source.Status.Restart = restarter.Resolve(t.Context(), source, &source.Status, progress).Status require.NotEmpty(t, source.Status.Restart.InProgress) - t.Log("Either selected member delivers one token to the same child") + t.Log("Both selected members deliver one token to the same child") handoff := &dgdLPXHandoff{client: kube} child, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) @@ -543,9 +627,6 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { require.NoError(t, kube.Status().Update(t.Context(), child)) require.Equal(t, []string{"draft"}, progress(t.Context(), source, []string{"draft", "lpx"})) require.Zero(t, pcsReads) - if scenario.strategy.Type != v1beta1.RestartStrategyTypeParallel { - return - } t.Log("A missing ordinary PCS keeps only live ordinary restart members pending") source.Spec.Components = append(source.Spec.Components, diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index b91859ad9587..fccc51b51d58 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -427,6 +427,9 @@ func TestLPXRenderingMetadata(t *testing.T) { const restartToken = "2026-09-08T00:00:00Z" t.Log("Author scheduler metadata and stale identity annotations before freezing the child") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) + second := dgd.Spec.Components[0].DeepCopy() + second.ComponentName = "independent" + dgd.Spec.Components = append(dgd.Spec.Components, *second) dgd.Labels = map[string]string{"project": "inference", "unrelated": "ignored"} dgd.Annotations["kai.scheduler/topology"] = "source-topology" dgd.Spec.Annotations = map[string]string{ @@ -439,8 +442,10 @@ func TestLPXRenderingMetadata(t *testing.T) { r.config.Discovery.Backend = configv1alpha1.DiscoveryBackendKubernetes workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) + require.Len(t, workloads, 2) pcs, resources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) require.NoError(t, err) + require.Len(t, pcs.Spec.Template.PodCliqueScalingGroupConfigs, 2) t.Log("Inherit scheduler metadata while explicit PCS metadata takes precedence") require.Equal(t, "inference", pcs.Labels["project"]) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index c1c70d9518f0..f2f3f4cf0c7c 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -8,6 +8,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" k8sptr "k8s.io/utils/ptr" ) @@ -20,7 +21,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { const alphaConductorTemplateErr = "spec.services[lpx].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" // Keep LPX inputs and oracles together without a separate admission execution path. - return []dgdAdmissionTestCase{ + tests := []dgdAdmissionTestCase{ { name: "singleton LPX with explicit conductor preserves omitted replicas", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { @@ -796,6 +797,71 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{`spec.components[0].type: Invalid value: "lpx": cannot change node topology between LPX and non-LPX after creation`}, }, } + + // Exercise restart admission and defaulting through both served API versions. + const parallelRestartErr = "spec: Invalid value: spec.restart.strategy.type must be Parallel when the graph contains LPX components" + for _, restart := range []struct { + name string + strategy *nvidiacomv1beta1.RestartStrategy + ordinary bool + addLPX bool + create bool + wantCELErr string + }{ + {name: "LPX admits Parallel restart", strategy: &nvidiacomv1beta1.RestartStrategy{Type: nvidiacomv1beta1.RestartStrategyTypeParallel}}, + {name: "LPX rejects Sequential restart", strategy: &nvidiacomv1beta1.RestartStrategy{Type: nvidiacomv1beta1.RestartStrategyTypeSequential}, wantCELErr: parallelRestartErr}, + {name: "LPX rejects omitted restart strategy", wantCELErr: parallelRestartErr}, + {name: "LPX rejects omitted restart strategy type", strategy: &nvidiacomv1beta1.RestartStrategy{}, wantCELErr: parallelRestartErr}, + {name: "LPX rejects Parallel restart on CREATE", strategy: &nvidiacomv1beta1.RestartStrategy{Type: nvidiacomv1beta1.RestartStrategyTypeParallel}, create: true, + wantCELErr: "spec: Invalid value: spec.restart must be unset on create; set spec.restart.id after creation to request a restart"}, + {name: "ordinary graph admits Sequential restart", strategy: &nvidiacomv1beta1.RestartStrategy{Type: nvidiacomv1beta1.RestartStrategyTypeSequential}, ordinary: true}, + {name: "ordinary graph admits omitted restart strategy", ordinary: true}, + {name: "ordinary graph admits omitted restart strategy type", strategy: &nvidiacomv1beta1.RestartStrategy{}, ordinary: true}, + {name: "adding LPX rejects an unchanged Sequential restart", strategy: &nvidiacomv1beta1.RestartStrategy{Type: nvidiacomv1beta1.RestartStrategyTypeSequential}, addLPX: true, wantCELErr: parallelRestartErr}, + } { + // Keep ordinary components beside LPX so the restriction covers mixed graphs. + alphaOld := alphaDGDForAdmission(nil) + betaOld := betaDGDForAdmission(nil) + if !restart.ordinary && !restart.addLPX { + alphaOld.Spec.Services["lpx"] = alphaLPXDGDForAdmission(nil).Spec.Services["lpx"] + betaOld.Spec.Components = append(betaOld.Spec.Components, betaLPXDGDForAdmission(nil).Spec.Components[0]) + } + + // Author equivalent restart requests without converting away source-version defaults. + alpha := alphaOld.DeepCopy() + beta := betaOld.DeepCopy() + alpha.Spec.Restart = &nvidiacomv1alpha1.Restart{ID: "restart-1"} + beta.Spec.Restart = &nvidiacomv1beta1.Restart{ID: "restart-1", Strategy: restart.strategy} + if restart.strategy != nil { + alpha.Spec.Restart.Strategy = &nvidiacomv1alpha1.RestartStrategy{Type: nvidiacomv1alpha1.RestartStrategyType(restart.strategy.Type)} + } + if restart.addLPX { + alphaOld.Spec.Restart = alpha.Spec.Restart.DeepCopy() + betaOld.Spec.Restart = beta.Spec.Restart.DeepCopy() + alpha.Spec.Services["lpx"] = alphaLPXDGDForAdmission(nil).Spec.Services["lpx"] + beta.Spec.Components = append(beta.Spec.Components, betaLPXDGDForAdmission(nil).Spec.Components[0]) + } + + // The shared harness submits each native object to its matching API endpoint. + for _, version := range []struct { + name string + deployment runtime.Object + old runtime.Object + }{ + {name: "v1alpha1", deployment: alpha, old: alphaOld}, + {name: "v1beta1", deployment: beta, old: betaOld}, + } { + test := dgdAdmissionTestCase{ + name: version.name + " " + restart.name, deployment: version.deployment, + oldDeployment: version.old, wantCELErr: restart.wantCELErr, + } + if restart.create { + test.oldDeployment = nil + } + tests = append(tests, test) + } + } + return tests } func betaLPXDGDForAdmission( diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 98efa2e2046d..426266bffba7 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -87,6 +87,7 @@ spec: Restart policy for the graph. Must be unset on creation; set or change `restart.id` on an existing DGD to trigger a restart. + Graphs containing LPX components require explicit `restart.strategy.type: Parallel`; Sequential and omitted strategies are rejected. From e93e77a52eb05c15fddd1d00c6943bed702cd1ec Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:52:29 +0300 Subject: [PATCH 130/196] docs(operator): correct LPX conductor and placement references Signed-off-by: Andrew Paprotsky --- .../reference/kubernetes-api/dynamo-component-deployment.mdx | 2 +- .../pages/reference/kubernetes-api/dynamo-graph-deployment.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 1be594409f6c..876c319f79aa 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -121,7 +121,7 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Named Pod-producing parts of a compound component, keyed by `name`. Multinode components must contain exactly one `leader` and one `worker`; omitted role counts default to `1` and `multinode.nodeCount - 1`, respectively. Omit the field to retain the implicit leader/worker layout. - LPX components use `agent` for Agent Pods and `conductor` for the serving runtime. Exactly one LPX component in the graph must declare a conductor role with an explicit template. In a speculative pair, the draft has only an agent role and the target declares both roles. A hybrid engine's GPU runtime uses the conductor template. + LPX components use `agent` for Agent Pods and `conductor` for the serving runtime. Independent LPX components each declare a conductor role with an explicit template. In a speculative pair, the draft has only an agent role and the target declares both roles. A hybrid engine's GPU runtime uses the conductor template. Standalone DCD roles cannot use LPX templates or DGD-only provider overrides. See [ComponentRoleSpec](full-api-reference.mdx#componentrolespec). diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index 426266bffba7..a2d93c805efd 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -225,7 +225,7 @@ The operator maintains observed state under `status`. - Scheduler placement score and reporting state. For LPX workloads, this field mirrors `status.lpx.placement`; generic placement scoring is not yet populated. + Scheduler placement score and reporting state. The schema is available, but the controller does not yet populate this field. From 91d4a73352e157fb0ef681e81edfa02f07062292 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:08:45 +0000 Subject: [PATCH 131/196] refactor(operator): remove unused pod spec helpers Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/common/podspec.go | 50 ---------------------- 1 file changed, 50 deletions(-) diff --git a/deploy/operator/internal/common/podspec.go b/deploy/operator/internal/common/podspec.go index 9157e4e96fd5..5468a9c2b194 100644 --- a/deploy/operator/internal/common/podspec.go +++ b/deploy/operator/internal/common/podspec.go @@ -16,53 +16,3 @@ func FindContainerByName(containers []corev1.Container, name string) *corev1.Con } return nil } - -// HasPodResourceClaim reports whether the pod spec references a claim by name. -func HasPodResourceClaim(podSpec *corev1.PodSpec, name string) bool { - for i := range podSpec.ResourceClaims { - if podSpec.ResourceClaims[i].Name == name { - return true - } - } - return false -} - -// HasContainerResourceClaim reports whether the container references a claim by name. -func HasContainerResourceClaim(container *corev1.Container, name string) bool { - for i := range container.Resources.Claims { - if container.Resources.Claims[i].Name == name { - return true - } - } - return false -} - -// HasVolume reports whether a volume exists by name. -func HasVolume(volumes []corev1.Volume, name string) bool { - for i := range volumes { - if volumes[i].Name == name { - return true - } - } - return false -} - -// HasVolumeMount reports whether a volume mount exists by name and path. -func HasVolumeMount(mounts []corev1.VolumeMount, name, mountPath string) bool { - for i := range mounts { - if mounts[i].Name == name && mounts[i].MountPath == mountPath { - return true - } - } - return false -} - -// HasEnvValue reports whether an environment variable exists with the given value. -func HasEnvValue(env []corev1.EnvVar, name, value string) bool { - for i := range env { - if env[i].Name == name && env[i].Value == value { - return true - } - } - return false -} From c9fab88f205e1cf970ceaed1ec1d03b0c7bb3ca4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:10:23 +0000 Subject: [PATCH 132/196] refactor(operator): remove unused LPX generation annotation Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_lpx_handoff.go | 3 --- deploy/operator/internal/controller/dgd_lpx_handoff_test.go | 2 -- deploy/operator/internal/controller/lpx/controller_test.go | 5 ++--- deploy/operator/internal/controller/lpx/render.go | 1 - deploy/operator/internal/controller/lpx/render_test.go | 3 +-- deploy/operator/internal/dynamo/lpx/model_projection.go | 2 -- 6 files changed, 3 insertions(+), 13 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index f803e9f65c9f..e32c2ee656cd 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -6,12 +6,10 @@ package controller import ( "context" "fmt" - "strconv" v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -63,7 +61,6 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra } child.Spec.InputRevision = revision metav1.SetMetaDataAnnotation(&child.ObjectMeta, dynamo.LPXRestartAnnotation, restart) - metav1.SetMetaDataAnnotation(&child.ObjectMeta, lpx.DGDGenerationAnnotation, strconv.FormatInt(source.Generation, 10)) if exists { err = r.client.Update(ctx, child) } else { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index cae1bdddb261..064af15574ff 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -17,7 +17,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -535,7 +534,6 @@ func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { updated, err := handoff.Reconcile(t.Context(), source) require.NoError(t, err) require.NotEqual(t, beforeChild.Spec.InputRevision, updated.Spec.InputRevision) - require.Equal(t, strconv.FormatInt(source.Generation, 10), updated.Annotations[lpx.DGDGenerationAnnotation]) require.Equal(t, beforeChild.Status, updated.Status) // Fake clients do not advance generations; the isolated API test covers that transition. updated.Generation++ diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index b4c40f80ff63..24f4e84efe98 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -970,7 +970,6 @@ func TestLPXWorkloadOrderAndUnrelatedEditsPreservePublication(t *testing.T) { afterSelected := resolveLPXTestWorkload(t, r.modelRegistry, t.Context(), child, dgd) afterPCS := renderLPXTestPodCliqueSet(t, t.Context(), r, child, dgd, afterSelected) require.Equal(t, beforePCS, afterPCS) - require.NotContains(t, afterPCS.Annotations, lpx.DGDGenerationAnnotation) observed := make(map[string]*lpxv1alpha1.LPUPipelineRequest, len(beforeRequests.Items)) for i := range beforeRequests.Items { request := &beforeRequests.Items[i] @@ -1631,7 +1630,7 @@ func TestLPXReconcileUsesOneInputSnapshot(t *testing.T) { return delegated.Update(ctx, dgd) } require.NoError(t, delegated.Get(ctx, client.ObjectKeyFromObject(child), child)) - child.Annotations[lpx.DGDGenerationAnnotation] = "999" + metav1.SetMetaDataAnnotation(&child.ObjectMeta, "example.com/concurrent-edit", "changed") return delegated.Update(ctx, child) } } @@ -2653,7 +2652,7 @@ func newLPXTestDeployment(t *testing.T, dgd *v1beta1.DynamoGraphDeployment) *v1a return &v1alpha1.LPXGraphDeployment{ TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.GroupVersion.String(), Kind: "LPXGraphDeployment"}, ObjectMeta: metav1.ObjectMeta{Name: dgd.Name, Namespace: dgd.Namespace, UID: types.UID("lpx-" + string(dgd.UID)), Generation: 1, - Annotations: map[string]string{lpx.DGDGenerationAnnotation: strconv.FormatInt(dgd.Generation, 10)}, + Annotations: map[string]string{}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(dgd, v1beta1.DynamoGraphDeploymentGVK)}, }, Spec: v1alpha1.LPXGraphDeploymentSpec{InputRevision: revision}, diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 2679b92d9f02..605df0c9fb99 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -178,7 +178,6 @@ func stampDeploymentIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grove if *annotations == nil { *annotations = make(map[string]string) } - delete(*annotations, lpx.DGDGenerationAnnotation) maps.Copy(*annotations, identity) } stampOwnerLabel := func(object client.Object) { diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index fccc51b51d58..0af54c36cc48 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -434,7 +434,7 @@ func TestLPXRenderingMetadata(t *testing.T) { dgd.Annotations["kai.scheduler/topology"] = "source-topology" dgd.Spec.Annotations = map[string]string{ "kai.scheduler/topology": "explicit-topology", - lpx.DGDUIDAnnotation: "stale", lpx.DGDGenerationAnnotation: "stale", + lpx.DGDUIDAnnotation: "stale", } child.Annotations[dynamo.LPXRestartAnnotation] = restartToken dgd.Generation++ @@ -471,7 +471,6 @@ func TestLPXRenderingMetadata(t *testing.T) { for _, annotations := range annotationMaps { require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) require.Equal(t, string(child.UID), annotations[deploymentUIDAnnotation]) - require.NotContains(t, annotations, lpx.DGDGenerationAnnotation) require.NotContains(t, annotations, "lpx.nvidia.com/deployment-generation") require.NotContains(t, annotations, "lpx.nvidia.com/input-revision") } diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index b11285b7371d..f483aff02165 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -17,8 +17,6 @@ import ( const ( // WorkloadDigestAnnotation records the immutable Dynamo workload projection digest on rendered objects. WorkloadDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-workload-digest" - // DGDGenerationAnnotation records the source DGD generation on rendered objects. - DGDGenerationAnnotation = "scheduling.lpu.nvidia.com/dgd-generation" // DGDUIDAnnotation records the source DGD UID on rendered objects. DGDUIDAnnotation = "scheduling.lpu.nvidia.com/dgd-uid" // WorkloadModeAnnotation records the projected LPX workload mode on rendered objects. From 443b81604c009db9f03a6474c6f1525617500f73 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:10:48 +0000 Subject: [PATCH 133/196] fix(operator): bind LPX allocation before authored references Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 2 +- .../internal/dynamo/lpx/runtime_agent.go | 12 ++++++- .../internal/dynamo/lpx/runtime_agent_test.go | 34 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 0af54c36cc48..1f68e402f11f 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -629,7 +629,7 @@ func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { var hash string for _, clique := range pcs.Spec.Template.Cliques { if clique.Name == plan.ConductorTemplate { - require.Equal(t, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}, clique.Spec.PodSpec.Containers[0].Env[0]) + require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, corev1.EnvVar{Name: "LPX_MODEL_PATH", Value: filepath.Join("/nfs", buildID)}) require.Contains(t, clique.Spec.PodSpec.Containers[0].Env, env) hash = clique.Annotations[v1alpha1.AnnotationExtraResourcesHash] } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent.go b/deploy/operator/internal/dynamo/lpx/runtime_agent.go index 70daf3a9961f..a4b5473437cc 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent.go @@ -23,7 +23,17 @@ func configureNodeLocalConductorRuntime( // Bind placement data without interpreting the template's executable or arguments. conductor := common.FindContainerByName(conductorPodSpec.Containers, commonconsts.MainContainerName) conductor.Name = dynamov1beta1.ComponentRoleLPXConductor - setContainerEnv(conductor, corev1.EnvVar{Name: allocationEnvVar, Value: allocation}) + + // Kubernetes expands environment references in order; publish allocation before authored bindings. + env := make([]corev1.EnvVar, 0, len(conductor.Env)+1) + env = append(env, corev1.EnvVar{Name: allocationEnvVar, Value: allocation}) + for _, variable := range conductor.Env { + if variable.Name != allocationEnvVar { + env = append(env, variable) + } + } + conductor.Env = env + retargetMainContainerReferences(conductorPodSpec, conductor) } diff --git a/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go b/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go index 6c1f81270a7e..829f81c38174 100644 --- a/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go +++ b/deploy/operator/internal/dynamo/lpx/runtime_agent_test.go @@ -10,6 +10,7 @@ import ( "slices" "testing" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -141,6 +142,39 @@ func TestRuntimePreservesAuthoredStartup(t *testing.T) { } } +func TestAllocationPrecedesAuthoredReferences(t *testing.T) { + for _, test := range []struct { + name string + bindings []corev1.EnvVar + }{ + {name: "absent"}, + {name: "existing", bindings: []corev1.EnvVar{{Name: allocationEnvVar, Value: "stale"}}}, + {name: "duplicates", bindings: []corev1.EnvVar{ + {Name: allocationEnvVar, Value: "stale"}, + {Name: allocationEnvVar, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + t.Log("Author environment references before any stale allocation bindings") + authored := []corev1.EnvVar{ + {Name: "NOVA_ALLOCATION", Value: "$(LPX_ALLOCATION)"}, + {Name: "OTHER", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + } + pod := corev1.PodSpec{Containers: []corev1.Container{{ + Name: commonconsts.MainContainerName, Command: []string{"custom-conductor"}, Args: []string{"--workers", "$(LPX_ALLOCATION)"}, + Env: append(slices.Clone(authored), test.bindings...), + }}} + want := pod.Containers[0].DeepCopy() + want.Name = v1beta1.ComponentRoleLPXConductor + want.Env = append([]corev1.EnvVar{{Name: allocationEnvVar, Value: "agt0:agt1"}}, authored...) + + t.Log("Publish exactly one authoritative allocation before references without changing startup or other environment") + configureNodeLocalConductorRuntime(&pod, "agt0:agt1") + require.Equal(t, *want, pod.Containers[0]) + }) + } +} + func TestModelPathsPrecedeAuthoredReferences(t *testing.T) { for _, test := range []struct { name string From 8961f70b4f62195fa23c45a0fc05a868cc5781f7 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:10:48 +0000 Subject: [PATCH 134/196] refactor(operator): remove obsolete environment helper Signed-off-by: Andrew Paprotsky --- .../operator/internal/dynamo/lpx/cyborg_runtime.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go index fd1782a11171..4c0b2801037a 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_runtime.go @@ -72,17 +72,3 @@ func validateCyborgReplicas(build *Build, replicas int32) error { return nil } - -// setContainerEnv updates the first matching variable or appends a new one. -func setContainerEnv(container *corev1.Container, variables ...corev1.EnvVar) { -variables: - for _, variable := range variables { - for index := range container.Env { - if container.Env[index].Name == variable.Name { - container.Env[index] = variable - continue variables - } - } - container.Env = append(container.Env, variable) - } -} From 70530a169c339af8067ee5c06de7feb0c5a2597e Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:11:55 +0000 Subject: [PATCH 135/196] refactor(operator): remove test-only conductor clique field Signed-off-by: Andrew Paprotsky --- .../operator/internal/controller/lpx/controller_test.go | 6 ++++-- .../internal/controller/lpx/pipeline_request_test.go | 1 - deploy/operator/internal/dynamo/lpx/materialization.go | 5 ----- .../operator/internal/dynamo/lpx/materialization_test.go | 8 +++++--- .../operator/internal/dynamo/lpx/resolve_workload_test.go | 1 - 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 24f4e84efe98..c0f74d6e09e5 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1772,7 +1772,8 @@ func TestLPXCorrectsTopLevelReplicaDrift(t *testing.T) { extraGroup.Name = grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: pcs.Name, Replica: 1}, desired.plan.ScalingGroupTemplate) extraGroup.UID = types.UID(extraGroup.Name + "-uid") extraGroup.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "1" - extraClique := getResource[*grovev1alpha1.PodClique](t, objects, desired.plan.ConductorClique).DeepCopy() + conductorName := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: 0}, desired.plan.ConductorTemplate) + extraClique := getResource[*grovev1alpha1.PodClique](t, objects, conductorName).DeepCopy() extraClique.Name = grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: extraGroup.Name, Replica: 0}, desired.plan.ConductorTemplate) extraClique.UID = types.UID(extraClique.Name + "-uid") extraClique.Labels[grovecommon.LabelPodCliqueSetReplicaIndex] = "1" @@ -3327,7 +3328,8 @@ func TestIndependentLPXWorkloadsScaleAndReportReadiness(t *testing.T) { t.Log("A missing role in the second workload leaves the first workload's readiness intact") clique := &grovev1alpha1.PodClique{} - require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: secondPlan.ConductorClique}, clique)) + conductorName := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: secondPlan.LPXScalingGroup, Replica: int(secondPlan.ReplicaIndex)}, secondPlan.ConductorTemplate) + require.NoError(t, r.Get(t.Context(), client.ObjectKey{Namespace: child.Namespace, Name: conductorName}, clique)) require.NoError(t, r.Delete(t.Context(), clique)) _, err = r.Reconcile(t.Context(), key) require.NoError(t, err) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 3f961861ef5c..5229ccd747a7 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -77,7 +77,6 @@ func TestImplicitV2LPXConductorlessGroveIdentityPublishesRequest(t *testing.T) { deployment, dgd, registry := newLPXTestDGD(t, lpx.PipelineLPX) reconciler, desired := newPreparedLPXTestReconciler(t, registry, ctx, deployment, dgd) require.Empty(t, desired.plan.ConductorTemplate) - require.Empty(t, desired.plan.ConductorClique) require.NotEmpty(t, desired.plan.CyborgClique) objects := lpxMaterializedObjects(t, reconciler, deployment, dgd, desired) diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 6caa04bcf91d..212d64d81478 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -49,8 +49,6 @@ type MaterializationPlan struct { ResourcePrefix string // PodCliqueSetName is the actual PCS used for every generated child address. PodCliqueSetName string - // ConductorClique is the materialized conductor PodClique name, if present. - ConductorClique string // ConductorTemplate is the conductor PodClique template name, if present. ConductorTemplate string // Agents contains the expected materialized Agent identities. @@ -137,9 +135,6 @@ func (p *MaterializationPlan) ForReplica(index int32) *MaterializationPlan { out.Agents = slices.Clone(p.Agents) // Materialize this replica beneath the same Grove scaling-group identity. - if p.ConductorTemplate != "" { - out.ConductorClique = materializedCliqueNameForReplica(p.LPXScalingGroup, p.ConductorTemplate, index) - } for agentIndex, agent := range out.Agents { out.Agents[agentIndex].CliqueName = materializedCliqueNameForReplica( p.LPXScalingGroup, diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index 44742fbb4c6c..b6a5d8f9b36d 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -60,9 +60,10 @@ func TestPlanMaterializationBounds(t *testing.T) { t.Log("Validate every role's Grove name budget and the last replica's Pod hostnames") lastReplica := plan.ForReplica(plan.Replicas - 1) hostnames := make([]string, 0, test.models+1) - if lastReplica.ConductorClique != "" { + if plan.ConductorTemplate != "" { require.Equal(t, "cond", plan.ConductorTemplate) - hostnames = append(hostnames, materializedPodHostname(lastReplica.ConductorClique, 0)) + clique := materializedCliqueNameForReplica(plan.LPXScalingGroup, plan.ConductorTemplate, lastReplica.ReplicaIndex) + hostnames = append(hostnames, materializedPodHostname(clique, 0)) require.LessOrEqual(t, len(pcsName)+len(lpxScalingGroupTemplateName)+len(plan.ConductorTemplate), commonconsts.MaxCombinedGroveResourceNameLength) } @@ -139,7 +140,8 @@ func TestWorkloadNames(t *testing.T) { t.Log("Keep Grove's combined names and the last replica's Pod hostnames valid") last := scoped.ForReplica(scoped.Replicas - 1) - require.Empty(t, validation.IsDNS1123Label(materializedPodHostname(last.ConductorClique, 0))) + clique := materializedCliqueNameForReplica(scoped.LPXScalingGroup, scoped.ConductorTemplate, last.ReplicaIndex) + require.Empty(t, validation.IsDNS1123Label(materializedPodHostname(clique, 0))) require.LessOrEqual(t, len(pcsName)+len(scoped.ScalingGroupTemplate)+len(scoped.ConductorTemplate), commonconsts.MaxCombinedGroveResourceNameLength) for _, agent := range last.Agents { require.LessOrEqual(t, len(pcsName)+len(scoped.ScalingGroupTemplate)+len(agent.TemplateName), commonconsts.MaxCombinedGroveResourceNameLength) diff --git a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go index 2aaa37221099..c24418675e9f 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go @@ -42,7 +42,6 @@ func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { require.Equal(t, "test-pcs-0-lpx", plan.LPXScalingGroup) require.Equal(t, "cond", plan.ConductorTemplate) require.Empty(t, plan.CyborgTemplate) - require.NotEmpty(t, plan.ConductorClique) t.Log("Scale Nova workloads without changing their model or workload digest") for _, replicas := range []int32{2, 10, 12, 123} { From 05ced1035476243f3c49afd85ac329b6b106a1ee Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:13:25 +0000 Subject: [PATCH 136/196] refactor(operator): remove test-only ConfigMap naming helper Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_lpx_handoff_envtest_test.go | 3 +-- deploy/operator/internal/controller/lpx/controller_test.go | 2 +- deploy/operator/internal/controller/lpx/render_test.go | 2 +- deploy/operator/internal/dynamo/lpx/lpu_configmap.go | 6 ------ deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go | 1 - 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index c23ed9dfb210..0ba7060d052f 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -20,7 +20,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" - dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" @@ -272,7 +271,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { if !assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) { return } - configMap.Name = dynamolpx.LPUConfigMapName(pcs.Name, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]) + configMap.Name = fmt.Sprintf("%s-lpu-%.16s", pcs.Name, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]) for _, resource := range []client.Object{pcs, configMap, service} { if assert.NoError(c, env.Client().Get(t.Context(), client.ObjectKeyFromObject(resource), resource)) { assert.Equal(c, metav1.NewControllerRef(child, v1alpha1.LPXGraphDeploymentGVK), metav1.GetControllerOf(resource)) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index c0f74d6e09e5..fa9c6be693cf 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1385,7 +1385,7 @@ func TestLPXMaterializationUsesDGDAndChildIdentity(t *testing.T) { root := dynamo.PCSNameForLPX(child) for _, object := range []client.Object{ &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{Name: root, Namespace: child.Namespace}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: lpx.LPUConfigMapName(root, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-lpu-%.16s", root, pcs.Spec.Template.Cliques[0].Annotations[v1alpha1.AnnotationExtraResourcesHash]), Namespace: child.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: root + "-serve", Namespace: child.Namespace}}, } { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 1f68e402f11f..885b07036815 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -635,7 +635,7 @@ func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { } } require.NotEmpty(t, hash) - configMap := getResource[*corev1.ConfigMap](t, resources, lpx.LPUConfigMapName(dynamo.PCSNameForLPX(child), hash)) + configMap := getResource[*corev1.ConfigMap](t, resources, fmt.Sprintf("%s-lpu-%.16s", dynamo.PCSNameForLPX(child), hash)) require.NotContains(t, configMap.Data, "model_config.toml") require.NotContains(t, configMap.Data, "datacenter.toml") if partitionData == nil { diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 03ef626e06af..f1e015f7a733 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -24,12 +24,6 @@ import ( const lpuConfigVolumeName = "config" -// LPUConfigMapName names the immutable runtime table using its Pod-template content hash. -// The root is the workload resource prefix, including its group when independent workloads share a PCS. -func LPUConfigMapName(root, configHash string) string { - return fmt.Sprintf("%s-lpu-%.16s", root, configHash) -} - func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index 1c8d12b63da3..a1d7f7c27f6d 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -42,7 +42,6 @@ func TestLPXRuntimeConfigNamesMatchPodIdentity(t *testing.T) { require.NoError(t, err) t.Log("Resolve the same LPU table from Pod identity and preserve each role suffix") - require.Equal(t, LPUConfigMapName(root, LPUConfigMapHash(lpu)), lpu.Name) require.Equal(t, root+"-lpu-"+LPUConfigMapHash(lpu)[:16], lpu.Name) require.Equal(t, root+"-decode-"+LPUConfigMapHash(decode)[:16], decode.Name) require.Empty(t, validation.IsDNS1123Subdomain(lpu.Name)) From 2aa53e896a0953016c8223ddc688b8ee0f6cd5df Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:09 +0000 Subject: [PATCH 137/196] fix(operator): admit LPX native scaling above one engine Signed-off-by: Andrew Paprotsky --- .../dynamocomponentdeployment_types.go | 2 +- .../dynamocomponentdeployment_types.go | 2 +- ...nvidia.com_dynamocomponentdeployments.yaml | 4 +- .../nvidia.com_dynamographdeployments.yaml | 4 +- .../controller/lpx/controller_test.go | 49 +++++++++++++++++ .../internal/dynamo/lpx/resolve_workload.go | 10 ++-- ...hdeployment_lpx_validation_envtest_test.go | 54 ++++++++++++++++--- .../dynamo-component-deployment.mdx | 2 + 8 files changed, 110 insertions(+), 17 deletions(-) diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 81fa9b0fae07..7a9d64f95da5 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -50,7 +50,7 @@ type DynamoComponentDeploymentSpec struct { DynamoComponentDeploymentSharedSpec `json:",inline"` } -// +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" +// +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (!has(self.replicas) && has(self.componentType) && self.componentType == 'lpx') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.componentType) || (has(self.componentType) && self.componentType == oldSelf.componentType)",message="componentType is immutable after it is set" // +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.componentType) && self.componentType == 'lpx')",message="lpx may only be set when componentType is lpx" diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index d5376cddfc03..1acd6f5da3ff 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -70,7 +70,7 @@ type DynamoComponentDeploymentSpec struct { // semantics. Users can add sidecars, init containers, and pod-level configuration // directly in `podTemplate` without any `extraPodSpec`-style escape hatch. // +kubebuilder:validation:XValidation:rule="!has(self.eppConfig) || (has(self.type) && self.type == 'epp')",message="eppConfig may only be set when type is epp" -// +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" +// +kubebuilder:validation:XValidation:rule="!has(self.minAvailable) || (!has(self.replicas) && has(self.type) && self.type == 'lpx') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)",message="minAvailable must be less than or equal to replicas unless replicas is 0" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)",message="minAvailable is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.type) || (has(self.type) && self.type == oldSelf.type)",message="type is immutable after it is set" // +kubebuilder:validation:XValidation:rule="!has(self.lpx) || (has(self.type) && self.type == 'lpx')",message="lpx may only be set when type is lpx" diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index c279bc5ee44a..8260c4e51f14 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -19713,7 +19713,7 @@ spec: - message: standalone LPX DynamoComponentDeployments are not supported; use DynamoGraphDeployment rule: '!(has(self.componentType) && self.componentType == ''lpx'')' - message: minAvailable must be less than or equal to replicas unless replicas is 0 - rule: '!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' + rule: '!has(self.minAvailable) || (!has(self.replicas) && has(self.componentType) && self.componentType == ''lpx'') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' - message: minAvailable is immutable after creation rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set @@ -36871,7 +36871,7 @@ spec: - message: eppConfig may only be set when type is epp rule: '!has(self.eppConfig) || (has(self.type) && self.type == ''epp'')' - message: minAvailable must be less than or equal to replicas unless replicas is 0 - rule: '!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' + rule: '!has(self.minAvailable) || (!has(self.replicas) && has(self.type) && self.type == ''lpx'') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' - message: minAvailable is immutable after creation rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: type is immutable after it is set diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 3689ab4b4258..e49fbaec7abc 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -20129,7 +20129,7 @@ spec: type: object x-kubernetes-validations: - message: minAvailable must be less than or equal to replicas unless replicas is 0 - rule: '!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' + rule: '!has(self.minAvailable) || (!has(self.replicas) && has(self.componentType) && self.componentType == ''lpx'') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' - message: minAvailable is immutable after creation rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: componentType is immutable after it is set @@ -37538,7 +37538,7 @@ spec: - message: eppConfig may only be set when type is epp rule: '!has(self.eppConfig) || (has(self.type) && self.type == ''epp'')' - message: minAvailable must be less than or equal to replicas unless replicas is 0 - rule: '!has(self.minAvailable) || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' + rule: '!has(self.minAvailable) || (!has(self.replicas) && has(self.type) && self.type == ''lpx'') || (has(self.replicas) && self.replicas == 0) || self.minAvailable <= (has(self.replicas) ? self.replicas : 1)' - message: minAvailable is immutable after creation rule: '!has(oldSelf.minAvailable) || (has(self.minAvailable) && self.minAvailable == oldSelf.minAvailable)' - message: type is immutable after it is set diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index fa9c6be693cf..220db404b853 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -1163,6 +1163,55 @@ func TestLPXExplicitScaleInDuringSchedulingFailure(t *testing.T) { require.Equal(t, pcs.UID, observedPCS.UID) } +func TestLPXNativeMinimumValidatedBeforePublishingPCS(t *testing.T) { + for _, tc := range []struct { + name string + shared bool + minimum int32 + }{ + {name: "single minimum", minimum: 2}, + {name: "single maximum", minimum: 2496}, + {name: "single oversized", minimum: 2497}, + {name: "shared minimum", shared: true, minimum: 2}, + {name: "shared maximum", shared: true, minimum: 2496}, + {name: "shared oversized", shared: true, minimum: 2497}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Start an externally managed workload at the conductor component's minimum") + var ( + child *v1alpha1.LPXGraphDeployment + dgd *v1beta1.DynamoGraphDeployment + registry lpx.ModelRegistry + ) + if tc.shared { + child, dgd, registry = newLPXSpecDecodeTestDGD(t) + } else { + child, dgd, registry = newLPXTestDGD(t, lpx.PipelineSingle) + } + component := dgd.GetComponentByName("lpx") + component.Replicas, component.MinAvailable = nil, ptr.To(tc.minimum) + r := newLPXTestReconciler(t, registry, child, dgd) + + t.Log("Apply the existing replica bound before creating any Grove workload") + _, reconcileErr := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) + pcs, err := getPodCliqueSet(t.Context(), r.Client, child) + require.NoError(t, err) + if tc.minimum > 2496 { + require.ErrorContains(t, reconcileErr, "between 0 and 2496") + require.Nil(t, pcs) + } else { + require.NoError(t, reconcileErr) + require.NotNil(t, pcs) + require.True(t, metav1.IsControlledBy(pcs, child)) + require.Equal(t, ptr.To(tc.minimum), pcs.Spec.Template.PodCliqueScalingGroupConfigs[0].Replicas) + } + requests := &lpxv1alpha1.LPUPipelineRequestList{} + require.NoError(t, r.List(t.Context(), requests)) + require.Empty(t, requests.Items, "requests wait for an observed Grove scaling group") + }) + } +} + func TestLPXExternalScaleRejectsInvalidReplicaCount(t *testing.T) { t.Log("Observe externally managed capacity above the materialization limit") ctx := t.Context() diff --git a/deploy/operator/internal/dynamo/lpx/resolve_workload.go b/deploy/operator/internal/dynamo/lpx/resolve_workload.go index 08fc93d49540..60991715568f 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_workload.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload.go @@ -138,11 +138,11 @@ func ResolveWorkload( } projections = projected } - scalingGroupReplicas := int32(1) - if len(components) == 1 { - // A sole component can flatten its replica axis into the outer scaling group. - scalingGroupReplicas = ptr.Deref(components[0].Replicas, 1) - } + + // The conductor component owns explicit capacity or the initial native seed. + conductor := components[len(components)-1] + scalingGroupReplicas := ptr.Deref(conductor.Replicas, ptr.Deref(conductor.MinAvailable, 1)) + // Canonical roles expand into default or draft0..draft7 followed by target. digest, err := workloadSetDigest(projections) if err != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index f2f3f4cf0c7c..d829b26dcaa0 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -23,10 +23,11 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { // Keep LPX inputs and oracles together without a separate admission execution path. tests := []dgdAdmissionTestCase{ { - name: "singleton LPX with explicit conductor preserves omitted replicas", + name: "singleton LPX preserves omitted replicas with minimum availability above one", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { component := &dgd.Spec.Components[0] component.Replicas = nil + component.MinAvailable = k8sptr.To(int32(2)) component.Roles[0].PodTemplate.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "model-storage", MountPath: "/nfs"}} component.Roles[0].PodTemplate.Spec.Volumes = []corev1.Volume{{Name: "model-storage", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "model-storage"}, @@ -42,28 +43,69 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantReplicas: map[string]*int32{"lpx": nil}, }, { - name: "v1alpha1 LPX preserves omitted replicas on CREATE", + name: "v1alpha1 LPX preserves omitted replicas with minimum availability above one on CREATE", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Replicas = nil + dgd.Spec.Services["lpx"].MinAvailable = k8sptr.To(int32(2)) }), wantReplicas: map[string]*int32{"lpx": nil}, }, { - name: "LPX preserves omitted replicas on UPDATE", - oldDeployment: betaLPXDGDForAdmission(nil), + name: "LPX preserves omitted replicas with minimum availability above one on UPDATE", + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].Replicas = k8sptr.To(int32(3)) + dgd.Spec.Components[0].MinAvailable = k8sptr.To(int32(2)) + }), deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Replicas = nil + dgd.Spec.Components[0].MinAvailable = k8sptr.To(int32(2)) }), wantReplicas: map[string]*int32{"lpx": nil}, }, { - name: "v1alpha1 LPX preserves omitted replicas on UPDATE", - oldDeployment: alphaLPXDGDForAdmission(nil), + name: "v1alpha1 LPX preserves omitted replicas with minimum availability above one on UPDATE", + oldDeployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Replicas = k8sptr.To(int32(3)) + dgd.Spec.Services["lpx"].MinAvailable = k8sptr.To(int32(2)) + }), deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Replicas = nil + dgd.Spec.Services["lpx"].MinAvailable = k8sptr.To(int32(2)) }), wantReplicas: map[string]*int32{"lpx": nil}, }, + { + name: "LPX rejects explicit replicas below minimum availability", + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components[0].MinAvailable = k8sptr.To(int32(2)) + }), + wantCELErr: "spec.components[0]: Invalid value: minAvailable must be less than or equal to replicas unless replicas is 0", + }, + { + name: "v1alpha1 LPX rejects explicit replicas below minimum availability", + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].MinAvailable = k8sptr.To(int32(2)) + }), + wantCELErr: "spec.services[lpx]: Invalid value: minAvailable must be less than or equal to replicas unless replicas is 0", + }, + { + name: "ordinary components reject omitted replicas with minimum availability above one", + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.Replicas = nil + worker.MinAvailable = k8sptr.To(int32(2)) + }), + wantCELErr: "spec.components[1]: Invalid value: minAvailable must be less than or equal to replicas unless replicas is 0", + }, + { + name: "v1alpha1 ordinary components reject omitted replicas with minimum availability above one", + deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services[dgdAdmissionWorkerName] + worker.Replicas = nil + worker.MinAvailable = k8sptr.To(int32(2)) + }), + wantCELErr: "spec.services[worker]: Invalid value: minAvailable must be less than or equal to replicas unless replicas is 0", + }, { name: "LPX admits nine hybrid replicas with lowered conductor volumes", deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index 876c319f79aa..b0758e216a94 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -98,6 +98,8 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Minimum complete component replicas guaranteed to be gang-scheduled before a shortfall triggers gang termination. Supported only for Grove-backed DGD components, where it defaults to `1`; rejected for non-Grove deployments. Minimum `1` and immutable after creation. Positive `replicas` must be at least this value. Scaling to `0` is allowed and retains `minAvailable` for the next scale-up. + + For LPX components with omitted `replicas`, `minAvailable` sets the initial Grove scaling-group size; subsequent capacity is externally managed. From 4c2f6f2ce5678c5072d4f87fa6543bfd67ea09c0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:27 +0000 Subject: [PATCH 138/196] refactor(operator): remove unused LPX ownership helpers Signed-off-by: Andrew Paprotsky --- .../internal/dynamo/lpx/grove_ownership.go | 57 ------------------- .../internal/dynamo/lpx/model_projection.go | 4 ++ 2 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 deploy/operator/internal/dynamo/lpx/grove_ownership.go diff --git a/deploy/operator/internal/dynamo/lpx/grove_ownership.go b/deploy/operator/internal/dynamo/lpx/grove_ownership.go deleted file mode 100644 index 4a23e8ed57be..000000000000 --- a/deploy/operator/internal/dynamo/lpx/grove_ownership.go +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package lpx - -import ( - "context" - - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" - groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" - grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // DeploymentNameAnnotation routes rendered-object events to their LPXGraphDeployment. - DeploymentNameAnnotation = "lpx.nvidia.com/deployment-name" - // DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. - DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" -) - -// OwnsPodClique verifies the LPX owner chain before ordinary DGD watches -// suppress a role-readiness event. Authorable annotations alone are not proof. -func OwnsPodClique(ctx context.Context, reader client.Reader, clique *grovev1alpha1.PodClique) bool { - if clique == nil || reader == nil || !isMaterializedRole(clique) { - return false - } - owner := metav1.GetControllerOf(clique) - if owner == nil || owner.Kind != groveconstants.KindPodCliqueScalingGroup || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { - return false - } - group := &grovev1alpha1.PodCliqueScalingGroup{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, group); err != nil || group.UID != owner.UID { - return false - } - owner = metav1.GetControllerOf(group) - if owner == nil || owner.Kind != groveconstants.KindPodCliqueSet || owner.APIVersion != grovev1alpha1.SchemeGroupVersion.String() || owner.UID == "" { - return false - } - pcs := &grovev1alpha1.PodCliqueSet{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: clique.Namespace, Name: owner.Name}, pcs); err != nil || pcs.UID != owner.UID { - return false - } - owner = metav1.GetControllerOf(pcs) - return owner != nil && owner.Kind == nvidiacomv1alpha1.LPXGraphDeploymentGVK.Kind && - owner.APIVersion == nvidiacomv1alpha1.GroupVersion.String() && owner.UID != "" && - owner.Name == clique.Annotations[DeploymentNameAnnotation] && - string(owner.UID) == clique.Annotations[DeploymentUIDAnnotation] -} - -func isMaterializedRole(clique *grovev1alpha1.PodClique) bool { - role := clique.Annotations[lpxv1alpha1.PodRoleAnnotation] - return clique.Annotations[WorkloadDigestAnnotation] != "" && - (role == lpxv1alpha1.PodRoleAgent || role == lpxv1alpha1.PodRoleConductor || role == lpxv1alpha1.PodRoleCyborgWorker) -} diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index f483aff02165..126e7efe816d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -15,6 +15,10 @@ import ( ) const ( + // DeploymentNameAnnotation routes rendered-object events to their LPXGraphDeployment. + DeploymentNameAnnotation = "lpx.nvidia.com/deployment-name" + // DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. + DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" // WorkloadDigestAnnotation records the immutable Dynamo workload projection digest on rendered objects. WorkloadDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-workload-digest" // DGDUIDAnnotation records the source DGD UID on rendered objects. From 3e48f18cfa4052450ba42a2b7bb4b1695c1248ba Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:27 +0000 Subject: [PATCH 139/196] refactor(operator): leave LPX child deletion to owner garbage collection Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_lpx_handoff.go | 15 ------- .../controller/dgd_lpx_handoff_test.go | 39 +++++-------------- .../dynamographdeployment_controller.go | 5 --- 3 files changed, 10 insertions(+), 49 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index e32c2ee656cd..c9c3faece129 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -76,18 +76,3 @@ func (r *dgdLPXHandoff) deleteChild(ctx context.Context, child *v1alpha1.LPXGrap uid, version := child.UID, child.ResourceVersion return client.IgnoreNotFound(r.client.Delete(ctx, child, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}})) } - -func (r *dgdLPXHandoff) Finalize(ctx context.Context, source *v1beta1.DynamoGraphDeployment) error { - child := &v1alpha1.LPXGraphDeployment{} - if err := r.client.Get(ctx, client.ObjectKeyFromObject(source), child); err != nil { - return client.IgnoreNotFound(err) - } - if !metav1.IsControlledBy(child, source) { - // It is not our dependent; do not delete it and do not wait for it. - return nil - } - if err := r.deleteChild(ctx, child); err != nil { - return err - } - return fmt.Errorf("waiting for LPXGraphDeployment %q to finish cleanup", child.Name) -} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 064af15574ff..aa98149d8266 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -190,12 +190,6 @@ func TestLPXHandoffCreatesOnlyAnOwnedReference(t *testing.T) { source.UID = "replacement-source" _, err = handoff.Reconcile(t.Context(), source) require.ErrorContains(t, err, "adoption is not supported") - - t.Log("Do not block finalization on or delete the child owned by the prior source") - require.NoError(t, handoff.Finalize(t.Context(), source)) - stored := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) - require.True(t, stored.DeletionTimestamp.IsZero()) } func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { @@ -740,22 +734,9 @@ func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { } } -func TestLPXChildFinalizationAndDeselectionWaitForCleanup(t *testing.T) { - t.Log("Deselect LPX while the child reports unfinished cleanup through its finalizer") +func TestDGDCheckpointFinalizationLeavesLPXChildToGarbageCollection(t *testing.T) { + t.Log("Finalize a DGD while its owned LPX child still exists") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") - child.Finalizers = []string{"test.example/child-cleanup"} - require.NoError(t, kube.Update(t.Context(), child)) - handoff := &dgdLPXHandoff{client: kube} - source.Spec.Components = nil - deleting, err := handoff.Reconcile(t.Context(), source) - require.NoError(t, err) - require.NotNil(t, deleting) - require.ErrorContains(t, handoff.Finalize(t.Context(), source), "finish cleanup") - stored := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) - require.False(t, stored.DeletionTimestamp.IsZero()) - - t.Log("The outer DGD finalizer must wait for LPX before inspecting checkpoint cleanup") snapshotReads := 0 observed := interceptor.NewClient(kube.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, delegated client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { @@ -766,14 +747,14 @@ func TestLPXChildFinalizationAndDeselectionWaitForCleanup(t *testing.T) { }, }) parent := &DynamoGraphDeploymentReconciler{Client: observed} - require.ErrorContains(t, parent.FinalizeResource(t.Context(), source), "waiting for LPXGraphDeployment") - require.Zero(t, snapshotReads) - - t.Log("Finish parent finalization only after the child releases its cleanup finalizer") - stored.Finalizers = nil - require.NoError(t, kube.Update(t.Context(), stored)) - require.NoError(t, handoff.Finalize(t.Context(), source)) - require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored))) + require.NoError(t, parent.FinalizeResource(t.Context(), source)) + require.Equal(t, 1, snapshotReads) + + t.Log("Leave the child and its owner reference intact for Kubernetes garbage collection") + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.True(t, stored.DeletionTimestamp.IsZero()) + require.Equal(t, metav1.GetControllerOf(child), metav1.GetControllerOf(stored)) } func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 3ff87eb87efa..60bcdc278e65 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -235,11 +235,6 @@ func (r *DynamoGraphDeploymentReconciler) persistWorkloadProgramResult( } func (r *DynamoGraphDeploymentReconciler) FinalizeResource(ctx context.Context, dynamoDeployment *nvidiacomv1beta1.DynamoGraphDeployment) error { - // Wait for the LPX child's cleanup before deleting graph-owned checkpoints. - if err := (&dgdLPXHandoff{client: r.Client}).Finalize(ctx, dynamoDeployment); err != nil { - return err - } - syncer := newDGDResourceSyncer(r.Client, r.Recorder) return newDGDCheckpointsReconciler( syncer, From c45db8cc8cabfa697e58008417eff553cd137a8a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:42 +0000 Subject: [PATCH 140/196] refactor(operator): remove unreachable LPX deselection cleanup Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_program.go | 21 +--- .../internal/controller/dgd_lpx_handoff.go | 15 +-- .../controller/dgd_lpx_handoff_test.go | 119 +++++++++--------- .../internal/controller/lpx/AGENTS.md | 3 +- .../internal/controller/lpx/controller.go | 2 +- .../operator/internal/controller/lpx/util.go | 2 +- .../internal/controller/lpx/util_test.go | 12 +- 7 files changed, 69 insertions(+), 105 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index f91593233ddb..f85b3c923bb0 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -116,23 +116,6 @@ func (p *groveProgram) Reconcile( "hasMultinode", req.DGD.HasAnyMultinodeComponent(), ) - // Removing LPX must not depend on ordinary workloads reconciling successfully. - var child *nvidiacomv1alpha1.LPXGraphDeployment - var err error - if !req.DGD.HasLPXComponent() { - child, err = p.lpx.Reconcile(ctx, req.DGD) - if err != nil { - return programResult, fmt.Errorf("reconcile LPX child: %w", err) - } - - // Do not keep reporting retired LPX capacity when ordinary reconciliation fails. - for name := range programResult.Status.Components { - if req.DGD.GetComponentByName(name) == nil { - delete(programResult.Status.Components, name) - } - } - } - if err := p.rollout.migrateCurrentWorkerHashIfNeeded(ctx, req.DGD); err != nil { log.FromContext(ctx).Error(err, "Failed to migrate worker hash") return programResult, failWorkloadProgram(reasonFailedToMigrateWorkerHash, err) @@ -187,6 +170,7 @@ func (p *groveProgram) Reconcile( } // Keep LPX creation and updates after ordinary reconciliation and restart selection. + var child *nvidiacomv1alpha1.LPXGraphDeployment if req.DGD.HasLPXComponent() { child, err = p.lpx.Reconcile(ctx, req.DGD) if err != nil { @@ -196,9 +180,6 @@ func (p *groveProgram) Reconcile( result = mergeLPXChildStatus(req.DGD, child, result) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) - if child != nil && !child.DeletionTimestamp.IsZero() { - programResult.RequeueAfter = 5 * time.Second - } if result.State != nvidiacomv1beta1.DGDStatePending || result.Reason != reasonWaitingForCheckpoint { if err := p.scalingAdapters.Reconcile(ctx, req.DGD); err != nil { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff.go b/deploy/operator/internal/controller/dgd_lpx_handoff.go index c9c3faece129..bf5b18b07dd7 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff.go @@ -22,6 +22,7 @@ type dgdLPXHandoff struct { client client.Client } +// Reconcile requires a source that selects LPX. func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGraphDeployment) (*v1alpha1.LPXGraphDeployment, error) { child := &v1alpha1.LPXGraphDeployment{} err := r.client.Get(ctx, client.ObjectKeyFromObject(source), child) @@ -32,12 +33,6 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra if exists && !metav1.IsControlledBy(child, source) { return nil, fmt.Errorf("LPXGraphDeployment %q belongs to a different source; adoption is not supported", child.Name) } - if !source.HasLPXComponent() { - if !exists { - return nil, nil - } - return child, r.deleteChild(ctx, child) - } if exists && !child.DeletionTimestamp.IsZero() { return child, nil } @@ -68,11 +63,3 @@ func (r *dgdLPXHandoff) Reconcile(ctx context.Context, source *v1beta1.DynamoGra } return child, err } - -func (r *dgdLPXHandoff) deleteChild(ctx context.Context, child *v1alpha1.LPXGraphDeployment) error { - if !child.DeletionTimestamp.IsZero() { - return nil - } - uid, version := child.UID, child.ResourceVersion - return client.IgnoreNotFound(r.client.Delete(ctx, child, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}})) -} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index aa98149d8266..a5eafbd65be4 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -668,70 +668,63 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { } } -func TestLPXRemovalSurvivesOrdinaryReconcileFailure(t *testing.T) { - for _, test := range []struct { - name string - retainLPX bool - childGone bool - }{ - {name: "remove"}, - {name: "retain", retainLPX: true}, - {name: "pending child already gone", childGone: true}, - } { - t.Run(test.name, func(t *testing.T) { - t.Log("Edit a mixed DGD while ordinary shared-resource reconciliation will fail") - child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") - if test.childGone { - require.NoError(t, kube.Delete(t.Context(), child)) - } else { - child.Finalizers = []string{"test.example/child-cleanup"} - require.NoError(t, kube.Update(t.Context(), child)) - } - if !test.retainLPX { - source.Spec.Components = nil - } else { - source.Spec.Components[0].Replicas = ptr.To(int32(2)) - } - source.Spec.Components = append(source.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, - }) - source.Generation++ - require.NoError(t, kube.Update(t.Context(), source)) - source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} - source.Status.Components = map[string]v1beta1.ComponentReplicaStatus{ - "lpx": {Replicas: 1}, "prefill": {Replicas: 2}, - } - if test.childGone { - source.Status.Components["lpx"] = v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup} - } - previousStatus := source.Status.DeepCopy() - program := (&DynamoGraphDeploymentReconciler{ - Client: kube, Config: &configv1alpha1.OperatorConfiguration{}, Recorder: events.NewFakeRecorder(10), - RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, - }).newGroveProgram() - - t.Log("Preserve the ordinary error while deleting only a deselected child") - result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) - require.ErrorContains(t, err, "RBAC manager not initialized") - require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) - stored := &v1alpha1.LPXGraphDeployment{} - if test.childGone { - require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored))) - } else { - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) - require.Equal(t, !test.retainLPX, !stored.DeletionTimestamp.IsZero()) - require.Equal(t, child.Spec, stored.Spec) - } - require.Equal(t, previousStatus, &source.Status) - require.Equal(t, source.Status.Components["prefill"], result.Status.Components["prefill"]) - if !test.retainLPX { - require.NotContains(t, result.Status.Components, "lpx") - } else { - require.Equal(t, source.Status.Components, result.Status.Components) - } - require.Equal(t, source.Status.Placement, result.Status.Placement) - }) +func TestGroveProgramWithoutLPXDoesNotReadLPXChild(t *testing.T) { + t.Log("Reconcile an ordinary DGD while LPX child reads would fail") + source := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ordinary", Namespace: "default"}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend"}}}}, + }}}, } + childReads := 0 + kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(source). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, delegated client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, child := obj.(*v1alpha1.LPXGraphDeployment); child { + childReads++ + return apierrors.NewServiceUnavailable("child observation unavailable") + } + return delegated.Get(ctx, key, obj, opts...) + }, + }).Build() + program := (&DynamoGraphDeploymentReconciler{ + Client: kube, Config: &configv1alpha1.OperatorConfiguration{}, Recorder: events.NewFakeRecorder(10), + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, + }).newGroveProgram() + + t.Log("Reach the ordinary shared-resource error without depending on an LPX child") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.ErrorContains(t, err, "RBAC manager not initialized") + require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) + require.Zero(t, childReads) +} + +func TestLPXHandoffWaitsForSharedResourceReconciliation(t *testing.T) { + t.Log("Edit LPX capacity while shared-resource reconciliation will fail") + child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") + source.Spec.Components[0].Replicas = ptr.To(int32(2)) + source.Generation++ + require.NoError(t, kube.Update(t.Context(), source)) + source.Status.Placement = &v1beta1.PlacementStatus{Score: ptr.To(0.92), State: v1beta1.PlacementScoreStateReported} + source.Status.Components = map[string]v1beta1.ComponentReplicaStatus{"lpx": {Replicas: 1}} + previousStatus := source.Status.DeepCopy() + program := (&DynamoGraphDeploymentReconciler{ + Client: kube, Config: &configv1alpha1.OperatorConfiguration{}, Recorder: events.NewFakeRecorder(10), + RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, + }).newGroveProgram() + + t.Log("Preserve the child revision and component status while reporting the shared-resource error") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.ErrorContains(t, err, "RBAC manager not initialized") + require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) + stored := &v1alpha1.LPXGraphDeployment{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) + require.True(t, stored.DeletionTimestamp.IsZero()) + require.Equal(t, child.Spec, stored.Spec) + require.Equal(t, previousStatus, &source.Status) + require.Equal(t, source.Status.Components, result.Status.Components) + require.Equal(t, source.Status.Placement, result.Status.Placement) } func TestDGDCheckpointFinalizationLeavesLPXChildToGarbageCollection(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/AGENTS.md b/deploy/operator/internal/controller/lpx/AGENTS.md index 33f3e276536e..a27d81f503cd 100644 --- a/deploy/operator/internal/controller/lpx/AGENTS.md +++ b/deploy/operator/internal/controller/lpx/AGENTS.md @@ -42,7 +42,8 @@ SPDX-License-Identifier: Apache-2.0 - LPRs have a controlling PCS owner reference with `blockOwnerDeletion: true`. Foreground PCS deletion keeps the owner present until blocking LPRs are gone. This is the supported lifecycle, not a guarantee against external orphaning. -- The parent controller deletes the LPXGD when the DGD no longer selects LPX. +- Admission prevents removing LPX components or changing them to non-LPX. + Deleting the DGD removes its LPXGD through owner garbage collection. A missing/deleting DGD or incomplete input handoff stops publication; it does not authorize cleanup. A DGD UID mismatch is an error. Disabled integration also leaves existing workloads intact. diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 6f94d847738e..325eacaa22fe 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -132,7 +132,7 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return ctrl.Result{}, err } - // The parent owns deselection/deletion; a cache miss or delayed handoff is not cleanup authority. + // A cache miss or delayed handoff is not cleanup authority; owner GC handles deletion. if dgd == nil { setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the matching DynamoGraphDeployment") return ctrl.Result{}, nil diff --git a/deploy/operator/internal/controller/lpx/util.go b/deploy/operator/internal/controller/lpx/util.go index 1be5eb20d996..a8caecda749c 100644 --- a/deploy/operator/internal/controller/lpx/util.go +++ b/deploy/operator/internal/controller/lpx/util.go @@ -41,7 +41,7 @@ func getDynamoGraphDeployment(ctx context.Context, reader client.Reader, deploym return nil, fmt.Errorf("DynamoGraphDeployment %q no longer has the referenced UID", owner.Name) } - // The parent deletes the child on deselection; GC owns the remaining workload. + // Only an active LPX source authorizes publication; owner GC handles deletion. if !dgd.DeletionTimestamp.IsZero() || !dgd.HasLPXComponent() { return nil, nil } diff --git a/deploy/operator/internal/controller/lpx/util_test.go b/deploy/operator/internal/controller/lpx/util_test.go index 2006c40a742b..340c18fc659f 100644 --- a/deploy/operator/internal/controller/lpx/util_test.go +++ b/deploy/operator/internal/controller/lpx/util_test.go @@ -11,6 +11,7 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -288,8 +289,8 @@ func TestLPXForeignRequestNameIsNeverAdopted(t *testing.T) { } } -func TestLPXDeselectionWaitsForParentDeletion(t *testing.T) { - t.Log("Keep a published workload while its source removes the LPX component") +func TestLPXDeletingSourcePreservesPublishedWorkload(t *testing.T) { + t.Log("Keep a published workload while the DGD is awaiting its own finalization") child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) r, desired := newPreparedLPXTestReconciler(t, registry, t.Context(), child, dgd) objects := lpxMaterializedObjects(t, r, child, dgd, desired) @@ -298,14 +299,15 @@ func TestLPXDeselectionWaitsForParentDeletion(t *testing.T) { pcs := findLPXTestPodCliqueSet(t, objects) before := getTestPipelineRequest(t, t.Context(), r.Client, child.Namespace, desired.requests[0].Name) require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(dgd), dgd)) - dgd.Spec.Components = nil + commoncontroller.AddFinalizer(dgd) require.NoError(t, r.Update(t.Context(), dgd)) + require.NoError(t, r.Delete(t.Context(), dgd)) - t.Log("Wait for the parent's deletion and GC without hashing empty LPX inputs or deleting the PCS") + t.Log("Stop publication without directly deleting the PCS or its requests") for range 2 { result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) - require.Zero(t, result, "the source and child watches observe deselection") + require.Zero(t, result, "the source and child watches observe deletion") } require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) require.True(t, pcs.DeletionTimestamp.IsZero()) From 8dd36214135450dcb01c156bf66a7e26a91a9c88 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:43 +0000 Subject: [PATCH 141/196] test(operator): preserve environment order in LPX render goldens Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/render_test.go | 7 ------- .../from_dgd_yaml/lpu-cyborg-specdec.yaml | 16 ++++++++-------- .../from_dgd_yaml/lpu-gpu-specdecode.yaml | 8 ++++---- .../lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 8 ++++---- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 8 ++++---- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 8 ++++---- .../from_dgd_yaml/node-local-v2-specdecode.yaml | 12 ++++++------ .../from_dgd_yaml/node-local-v3-hx-hybrid.yaml | 8 ++++---- .../from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 8 ++++---- .../node-local-v3-hx-specdecode.yaml | 12 ++++++------ .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 8 ++++---- 11 files changed, 48 insertions(+), 55 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 885b07036815..1bda8b3e77bd 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -126,13 +126,6 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { sort.Slice(pcs.Spec.Template.Cliques, func(i, j int) bool { return pcs.Spec.Template.Cliques[i].Name < pcs.Spec.Template.Cliques[j].Name }) - for _, clique := range pcs.Spec.Template.Cliques { - for i := range clique.Spec.PodSpec.Containers { - slices.SortFunc(clique.Spec.PodSpec.Containers[i].Env, func(a, b corev1.EnvVar) int { - return strings.Compare(a.Name, b.Name) - }) - } - } extraResources = append(extraResources, pcs) } diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml index 5d81ddbdfb1e..13b2b3089ef5 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml @@ -188,6 +188,10 @@ spec: command: - /opt/dynamo-lpu env: + - name: LPX_ALLOCATION + value: draft-agt + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -200,10 +204,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: draft-agt - - name: LPX_MODEL_PATH - value: /models/node-local-v2-connected - name: NATS_SERVER value: nats-address - name: POD_IP @@ -276,6 +276,8 @@ spec: command: - /opt/dynamo-lpu env: + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected-lpx - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -288,8 +290,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_MODEL_PATH - value: /models/node-local-v2-connected-lpx - name: NATS_SERVER value: nats-address - name: POD_IP @@ -365,6 +365,8 @@ spec: - -m - dynamo.cyborg env: + - name: GBUILD_MANIFEST_PATH + value: /models/node-local-v2-connected-lpx/manifest.v2.capnp.bin - name: DYN_COMPONENT value: decode - name: DYN_DISCOVERY_BACKEND @@ -387,8 +389,6 @@ spec: value: '["generate"]' - name: ETCD_ENDPOINTS value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /models/node-local-v2-connected-lpx/manifest.v2.capnp.bin - name: NATS_SERVER value: nats-address - name: NIXL_TELEMETRY_ENABLE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml index 23d9266863b3..4d1417a56a3e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml @@ -156,6 +156,10 @@ spec: command: - /opt/dynamo-lpu env: + - name: LPX_ALLOCATION + value: agt + - name: LPX_MODEL_PATH + value: /models/node-local-v2-connected - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -168,10 +172,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt - - name: LPX_MODEL_PATH - value: /models/node-local-v2-connected - name: NATS_SERVER value: nats-address - name: POD_IP diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index ea328c95c71c..bf637f288dbe 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -262,6 +262,8 @@ spec: --metadata-file "$(mktemp)" \ --readiness-port "${READINESS_PORT}" env: + - name: LPX_MODEL_PATH + value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -282,8 +284,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - - name: LPX_MODEL_PATH - value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -442,6 +442,8 @@ spec: - -- - /usr/local/bin/cyborg env: + - name: GBUILD_MANIFEST_PATH + value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: @@ -468,8 +470,6 @@ spec: value: '["generate"]' - name: ETCD_ENDPOINTS value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/llama3_2-1b-lpu-gpu-v2/build_0m851219t7py3mp8x1j5rg9j8c/manifest.v2.capnp.bin - name: GLUE_RDMA_PATH_MTU value: "1024" - name: NATS_SERVER diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 3b9e0c0a2352..f74b76e213a4 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -77,6 +77,8 @@ spec: command: - /opt/dynamo-lpu env: + - name: LPX_MODEL_PATH + value: /nfs/node-local-v2-connected-lpx - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -97,8 +99,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - - name: LPX_MODEL_PATH - value: /nfs/node-local-v2-connected-lpx - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -240,6 +240,8 @@ spec: - -- - /opt/gpu-runtime env: + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v2-connected-lpx/manifest.v2.capnp.bin - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: @@ -266,8 +268,6 @@ spec: value: '["generate"]' - name: ETCD_ENDPOINTS value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/node-local-v2-connected-lpx/manifest.v2.capnp.bin - name: NATS_SERVER value: nats-address - name: NIXL_TELEMETRY_ENABLE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 7646f7b8f1a2..81d1214c0712 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -239,6 +239,10 @@ spec: command: - /opt/dynamo-lpu env: + - name: LPX_ALLOCATION + value: agt + - name: LPX_MODEL_PATH + value: /nfs/node-local-v2-connected - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -251,10 +255,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt - - name: LPX_MODEL_PATH - value: /nfs/node-local-v2-connected - name: NATS_SERVER value: nats-address - name: NOVA_MODEL_BASE_PATHS diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 9b34d61480cb..1af2ad76cd1a 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -556,6 +556,12 @@ spec: command: - /bin/nova env: + - name: LPX_ALLOCATION + value: agt0:agt1:agt2 + - name: LPX_DRAFT_MODEL_PATH + value: /nfs/node-local-v2-connected + - name: LPX_TARGET_MODEL_PATH + value: /nfs/node-local-v2-connected - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -568,12 +574,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt0:agt1:agt2 - - name: LPX_DRAFT_MODEL_PATH - value: /nfs/node-local-v2-connected - - name: LPX_TARGET_MODEL_PATH - value: /nfs/node-local-v2-connected - name: NATS_SERVER value: nats-address - name: NOVA_DRAFT_MODEL_PATH diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index 2f06a4ad0f77..debb40b8085d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -64,6 +64,8 @@ spec: command: - /bin/hydra-entrypoint env: + - name: LPX_MODEL_PATH + value: /nfs/node-local-v3-hx-connected-lpx - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -84,8 +86,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['scheduling.lpu.nvidia.com/model'] - - name: LPX_MODEL_PATH - value: /nfs/node-local-v3-hx-connected-lpx - name: NATS_SERVER value: nats-address - name: NIC_NAME @@ -223,6 +223,8 @@ spec: command: - /opt/gpu-runtime env: + - name: GBUILD_MANIFEST_PATH + value: /nfs/node-local-v3-hx-connected-lpx/manifest.v2.capnp.bin - name: CYBORG_FPGA_GPI_REPLICA_INDEX valueFrom: fieldRef: @@ -249,8 +251,6 @@ spec: value: '["generate"]' - name: ETCD_ENDPOINTS value: etcd-address - - name: GBUILD_MANIFEST_PATH - value: /nfs/node-local-v3-hx-connected-lpx/manifest.v2.capnp.bin - name: NATS_SERVER value: nats-address - name: NIXL_TELEMETRY_ENABLE diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index ac76985f7be3..c700f47f2436 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -264,6 +264,10 @@ spec: command: - /bin/nova env: + - name: LPX_ALLOCATION + value: agt + - name: LPX_MODEL_PATH + value: /nfs/node-local-v3-hx-connected - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -276,10 +280,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt - - name: LPX_MODEL_PATH - value: /nfs/node-local-v3-hx-connected - name: NATS_SERVER value: nats-address - name: NOVA_MODEL_BASE_PATHS diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 4e2c60b65926..64cb0567f27f 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -597,6 +597,12 @@ spec: command: - /bin/nova env: + - name: LPX_ALLOCATION + value: agt0:agt1:agt2 + - name: LPX_DRAFT_MODEL_PATH + value: /nfs/node-local-v3-hx-sd-draft + - name: LPX_TARGET_MODEL_PATH + value: /nfs/node-local-v3-hx-sd-target - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -609,12 +615,6 @@ spec: value: test - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt0:agt1:agt2 - - name: LPX_DRAFT_MODEL_PATH - value: /nfs/node-local-v3-hx-sd-draft - - name: LPX_TARGET_MODEL_PATH - value: /nfs/node-local-v3-hx-sd-target - name: NATS_SERVER value: nats-address - name: NOVA_DRAFT_MODEL_PATH diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 318a50f2021a..adbe62b41146 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -319,6 +319,10 @@ spec: command: - /bin/nova env: + - name: LPX_ALLOCATION + value: agt + - name: LPX_MODEL_PATH + value: /nfs/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: DYN_COMPONENT value: lpx - name: DYN_DISCOVERY_BACKEND @@ -331,10 +335,6 @@ spec: value: inference-engine - name: ETCD_ENDPOINTS value: etcd-address - - name: LPX_ALLOCATION - value: agt - - name: LPX_MODEL_PATH - value: /nfs/llama3_2-1b-lpu-v2/build_0m851219t7py3mp8x1j5rg9j8c - name: MPI_BTL_TCP_IF_INCLUDE value: 192.0.2.0/24 - name: NATS_SERVER From 8af22cf8c6bf077be47e6e115b7d3eb38c73c831 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 12:57:43 +0000 Subject: [PATCH 142/196] refactor(operator): remove duplicate LPX conductor validation Signed-off-by: Andrew Paprotsky --- .../internal/webhook/validation/dynamographdeployment.go | 3 --- .../dynamographdeployment_lpx_validation_envtest_test.go | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 93f6da4b3e47..1d180ace3578 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -337,9 +337,6 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( if len(spec.Components) == 0 { allErrs = append(allErrs, field.Required(componentsPath, "must have at least one component")) } - if lpxComponentCount > 2 && independentEngineCount != lpxComponentCount { - allErrs = append(allErrs, field.Forbidden(componentsPath, "requires one complete LPX component or a shared draft and target pair")) - } components := componentsByName(spec.Components) hasLPXComponent := lpxComponentCount > 0 for i := range spec.Components { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index d829b26dcaa0..687ea46cd07e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -562,7 +562,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { extra.ComponentName = "extra-draft" dgd.Spec.Components = append(dgd.Spec.Components, *extra) }), - wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair", conductorRoleErr}, + wantWebhookErrs: []string{conductorRoleErr}, }, { name: "LPX rejects zero draft replicas", @@ -785,7 +785,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { second.ComponentName = "independent" dgd.Spec.Components = append(dgd.Spec.Components, *second) }), - wantWebhookErrs: []string{"spec.components: Forbidden: requires one complete LPX component or a shared draft and target pair", conductorRoleErr}, + wantWebhookErrs: []string{conductorRoleErr}, }, { From 6aca36901a5e75893eb4033412eb92628b2af2da Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 15:30:03 +0000 Subject: [PATCH 143/196] refactor(operator): restore pod-cache projection parity Signed-off-by: Andrew Paprotsky --- .../operator/internal/podcache/transform.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/podcache/transform.go b/deploy/operator/internal/podcache/transform.go index 0235f6c295e5..d9aa5154ddea 100644 --- a/deploy/operator/internal/podcache/transform.go +++ b/deploy/operator/internal/podcache/transform.go @@ -88,8 +88,8 @@ func projectSpec(in corev1.PodSpec) corev1.PodSpec { Containers: projectContainers(in.Containers), // GMS Pod replacement requires native init-sidecar identity. InitContainers: projectInitContainers(in.InitContainers), - // Topology uses DownwardAPI fields. - Volumes: projectVolumes(in.Volumes), + // Topology label discovery inspects DownwardAPI label field paths. + Volumes: projectTopologyVolumes(in.Volumes), } } @@ -121,7 +121,7 @@ func projectInitContainers(in []corev1.Container) []corev1.Container { return nil } -func projectVolumes(in []corev1.Volume) []corev1.Volume { +func projectTopologyVolumes(in []corev1.Volume) []corev1.Volume { out := make([]corev1.Volume, 0, len(in)) for i := range in { if in[i].DownwardAPI == nil { @@ -165,8 +165,8 @@ func projectStatus(in corev1.PodStatus) corev1.PodStatus { // Failover, DGDR diagnostics, and Recreate drain barriers require the // lifecycle phase. Phase: in.Phase, - // Model endpoint classification requires Ready; Snapshot-aware GMS replacement - // requires the public restore outcome. + // Model endpoint classification requires Ready; Snapshot-aware GMS + // replacement requires the public restore outcome. Conditions: projectConditions(in.Conditions), // DGDR diagnostics inspect container failures. ContainerStatuses: projectContainerStatuses(in.ContainerStatuses), @@ -178,16 +178,16 @@ func projectStatus(in corev1.PodStatus) corev1.PodStatus { func projectConditions(in []corev1.PodCondition) []corev1.PodCondition { out := make([]corev1.PodCondition, 0, len(in)) for i := range in { + if in[i].Type != corev1.PodReady && + in[i].Type != corev1.PodConditionType(podcontract.RestoredCondition) { + continue + } condition := corev1.PodCondition{ Type: in[i].Type, Status: in[i].Status, } - switch in[i].Type { - case corev1.PodReady: - case corev1.PodConditionType(podcontract.RestoredCondition): + if in[i].Type == corev1.PodConditionType(podcontract.RestoredCondition) { condition.Reason = in[i].Reason - default: - continue } out = append(out, condition) } From 60749baa5840e53e91ce0665563fb695d1aab0e5 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 15:52:25 +0000 Subject: [PATCH 144/196] refactor(operator): render Cyborg configuration at its consumer Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph_lpx.go | 6 ------ .../operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go | 4 ++-- .../internal/dynamo/lpx/hybrid_cyborg_bindings_test.go | 6 +++--- deploy/operator/internal/dynamo/lpx/render.go | 7 ++++--- deploy/operator/internal/dynamo/lpx/render_test.go | 6 ------ 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 15d8f9bbd7c2..2720cd315bf2 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -307,12 +307,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn role.Replicas = ptr.To(replicas) role.MinAvailable = nil defaults := &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} - if workload.BuildFamily() == dynamolpx.BuildFamilyXT { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(plan) - if err != nil { - return nil, err - } - } gpu := p gpu.component = role gpu.componentName = component.ComponentName diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 93d8b3b2f829..9c5da3dc79e6 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -12,10 +12,10 @@ import ( corev1 "k8s.io/api/core/v1" ) -// RenderCyborgConfigMap renders the XT hybrid Agent endpoints. +// renderCyborgConfigMap renders the XT hybrid Agent endpoints. // The workload and plan must be non-nil, validated, and describe the selected hybrid model. // The workload and plan are not mutated. The caller assigns the returned ConfigMap's namespace. -func (w *Workload) RenderCyborgConfigMap(plan *MaterializationPlan) (*corev1.ConfigMap, error) { +func (w *Workload) renderCyborgConfigMap(plan *MaterializationPlan) (*corev1.ConfigMap, error) { build := &w.modelProjections[0].configuredBuild // Cyborg supplies the PCS prefix; startup supplies this workload's Grove index. diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 2e62076c0472..83a92424cea9 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -39,7 +39,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Render the generated Agent endpoints") plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - initial, err := workload.RenderCyborgConfigMap(plan) + initial, err := workload.renderCyborgConfigMap(plan) require.NoError(t, err) t.Log("Verify every workload replica addresses only its own Agents") @@ -48,7 +48,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { workload.scalingGroupReplicas = replicas scaledPlan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - configMap, err := workload.RenderCyborgConfigMap(scaledPlan) + configMap, err := workload.renderCyborgConfigMap(scaledPlan) require.NoError(t, err) require.Equal(t, initial, configMap) require.True(t, *configMap.Immutable) @@ -105,7 +105,7 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { require.NoError(t, err) require.Len(t, projection.RequestSpec(plan, "agents").Partitions, test.partitions) require.Equal(t, 2*test.partitions, projection.agentReplicas) - configMap, err := workload.RenderCyborgConfigMap(plan) + configMap, err := workload.renderCyborgConfigMap(plan) require.NoError(t, err) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(test.wantOffsets)) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 240766c411fe..2eb440e2d9f4 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -30,8 +30,6 @@ const StageLabel = "lpx.nvidia.com/stage" // RenderInput contains the fresh stage templates and runtime settings consumed by rendering. type RenderInput struct { - // CyborgConfigMap is rendered before the user's Cyborg defaults are merged. - CyborgConfigMap *corev1.ConfigMap // MinAvailable is the minimum number of complete workload replicas in the gang. MinAvailable *int32 // Stages contains an independently merged LPU template for every projected stage. @@ -110,7 +108,10 @@ func RenderNodeLocal( extraResources []client.Object ) if v2HybridRuntime { - cyborgConfigMap = input.CyborgConfigMap + cyborgConfigMap, err = workload.renderCyborgConfigMap(plan) + if err != nil { + return nil, err + } // Preserve the legacy graph order: Cyborg config first, LPU config last. extraResources = []client.Object{cyborgConfigMap, configMap} } else { diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 45c511952985..56271a6a03ce 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -46,12 +46,6 @@ func renderSelectedForTest(pcs *grovev1alpha1.PodCliqueSet, projections []*Model if err != nil { return nil, err } - if workload.BuildFamily() == BuildFamilyXT && workload.Pipeline() == PipelineLPX { - input.CyborgConfigMap, err = workload.RenderCyborgConfigMap(plan) - if err != nil { - return nil, err - } - } if workload.Pipeline() == PipelineLPX { input.Cyborg = pcs.Spec.Template.Cliques[0] } From 629cbab26636c2a1291081151cc51188bd76f7ec Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 15:57:02 +0000 Subject: [PATCH 145/196] refactor(operator): skip ordinary PCS handling for LPX-only graphs Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_workload_renderer.go | 14 ++- .../dgd_grove_workloads_reconciler.go | 35 ++------ .../dgd_grove_workloads_reconciler_test.go | 86 +++---------------- .../controller/dgd_lpx_handoff_test.go | 24 ++---- 4 files changed, 37 insertions(+), 122 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index 087ec4e4996a..1c77dfe17ad5 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -83,6 +83,12 @@ func (r *groveWorkloadRenderer) Render( if r.reader == nil { return nil, fmt.Errorf("cannot render Grove PodCliqueSet without a Kubernetes reader") } + + // An LPX-only graph has no ordinary PodCliqueSet to observe or render. + if len(ordinary.Spec.Components) == 0 { + return &grovePodCliqueSetRender{renderDeployment: ordinary}, nil + } + existingPodCliqueSet := &grovev1alpha1.PodCliqueSet{} key := types.NamespacedName{ Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), @@ -95,14 +101,6 @@ func (r *groveWorkloadRenderer) Render( existingPodCliqueSet = nil } - // Represent an empty ordinary projection as no desired PodCliqueSet. - if len(ordinary.Spec.Components) == 0 { - return &grovePodCliqueSetRender{ - existing: existingPodCliqueSet, - renderDeployment: ordinary, - }, nil - } - workerHashSuffixNeeded := shouldRenderGroveWorkerHashSuffix(ordinary, existingPodCliqueSet, workerGenerationChanged) renderDeployment, err := groveRenderDeployment(ordinary, source, existingPodCliqueSet, workerHashSuffixNeeded) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 1f22e95a8fb8..aed895aeab4b 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -28,7 +28,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" @@ -96,21 +95,21 @@ func (r *groveWorkloadsReconciler) Reconcile( return ReconcileResult{}, fmt.Errorf("failed to generate the Grove GangSet: %w", err) } - // Converge the ordinary PCS before rollout or readiness observation. - syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) - if err != nil { - return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) - } + // An LPX-only graph still reconciles its stable resources. if renderedPodCliqueSet.desired == nil { - if syncedPodCliqueSet != nil { - return ReconcileResult{State: nvidiacomv1beta1.DGDStatePending, Reason: "RemovingEmptyPodCliqueSet", Message: "Waiting for the removed ordinary workload"}, nil - } stableResources, err := r.stableResources.Reconcile(ctx, source, renderedPodCliqueSet.renderDeployment) if err != nil { return ReconcileResult{}, err } return checkResourcesReadiness(stableResources), nil } + + // Converge the ordinary PCS before rollout or readiness observation. + syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) + if err != nil { + return ReconcileResult{}, fmt.Errorf("failed to reconcile the Grove PodCliqueSet: %w", err) + } + // Defer commit if the PCS was written this reconcile; the informer hasn't caught up yet. if workerHashTransition.needsCommit() && !pcsWasWritten { // Pre-existing legacy PCS without a suffix is the only case where unstamped is valid. @@ -160,28 +159,12 @@ func (r *groveWorkloadsReconciler) Reconcile( } // reconcilePodCliqueSet returns the current PCS and whether it was created or updated. -// A nil desired PCS retires the observation, retaining it until deletion is observed. +// The rendered desired PCS must be non-nil. func (r *groveWorkloadsReconciler) reconcilePodCliqueSet( ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment, rendered *grovePodCliqueSetRender, ) (*grovev1alpha1.PodCliqueSet, bool, error) { - // Retire only the exact owned observation when no ordinary workload is desired. - if rendered.desired == nil { - if existing := rendered.existing; existing != nil { - if !metav1.IsControlledBy(existing, dgd) { - return nil, false, fmt.Errorf("refusing to delete a foreign empty-graph PodCliqueSet %q", existing.Name) - } - if existing.DeletionTimestamp.IsZero() { - uid, version := existing.UID, existing.ResourceVersion - if err := r.syncer.Delete(ctx, existing, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); client.IgnoreNotFound(err) != nil { - return nil, false, err - } - } - } - return rendered.existing, false, nil - } - // Compose opaque provider fields only at the serialization boundary. desiredProviderObject, err := provideroverride.ComposeGroveOverrides(dgd, rendered.desired) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 885a82a650e0..595b56b0098e 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -137,22 +137,24 @@ func TestGroveWorkloadsReconciler_EvaluatesReadinessOnce(t *testing.T) { } func TestGroveWorkloadsReconcilerUsesStableReadinessWithoutOrdinaryPodCliqueSet(t *testing.T) { - t.Log("Project an LPX-only graph with an obsolete ordinary PodCliqueSet") + t.Log("Project an LPX-only graph without an ordinary PodCliqueSet") const dgdName = "graph" source := newLPXHandoffSource(t, "node-local-v2-lpu-only") source.Name, source.Namespace, source.UID = dgdName, corev1.NamespaceDefault, "dgd-uid" ordinary := projectWithoutExternallyManagedComponents(source) - owned := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), - Namespace: source.Namespace, - UID: "pcs-uid", - ResourceVersion: "1", - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - }} + pcsReads := 0 kubeClient := fake.NewClientBuilder(). WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). WithRESTMapper(groveScaleRESTMapper()). - WithObjects(source, owned). + WithObjects(source). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, reader client.WithWatch, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if isGrovePodCliqueSetObject(object) { + pcsReads++ + } + return reader.Get(ctx, key, object, options...) + }, + }). Build() reconciler := &DynamoGraphDeploymentReconciler{ Client: kubeClient, @@ -164,78 +166,16 @@ func TestGroveWorkloadsReconcilerUsesStableReadinessWithoutOrdinaryPodCliqueSet( }}, } - t.Log("Retire the obsolete workload under the compatible public condition reason") + t.Log("Report stable-resource readiness without looking up an ordinary PodCliqueSet") result, err := reconciler.newGroveProgram().workloads.Reconcile(t.Context(), source, ordinary, nil, nil) require.NoError(t, err) - require.Equal(t, nvidiacomv1beta1.DGDStatePending, result.State) - require.Equal(t, Reason("RemovingEmptyPodCliqueSet"), result.Reason) - - t.Log("Report stable-resource readiness after observing the PodCliqueSet absent") - result, err = reconciler.newGroveProgram().workloads.Reconcile(t.Context(), source, ordinary, nil, nil) - require.NoError(t, err) require.Equal(t, nvidiacomv1beta1.DGDStateSuccessful, result.State) + require.Zero(t, pcsReads) pcsList := &grovev1alpha1.PodCliqueSetList{} require.NoError(t, kubeClient.List(t.Context(), pcsList)) require.Empty(t, pcsList.Items) } -func TestReconcilePodCliqueSetAbsent(t *testing.T) { - const deleting = "deleting" - for _, state := range []string{"absent", "owned", deleting, "foreign", "stale"} { - t.Run(state, func(t *testing.T) { - t.Log("Observe an ordinary PCS that is no longer desired") - source := &nvidiacomv1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{ - Name: "graph", Namespace: "default", UID: "dgd-uid", - }} - observed := &grovev1alpha1.PodCliqueSet{ObjectMeta: metav1.ObjectMeta{ - Name: "graph", Namespace: source.Namespace, UID: "pcs-uid", ResourceVersion: "1", - Finalizers: []string{"hold-deletion"}, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(source, nvidiacomv1beta1.DynamoGraphDeploymentGVK)}, - }} - if state == deleting { - observed.DeletionTimestamp = ptr.To(metav1.Now()) - } - if state == "foreign" { - observed.OwnerReferences[0].UID = "another-dgd" - } - builder := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)) - if state == "absent" { - observed = nil - } else { - builder.WithObjects(observed) - } - kube := builder.Build() - if state == "stale" { - observed.ResourceVersion = "0" - } - before := observed.DeepCopy() - reconciler := &groveWorkloadsReconciler{syncer: newDGDResourceSyncer(kube, nil)} - - t.Log("Converge absence in the PCS reconciler without changing the observation") - pcs, _, err := reconciler.reconcilePodCliqueSet(t.Context(), source, &grovePodCliqueSetRender{existing: observed}) - require.Equal(t, before, observed) - if state == "foreign" { - require.ErrorContains(t, err, "foreign") - } else if state == "stale" { - require.True(t, apierrors.IsConflict(err), "expected deletion precondition conflict, got %v", err) - } else { - require.NoError(t, err) - require.Equal(t, observed, pcs) - } - - t.Log("Only an owned current observation may start deletion") - live := &grovev1alpha1.PodCliqueSet{} - err = kube.Get(t.Context(), client.ObjectKeyFromObject(source), live) - if state == "absent" { - require.True(t, apierrors.IsNotFound(err)) - } else { - require.NoError(t, err) - require.Equal(t, state == "owned" || state == deleting, !live.DeletionTimestamp.IsZero()) - } - }) - } -} - func TestGroveWorkloadsReconciler_DoesNotCommitWorkerHashWhenPodCliqueSetSyncFails(t *testing.T) { tests := []struct { name string diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index a5eafbd65be4..0df94b15e662 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -97,7 +97,7 @@ func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing require.Nil(t, ordinary.GetComponentByName(lpxName)) require.Len(t, ordinary.Spec.Components, len(source.Spec.Components)-1) - t.Log("Seed only the projected PCS identity, including the truncated-name case") + t.Log("Seed the projected PCS only for mixed graphs, including the truncated-name case") pcsName := dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components) if test.longNames { require.NotEqual(t, source.Name, pcsName) @@ -115,17 +115,22 @@ func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, } - kube := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)).WithObjects(pcs, clique).Build() + builder := fake.NewClientBuilder().WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)) + if !test.lpxOnly { + builder.WithObjects(pcs, clique) + } + kube := builder.Build() t.Log("Render and observe restarts against that same ordinary PCS") renderer := newGroveWorkloadRenderer(kube, &configv1alpha1.OperatorConfiguration{}, &commoncontroller.RuntimeConfig{}, nil) rendered, err := renderer.Render(t.Context(), source, ordinary, nil, nil, false) require.NoError(t, err) - require.NotNil(t, rendered.existing) - require.Equal(t, pcsName, rendered.existing.Name) if test.lpxOnly { + require.Nil(t, rendered.existing) require.Nil(t, rendered.desired) } else { + require.NotNil(t, rendered.existing) + require.Equal(t, pcsName, rendered.existing.Name) require.Equal(t, pcsName, rendered.desired.Name) } remaining := resolveCompositeGroveRestartProgress(t.Context(), source, ordinary, @@ -857,15 +862,4 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { require.Equal(t, v1alpha1.LPXReadyReasonFailed, failed.Reason) require.Contains(t, failed.Message, "storage volume mount is missing") require.Equal(t, v1beta1.ComponentCheckpointStatus{CheckpointName: friendlyCheckpointName}, result.Status.Checkpoints["prefill"]) - - t.Log("Removing the last ordinary component deletes its PCS without disturbing the child") - source.Spec.Components = source.Spec.Components[:1] - for range 2 { - _, err = program.workloads.Reconcile(t.Context(), source, projectWithoutExternallyManagedComponents(source), nil, nil) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(kube.Get(t.Context(), key, pcs))) - } - stored := &v1alpha1.LPXGraphDeployment{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), stored)) - require.Equal(t, child, stored) } From 455d5c6208c98e6c496f3686a13e390715d448e4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 16:28:14 +0000 Subject: [PATCH 146/196] refactor(operator): remove unused manifest build-root mode Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/model_manifest_common.go | 5 +---- deploy/operator/internal/dynamo/lpx/model_manifest_v2.go | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go index b22d2dbbe89c..f57432ddc05f 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go @@ -165,7 +165,7 @@ func decodePropSyncChain(rawIDs capnp.UInt32List, chainPath string) ([]int, erro return partitionIDs, nil } -func cleanManifestRelativeBuildPath(field, rawPath string, allowBuildRoot bool) (string, error) { +func cleanManifestRelativeBuildPath(field, rawPath string) (string, error) { if strings.ContainsAny(rawPath, "\x00\r\n") { return "", fmt.Errorf("%s %q must not contain NUL bytes or line breaks", field, rawPath) } @@ -177,9 +177,6 @@ func cleanManifestRelativeBuildPath(field, rawPath string, allowBuildRoot bool) return "", fmt.Errorf("%s %q must be relative and stay within build directory", field, rawPath) } assetPath = filepath.ToSlash(filepath.Clean(assetPath)) - if assetPath == "." && allowBuildRoot { - return assetPath, nil - } if assetPath == "." || assetPath == ".." || strings.HasPrefix(assetPath, "../") { return "", fmt.Errorf("%s %q must be relative and stay within build directory", field, rawPath) } diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go index de0986d9b319..fed64257252f 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -296,7 +296,7 @@ func buildPartitionFromManifestV2(raw manifestcapnpv2.PartitionInfo) (BuildParti } // Validate shared artifact fields once, after the selected geometry is accepted. - partition.PartPath, err = cleanManifestRelativeBuildPath(subject+" path", path, false) + partition.PartPath, err = cleanManifestRelativeBuildPath(subject+" path", path) if err != nil { return BuildPartition{}, false, err } @@ -361,6 +361,5 @@ func runtimeTokenEmbeddingsPathFromManifestV2(artifacts manifestcapnpv2.Artifact return cleanManifestRelativeBuildPath( fmt.Sprintf("%s artifacts.runtimeAssets.tokenEmbeddingsPath", gbuildManifestV2CapnpFile), rawPath, - false, ) } From cedf1b277303d8d27389fa11ea59cbe50cdc0e3a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 16:28:22 +0000 Subject: [PATCH 147/196] refactor(operator): remove unused LPX pending error Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph_lpx.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 2720cd315bf2..f804d2edf447 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -8,7 +8,6 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "errors" "fmt" "maps" "slices" @@ -120,9 +119,6 @@ func LPXRestartToken(source *v1beta1.DynamoGraphDeployment, previous string) str return previous } -// ErrLPXSourcePending means the source and child snapshots await a matching handoff. -var ErrLPXSourcePending = errors.New("LPXGraphDeployment is waiting for the current source input revision") - // lpxInputRevisionPayload is the normalized source intent hashed for inputRevision. // Keep complete selection and rendering inputs so nested role templates and build // settings cannot silently fall out of the revision. From 0a4bcabf7c30a12aecec454ace4b78b60e59c47a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 17:22:27 +0000 Subject: [PATCH 148/196] fix(operator): preserve failover group availability Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/grove.go | 134 +------- deploy/operator/internal/dynamo/grove_test.go | 314 +++--------------- 2 files changed, 64 insertions(+), 384 deletions(-) diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index e5f48f54cbe8..366066f270c9 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -3,10 +3,8 @@ package dynamo import ( "context" "fmt" - "strconv" "strings" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/go-logr/logr" @@ -22,7 +20,6 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" ) @@ -141,104 +138,6 @@ func ComponentNameBudget(component *v1beta1.DynamoComponentDeploymentSharedSpec) return len(componentName) } -func observeComponentPCSGReadiness( - ctx context.Context, - reader client.Reader, - pcs *grovev1alpha1.PodCliqueSet, - resourceName, - namespace, - component string, - dgdReplicas *int32, -) (groveComponentReadiness, error) { - logger := log.FromContext(ctx) - pcsg := &grovev1alpha1.PodCliqueScalingGroup{} - err := reader.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: namespace}, pcsg) - if err != nil && !errors.IsNotFound(err) { - return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) - } - - // A cached group from a previous PCS instance cannot contribute readiness or status. - if errors.IsNotFound(err) || pcs == nil || !metav1.IsControlledBy(pcsg, pcs) { - status := v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, - ComponentNames: []string{resourceName}, - } - return groveComponentReadiness{status: status}.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - - // Reuse the native observation while retaining the component's requested capacity for cutover. - desiredReplicas := ptr.Deref(dgdReplicas, pcsg.Spec.Replicas) - componentReadiness := pcsgReadiness(pcsg, logger) - componentReadiness.revision.desiredReplicas = desiredReplicas - if !componentReadiness.revision.generationObserved { - return componentReadiness, nil - } - - if pcsg.Spec.Replicas != desiredReplicas { - return componentReadiness.withResult(false, fmt.Sprintf("desired replicas=%d, PodCliqueScalingGroup spec replicas=%d", desiredReplicas, pcsg.Spec.Replicas), v1beta1.DGDReadyReasonUpdating), nil - } - - acceptedPCSRevisionHash := getAcceptedPCSRevisionHash(pcs) - if acceptedPCSRevisionHash == nil { - return componentReadiness.withResult(false, "PodCliqueSet revision is not accepted", v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - if pcsg.Status.CurrentPodCliqueSetGenerationHash == nil || *pcsg.Status.CurrentPodCliqueSetGenerationHash != *acceptedPCSRevisionHash { - return componentReadiness.withResult(false, "PodCliqueScalingGroup has not applied the current PodCliqueSet generation", v1beta1.DGDReadyReasonUpdating), nil - } - - if desiredReplicas == 0 { - return componentReadiness.withResult(true, "", ""), nil - } - - podCliqueList := &grovev1alpha1.PodCliqueList{} - if err := reader.List( - ctx, - podCliqueList, - client.InNamespace(namespace), - client.MatchingLabels{grovecommon.LabelPodCliqueScalingGroup: resourceName, commonconsts.KubeLabelDynamoComponent: component}, - ); err != nil { - logger.V(1).Info("Failed to get PodCliques for PodCliqueScalingGroup", "error", err, "resourceName", resourceName) - return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliques for PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) - } - - if len(podCliqueList.Items) == 0 { - logger.V(2).Info("Failed to find PodCliques for PodCliqueScalingGroup", "resourceName", resourceName) - return componentReadiness.withResult(false, groveResourceNotFoundReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - - podCliquesByReplica := make(map[int][]grovev1alpha1.PodClique) - - for _, clique := range podCliqueList.Items { - // Matching labels do not establish ownership of a recreated group's children. - if !metav1.IsControlledBy(&clique, pcsg) { - continue - } - - index, err := strconv.Atoi(clique.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex]) - if err != nil { - logger.V(1).Info("Failed to read PodClique metadata", "error", err, "resourceName", clique.Name) - return groveComponentReadiness{}, fmt.Errorf("failed to read PodClique metadata %s/%s: %w", namespace, clique.Name, err) - } - podCliquesByReplica[index] = append(podCliquesByReplica[index], clique) - } - - for replica := int32(0); replica < desiredReplicas; replica++ { - cliques := podCliquesByReplica[int(replica)] - if len(cliques) == 0 { - return componentReadiness.withResult(false, fmt.Sprintf("%s: %s", resourceName, v1beta1.DGDReadyReasonSomeResourcesNotReady), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil - } - - for i := range cliques { - cliqueReadiness := podCliqueReadiness(&cliques[i], logger) - if !cliqueReadiness.ready { - return componentReadiness.withResult(false, fmt.Sprintf("%s: %s", resourceName, cliqueReadiness.reason), cliqueReadiness.classification), nil - } - } - } - - return componentReadiness, nil -} - // evaluateGroveComponents is the single per-component evaluation loop shared // by all public Grove readiness views. // @@ -266,15 +165,7 @@ func evaluateGroveComponents( var checkErr error resourceName := GroveComponentResourceName(dgd, componentName) if component.UsesPCSG() { - componentReadiness, checkErr = observeComponentPCSGReadiness( - ctx, - reader, - pcs, - resourceName, - dgd.Namespace, - componentName, - component.Replicas, - ) + componentReadiness, checkErr = observePCSGReadiness(ctx, reader, resourceName, dgd.Namespace, logger) } else { componentReadiness, checkErr = observePodCliqueReadiness(ctx, reader, resourceName, dgd.Namespace, component.Replicas, logger) } @@ -692,12 +583,7 @@ func observePCSGReadiness(ctx context.Context, reader client.Reader, resourceNam logger.V(1).Info("Failed to get PodCliqueScalingGroup", "error", err, "resourceName", resourceName) return groveComponentReadiness{}, fmt.Errorf("failed to get PodCliqueScalingGroup %s/%s: %w", namespace, resourceName, err) } - return pcsgReadiness(pcsg, logger), nil -} -// pcsgReadiness observes a nonnil native scaling group without mutating it. -func pcsgReadiness(pcsg *grovev1alpha1.PodCliqueScalingGroup, logger logr.Logger) groveComponentReadiness { - resourceName := pcsg.Name desiredReplicas := pcsg.Spec.Replicas availableReplicas := pcsg.Status.AvailableReplicas updatedReplicas := pcsg.Status.UpdatedReplicas @@ -739,26 +625,26 @@ func pcsgReadiness(pcsg *grovev1alpha1.PodCliqueScalingGroup, logger logr.Logger if observedGeneration == nil { logger.V(1).Info("PodCliqueScalingGroup observedGeneration is nil", "resourceName", resourceName) - return componentReadiness.withResult(false, groveObservedGenerationNilReason, v1beta1.DGDReadyReasonSomeResourcesNotReady) + return componentReadiness.withResult(false, groveObservedGenerationNilReason, v1beta1.DGDReadyReasonSomeResourcesNotReady), nil } if *observedGeneration < generation { logger.V(1).Info("PodCliqueScalingGroup spec not yet processed", "resourceName", resourceName, "generation", generation, "observedGeneration", observedGeneration) - return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady) + return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady), nil } componentReadiness.status.ScheduledReplicas = &scheduledReplicas if desiredReplicas == 0 { // No replicas desired, so it's ready. - return componentReadiness.withResult(true, "", "") + return componentReadiness.withResult(true, "", ""), nil } // Fully ready: replicas exist, are updated, and are available. Checked // first so a healthy PCSG is never mis-diagnosed as InsufficientCapacity // when Grove does not populate scheduledReplicas on a ready group. if replicas == desiredReplicas && updatedReplicas == desiredReplicas && availableReplicas == desiredReplicas { - return componentReadiness.withResult(true, "", "") + return componentReadiness.withResult(true, "", ""), nil } // Not ready: the explicit MinAvailableBreached scheduling condition, @@ -773,26 +659,26 @@ func pcsgReadiness(pcsg *grovev1alpha1.PodCliqueScalingGroup, logger logr.Logger (cond.Reason == groveconstants.ConditionReasonScheduledReplicasBelowMinAvailable || cond.Reason == legacyConditionReasonInsufficientScheduledPCSGReplicas) { logger.V(1).Info("PodCliqueScalingGroup MinAvailableBreached reports insufficient capacity", "resourceName", resourceName, "reason", cond.Reason, "message", cond.Message) - return componentReadiness.withResult(false, fmt.Sprintf("min-available breached (%s): %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity) + return componentReadiness.withResult(false, fmt.Sprintf("min-available breached (%s): %s", cond.Reason, cond.Message), v1beta1.DGDReadyReasonInsufficientCapacity), nil } if scheduledReplicas > 0 && scheduledReplicas < desiredReplicas { logger.V(1).Info("PodCliqueScalingGroup partially scheduled", "resourceName", resourceName, "desired", desiredReplicas, "scheduled", scheduledReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity) + return componentReadiness.withResult(false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduledReplicas, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity), nil } if desiredReplicas != updatedReplicas { logger.V(1).Info("PodCliqueScalingGroup not fully updated", "resourceName", resourceName, "desired", desiredReplicas, "updated", updatedReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating) + return componentReadiness.withResult(false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, updatedReplicas), v1beta1.DGDReadyReasonUpdating), nil } if replicas != desiredReplicas { logger.V(1).Info("PodCliqueScalingGroup performing rolling update", "resourceName", resourceName, "desired", desiredReplicas, "replicas", replicas) - return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating) + return componentReadiness.withResult(false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, replicas), v1beta1.DGDReadyReasonUpdating), nil } // Scheduled and rolled out, but not enough available replicas. logger.V(1).Info("PodCliqueScalingGroup not ready", "resourceName", resourceName, "desired", desiredReplicas, "available", availableReplicas) - return componentReadiness.withResult(false, fmt.Sprintf("scheduled but available=%d/%d", availableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady) + return componentReadiness.withResult(false, fmt.Sprintf("scheduled but available=%d/%d", availableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady), nil } // specToGroveTopologyConstraint converts a deployment-level topology constraint diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index 86a082a0028c..89e49d7b29c0 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -12,7 +12,6 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/onsi/gomega" @@ -22,7 +21,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -1031,38 +1029,27 @@ func TestEvaluateGroveReadinessUsesDeclaredLayout(t *testing.T) { g.Expect(readiness.ComponentStatuses).To(gomega.HaveLen(1)) g.Expect(readiness.ComponentStatuses["Worker"].ComponentKind).To(gomega.Equal(wantKind)) g.Expect(readiness.ComponentStatuses["Worker"].ComponentNames).To(gomega.Equal([]string{wantName})) + + if dgd.Spec.Components[0].UsesPCSG() { + t.Log("Ordinary and GMS layouts use group availability without requiring child observations") + g.Expect(reader.Create(t.Context(), &grovev1alpha1.PodCliqueScalingGroup{ + ObjectMeta: metav1.ObjectMeta{Name: wantName, Namespace: dgd.Namespace, Generation: 1}, + Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 2}, + Status: grovev1alpha1.PodCliqueScalingGroupStatus{ + Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, ObservedGeneration: ptr.To(int64(1)), + }, + })).To(gomega.Succeed()) + readiness, err = EvaluateGroveReadiness(t.Context(), reader, dgd, pcs) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(readiness.Ready).To(gomega.BeTrue()) + } }) } } func TestEvaluateGroveReadiness(t *testing.T) { ctx := context.Background() - newPCSGPodClique := func(name, pcsg, component string, groupReplica, replicas, readyReplicas int32) *grovev1alpha1.PodClique { - return &grovev1alpha1.PodClique{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - Labels: map[string]string{ - grovecommon.LabelPartOfKey: "test-dgd", - grovecommon.LabelPodCliqueScalingGroup: pcsg, - grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(groupReplica), - commonconsts.KubeLabelDynamoComponent: component, - }, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: grovev1alpha1.SchemeGroupVersion.String(), Kind: "PodCliqueScalingGroup", - Name: pcsg, UID: types.UID(pcsg), Controller: ptr.To(true), - }}, - }, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: replicas}, - Status: grovev1alpha1.PodCliqueStatus{ - Replicas: replicas, - UpdatedReplicas: replicas, - ReadyReplicas: readyReplicas, - ScheduledReplicas: replicas, - ObservedGeneration: ptr.To(int64(1)), - }, - } - } + tests := []struct { name string dgdSpec v1alpha1.DynamoGraphDeploymentSpec @@ -1139,7 +1126,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ - newReadyPCS("test-dgd"), &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-decode", @@ -1149,12 +1135,11 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 2, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), - CurrentPodCliqueSetGenerationHash: ptr.To("current"), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 2, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), }, }, &grovev1alpha1.PodCliqueScalingGroup{ @@ -1166,19 +1151,13 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 3, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 3, - UpdatedReplicas: 3, - AvailableReplicas: 3, - ScheduledReplicas: 3, - ObservedGeneration: ptr.To(int64(1)), - CurrentPodCliqueSetGenerationHash: ptr.To("current"), + Replicas: 3, + UpdatedReplicas: 3, + AvailableReplicas: 3, + ScheduledReplicas: 3, + ObservedGeneration: ptr.To(int64(1)), }, }, - newPCSGPodClique("test-dgd-0-decode-0", "test-dgd-0-decode", "decode", 0, 1, 1), - newPCSGPodClique("test-dgd-0-decode-1", "test-dgd-0-decode", "decode", 1, 1, 1), - newPCSGPodClique("test-dgd-0-prefill-0", "test-dgd-0-prefill", "prefill", 0, 1, 1), - newPCSGPodClique("test-dgd-0-prefill-1", "test-dgd-0-prefill", "prefill", 1, 1, 1), - newPCSGPodClique("test-dgd-0-prefill-2", "test-dgd-0-prefill", "prefill", 2, 1, 1), }, wantReady: true, wantReason: "", @@ -1202,7 +1181,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, { - name: "multinode service checks child PodClique readiness", + name: "multinode service not ready - PCSG not ready", dgdSpec: v1alpha1.DynamoGraphDeploymentSpec{ Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { @@ -1217,7 +1196,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ - newReadyPCS("test-dgd"), &grovev1alpha1.PodCliqueScalingGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-worker", @@ -1227,26 +1205,23 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 2, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), - CurrentPodCliqueSetGenerationHash: ptr.To("current"), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 1, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), }, }, - newPCSGPodClique("test-dgd-0-worker-0", "test-dgd-0-worker", "worker", 0, 1, 1), - newPCSGPodClique("test-dgd-0-worker-1", "test-dgd-0-worker", "worker", 1, 1, 0), }, wantReady: false, - wantReason: "worker: test-dgd-0-worker: scheduled but ready=0/1", + wantReason: "worker: scheduled but available=1/2", wantServiceStatuses: map[string]v1beta1.ComponentReplicaStatus{ "worker": { ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{"test-dgd-0-worker"}, Replicas: 2, UpdatedReplicas: 2, - AvailableReplicas: ptr.To(int32(2)), + AvailableReplicas: ptr.To(int32(1)), ScheduledReplicas: ptr.To(int32(2)), }, }, @@ -1282,7 +1257,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { }, }, existingGroveResources: []client.Object{ - newReadyPCS("test-dgd"), &grovev1alpha1.PodClique{ ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd-0-frontend", @@ -1308,12 +1282,11 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 1, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), - CurrentPodCliqueSetGenerationHash: ptr.To("current"), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 1, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), }, }, &grovev1alpha1.PodCliqueScalingGroup{ @@ -1325,18 +1298,13 @@ func TestEvaluateGroveReadiness(t *testing.T) { Replicas: 2, }, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, - UpdatedReplicas: 2, - AvailableReplicas: 2, - ScheduledReplicas: 2, - ObservedGeneration: ptr.To(int64(1)), - CurrentPodCliqueSetGenerationHash: ptr.To("current"), + Replicas: 2, + UpdatedReplicas: 2, + AvailableReplicas: 2, + ScheduledReplicas: 2, + ObservedGeneration: ptr.To(int64(1)), }, }, - newPCSGPodClique("test-dgd-0-decode-0", "test-dgd-0-decode", "decode", 0, 1, 1), - newPCSGPodClique("test-dgd-0-decode-1", "test-dgd-0-decode", "decode", 1, 1, 1), - newPCSGPodClique("test-dgd-0-prefill-0", "test-dgd-0-prefill", "prefill", 0, 1, 1), - newPCSGPodClique("test-dgd-0-prefill-1", "test-dgd-0-prefill", "prefill", 1, 1, 1), }, wantReady: false, wantReason: "decode: scheduled but available=1/2", @@ -1413,41 +1381,6 @@ func TestEvaluateGroveReadiness(t *testing.T) { var objects []client.Object objects = append(objects, betaDGD) objects = append(objects, tt.existingGroveResources...) - var observedPCS *grovev1alpha1.PodCliqueSet - for _, object := range tt.existingGroveResources { - if pcs, ok := object.(*grovev1alpha1.PodCliqueSet); ok { - observedPCS = pcs.DeepCopy() - break - } - } - if observedPCS != nil { - t.Log("Give each scaling group the current PCS controller identity") - for _, object := range tt.existingGroveResources { - if group, ok := object.(*grovev1alpha1.PodCliqueScalingGroup); ok { - group.UID = types.UID(group.Name) - group.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(observedPCS, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - } - } - - if betaDGD.Status.Components == nil { - betaDGD.Status.Components = make(map[string]v1beta1.ComponentReplicaStatus) - } - for i := range betaDGD.Spec.Components { - component := &betaDGD.Spec.Components[i] - if !IsWorkerComponent(string(component.ComponentType)) { - continue - } - observedPCS.Spec.Template.Cliques = append( - observedPCS.Spec.Template.Cliques, - &grovev1alpha1.PodCliqueTemplateSpec{Labels: map[string]string{ - commonconsts.KubeLabelDynamoComponent: component.ComponentName, - }}, - ) - status := betaDGD.Status.Components[component.ComponentName] - status.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) - betaDGD.Status.Components[component.ComponentName] = status - } - } fakeKubeClient := fake.NewClientBuilder(). WithScheme(s). @@ -1455,7 +1388,7 @@ func TestEvaluateGroveReadiness(t *testing.T) { WithStatusSubresource(objects...). Build() - readiness, err := EvaluateGroveReadiness(ctx, fakeKubeClient, betaDGD, observedPCS) + readiness, err := EvaluateGroveReadiness(ctx, fakeKubeClient, betaDGD, nil) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(readiness.Ready).To(gomega.Equal(tt.wantReady)) @@ -1468,7 +1401,9 @@ func TestEvaluateGroveReadiness(t *testing.T) { if component == nil { continue } - wantStatus.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) + if !IsWorkerComponent(string(component.ComponentType)) { + wantStatus.RuntimeNamespace = betaDGD.GetDynamoNamespaceForComponent(component) + } wantServiceStatuses[componentName] = wantStatus } g.Expect(readiness.ComponentStatuses).To(gomega.Equal(wantServiceStatuses)) @@ -2244,17 +2179,12 @@ func TestCheckPodCliqueReadyClassification(t *testing.T) { } } -func TestPCSGBackedComponentReadyClassification(t *testing.T) { +func TestCheckPCSGReadyClassification(t *testing.T) { ctx := context.Background() tests := []struct { name string pcsg *grovev1alpha1.PodCliqueScalingGroup - dgdReplicas *int32 - pcsgReplicas *int32 - oldPCS bool - oldPCSG bool - foreignClique bool wantReady bool wantClassification string wantReasonContains string @@ -2265,71 +2195,15 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), - dgdReplicas: ptr.To(int32(2)), wantReady: true, wantClassification: "", }, - { - name: "omitted DGD replicas defer to the externally scaled PCSG", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, - ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), - }), - wantReady: true, - }, - { - name: "ready scaling group belongs to the previous same-name PCS", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, - ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), - }), - oldPCS: true, - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - }, - { - name: "zero-replica scaling group belongs to the previous same-name PCS", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - ObservedGeneration: ptr.To(int64(1)), - }), - pcsgReplicas: ptr.To(int32(0)), - oldPCS: true, - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - }, - { - name: "ready cliques belong to the previous same-name scaling group", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, - ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), - }), - oldPCSG: true, - wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, - }, - { - name: "foreign clique with a malformed replica label does not affect readiness", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, - ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), - }), - foreignClique: true, - wantReady: true, - }, - { - name: "pinned DGD replicas take precedence over PCSG replicas", - pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ - Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 2, - ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), - }), - dgdReplicas: ptr.To(int32(3)), - wantClassification: v1beta1.DGDReadyReasonUpdating, - wantReasonContains: "desired replicas=3", - }, { name: "available below desired, fully scheduled -> PodsNotReady", pcsg: newPCSG(grovev1alpha1.PodCliqueScalingGroupStatus{ Replicas: 2, AvailableReplicas: 0, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonPodsNotReady, wantReasonContains: "available=0/2", @@ -2340,7 +2214,6 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 0, UpdatedReplicas: 2, ScheduledReplicas: 1, ObservedGeneration: ptr.To(int64(1)), }), - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonInsufficientCapacity, wantReasonContains: "scheduled=1/2", @@ -2355,11 +2228,10 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { // Status=False (Status=True is the availability reason). Type: groveconstants.ConditionTypeMinAvailableBreached, Status: metav1.ConditionFalse, - Reason: legacyConditionReasonInsufficientScheduledPCSGReplicas, + Reason: "InsufficientScheduledPodCliqueScalingGroupReplicas", LastTransitionTime: metav1.Now(), }}, }), - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonInsufficientCapacity, wantReasonContains: "min-available breached", @@ -2370,7 +2242,6 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 2, UpdatedReplicas: 1, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonUpdating, wantReasonContains: "updated=1", @@ -2381,7 +2252,6 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { Replicas: 2, AvailableReplicas: 1, UpdatedReplicas: 2, ScheduledReplicas: 2, ObservedGeneration: ptr.To(int64(1)), }), - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonPodsNotReady, wantReasonContains: "available=1/2", @@ -2389,7 +2259,6 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { { name: "not found -> Unclassified", pcsg: nil, - dgdReplicas: ptr.To(int32(2)), wantReady: false, wantClassification: v1beta1.DGDReadyReasonSomeResourcesNotReady, wantReasonContains: "resource not found", @@ -2400,71 +2269,18 @@ func TestPCSGBackedComponentReadyClassification(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := gomega.NewGomegaWithT(t) - t.Log("Build the observed PCS, scaling group, and owned ready cliques") - pcs := newReadyPCS("pcs") - objs := []client.Object{pcs} + var objs []client.Object if tt.pcsg != nil { - tt.pcsg.UID = "current-pcsg" - tt.pcsg.Spec.Replicas = ptr.Deref(tt.pcsgReplicas, tt.pcsg.Spec.Replicas) - tt.pcsg.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))} - if tt.oldPCS { - tt.pcsg.OwnerReferences[0].UID = "previous-pcs" - } - tt.pcsg.Status.CurrentPodCliqueSetGenerationHash = ptr.To("current") objs = append(objs, tt.pcsg) - cliqueOwner := *metav1.NewControllerRef(tt.pcsg, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueScalingGroup")) - if tt.oldPCSG { - cliqueOwner.UID = "previous-pcsg" - } - for replica := range testPCSGReplicas { - objs = append(objs, &grovev1alpha1.PodClique{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("pcsg-%d-component", replica), - Namespace: "default", - Generation: 1, - Labels: map[string]string{ - grovecommon.LabelPodCliqueScalingGroup: testPCSGName, - grovecommon.LabelPodCliqueScalingGroupReplicaIndex: fmt.Sprint(replica), - commonconsts.KubeLabelDynamoComponent: "component", - }, - OwnerReferences: []metav1.OwnerReference{cliqueOwner}, - }, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, - Status: grovev1alpha1.PodCliqueStatus{ - Replicas: 1, - UpdatedReplicas: 1, - ReadyReplicas: 1, - ScheduledReplicas: 1, - ObservedGeneration: ptr.To(int64(1)), - }, - }) - } - if tt.foreignClique { - foreign := objs[len(objs)-1].(*grovev1alpha1.PodClique).DeepCopy() - foreign.Name = "foreign-clique" - foreign.OwnerReferences[0].UID = "previous-pcsg" - foreign.Labels[grovecommon.LabelPodCliqueScalingGroupReplicaIndex] = "invalid" - objs = append(objs, foreign) - } } - - t.Log("Evaluate readiness against the current controller identities") c := newFakeGroveClient(g, objs...) - readiness, checkErr := observeComponentPCSGReadiness( - ctx, c, pcs, testPCSGName, "default", "component", tt.dgdReplicas, - ) + ready, reason, _, classification, checkErr := CheckPCSGReady(ctx, c, testPCSGName, "default", log.FromContext(ctx)) g.Expect(checkErr).NotTo(gomega.HaveOccurred()) - t.Log("Verify foreign resources cannot contribute readiness or replica status") - g.Expect(readiness.ready).To(gomega.Equal(tt.wantReady)) - g.Expect(readiness.classification).To(gomega.Equal(tt.wantClassification)) - if tt.oldPCS { - g.Expect(readiness.status).To(gomega.Equal(v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodCliqueScalingGroup, ComponentNames: []string{testPCSGName}, - })) - } + g.Expect(ready).To(gomega.Equal(tt.wantReady)) + g.Expect(classification).To(gomega.Equal(tt.wantClassification)) if tt.wantReasonContains != "" { - g.Expect(readiness.reason).To(gomega.ContainSubstring(tt.wantReasonContains)) + g.Expect(reason).To(gomega.ContainSubstring(tt.wantReasonContains)) } }) } @@ -2500,16 +2316,6 @@ func newPCSG(status grovev1alpha1.PodCliqueScalingGroupStatus) *grovev1alpha1.Po } } -func newReadyPCS(name string) *grovev1alpha1.PodCliqueSet { - return &grovev1alpha1.PodCliqueSet{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Generation: 1, UID: types.UID(name)}, - Status: grovev1alpha1.PodCliqueSetStatus{ - ObservedGeneration: ptr.To(int64(1)), - CurrentGenerationHash: ptr.To("current"), - }, - } -} - func newFakeGroveClient(g *gomega.WithT, objects ...client.Object) client.Client { s := scheme.Scheme g.Expect(grovev1alpha1.AddToScheme(s)).To(gomega.Succeed()) @@ -2570,18 +2376,6 @@ func TestGroveReadinessTransientErrorsPropagate(t *testing.T) { g.Expect(classification).To(gomega.BeEmpty()) }) - t.Run("PCSG-backed component returns error on non-NotFound get failure", func(t *testing.T) { - g := gomega.NewGomegaWithT(t) - c := newClient(g) - readiness, err := observeComponentPCSGReadiness( - ctx, c, newReadyPCS("pcs"), "test-pcsg", "default", "component", ptr.To(int32(1)), - ) - g.Expect(err).To(gomega.HaveOccurred()) - g.Expect(err.Error()).To(gomega.ContainSubstring("transient API error")) - g.Expect(readiness.ready).To(gomega.BeFalse()) - g.Expect(readiness.classification).To(gomega.BeEmpty()) - }) - t.Run("EvaluateGroveReadiness propagates a child read error", func(t *testing.T) { g := gomega.NewGomegaWithT(t) dgd := betaDGD(t, &v1alpha1.DynamoGraphDeployment{ From e1e12de28be1073cee47700347bfae457f89a695 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 19:59:26 +0000 Subject: [PATCH 149/196] fix(operator): deliver ordinary restarts before persisting selection Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_program.go | 11 ++-- .../controller/dgd_lpx_handoff_test.go | 53 +++++++++++++------ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index f85b3c923bb0..19bf536dfd08 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -147,11 +147,6 @@ func (p *groveProgram) Reconcile( ) recordRestartTransition(previousRestart, restart.Status, &programResult) programResult.Status.Restart = restart.Status - if req.DGD.HasLPXComponent() && !apiequality.Semantic.DeepEqual(req.DGD.Status.Restart, restart.Status) { - // Persist the selected restart before delivering its token to the child. - programResult.RequeueAfter = time.Nanosecond - return programResult, nil - } result, err := p.workloads.Reconcile( ctx, @@ -169,6 +164,12 @@ func (p *groveProgram) Reconcile( return programResult, fmt.Errorf("failed to reconcile Grove workloads: %w", err) } + if req.DGD.HasLPXComponent() && !apiequality.Semantic.DeepEqual(req.DGD.Status.Restart, restart.Status) { + // Persist the selected restart before delivering its token to the child. + programResult.RequeueAfter = time.Nanosecond + return programResult, nil + } + // Keep LPX creation and updates after ordinary reconciliation and restart selection. var child *nvidiacomv1alpha1.LPXGraphDeployment if req.DGD.HasLPXComponent() { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 0df94b15e662..c7733963e346 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -391,7 +391,7 @@ func TestLPXRestartUsesPersistedSelectionAndCurrentChildStatus(t *testing.T) { } func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testing.T) { - t.Log("Request one parallel restart of two independent LPX workloads and a frontend") + t.Log("Create two independent LPX workloads and an already-ready frontend") child, source, kube := newLPXHandoffFixture(t, "node-local-v2-lpu-only") require.NoError(t, rbacv1.AddToScheme(kube.Scheme())) second := source.Spec.Components[0].DeepCopy() @@ -401,7 +401,6 @@ func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testi PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend"}}}}, }) source.Annotations[consts.KubeAnnotationDynamoDiscoveryBackend] = string(configv1alpha1.DiscoveryBackendKubernetes) - source.Spec.Restart = &v1beta1.Restart{ID: "parallel-restart", Strategy: &v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeParallel}} require.NoError(t, kube.Update(t.Context(), source)) config := &configv1alpha1.OperatorConfiguration{} config.Namespace.Restricted = source.Namespace @@ -409,15 +408,46 @@ func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testi Client: kube, Config: config, Recorder: events.NewFakeRecorder(100), RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, }).newGroveProgram() + program.workloads.scaler.client = interceptor.NewClient(kube.(client.WithWatch), groveScaleInterceptor(interceptor.Funcs{}, nil)) + + _, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) + ordinary := projectWithoutExternallyManagedComponents(source) + pcs := &grovev1alpha1.PodCliqueSet{} + require.NoError(t, kube.Get(t.Context(), client.ObjectKey{ + Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), + }, pcs)) + pcs.Generation = 1 + pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) + require.NoError(t, kube.Update(t.Context(), pcs)) + frontend := &grovev1alpha1.PodClique{ + ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(ordinary, "frontend"), Namespace: source.Namespace, Generation: 1}, + Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, + Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, + } + require.NoError(t, kube.Create(t.Context(), frontend)) - t.Log("Persist selection of all three components before publishing their restart") + t.Log("Deliver the ordinary restart before persisting selection; LPX still waits for persistence") + source.Spec.Restart = &v1beta1.Restart{ID: "parallel-restart", Strategy: &v1beta1.RestartStrategy{Type: v1beta1.RestartStrategyTypeParallel}} + require.NoError(t, kube.Update(t.Context(), source)) result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) require.NoError(t, err) require.Equal(t, []string{"frontend", "lpx", "second"}, result.Status.Restart.InProgress) + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) + require.Len(t, pcs.Spec.Template.Cliques, 1) + require.Equal(t, source.Spec.Restart.ID, pcs.Spec.Template.Cliques[0].Annotations[consts.RestartAnnotation]) + require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), child)) + require.Empty(t, child.Annotations[dynamo.LPXRestartAnnotation]) source.Status = result.Status require.NoError(t, kube.Status().Update(t.Context(), source)) - t.Log("Deliver the same token to LPX and the ordinary Pod template while every workload is pending") + t.Log("Model the API-server generation change and Grove's still-unready replacement") + pcs.Generation++ + require.NoError(t, kube.Update(t.Context(), pcs)) + frontend.Status.ReadyReplicas = 0 + require.NoError(t, kube.Update(t.Context(), frontend)) + + t.Log("Deliver the same token to LPX while the ordinary restart is pending") result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) require.NoError(t, err) require.Equal(t, v1beta1.DGDStatePending, result.Status.State) @@ -425,14 +455,6 @@ func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testi require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(child), child)) require.Equal(t, source.Spec.Restart.ID, child.Annotations[dynamo.LPXRestartAnnotation]) require.Empty(t, child.Status.Components) - ordinary := projectWithoutExternallyManagedComponents(source) - pcs := &grovev1alpha1.PodCliqueSet{} - require.NoError(t, kube.Get(t.Context(), client.ObjectKey{ - Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), - }, pcs)) - require.Len(t, pcs.Spec.Template.Cliques, 1) - require.Equal(t, source.Spec.Restart.ID, pcs.Spec.Template.Cliques[0].Annotations[consts.RestartAnnotation]) - require.Nil(t, pcs.Status.ObservedGeneration) source.Status = result.Status require.NoError(t, kube.Status().Update(t.Context(), source)) @@ -465,11 +487,8 @@ func TestParallelRestartStartsAllLPXAndOrdinaryWorkloadsBeforeReadiness(t *testi require.NoError(t, kube.Get(t.Context(), client.ObjectKeyFromObject(pcs), pcs)) pcs.Status.ObservedGeneration = ptr.To(pcs.Generation) require.NoError(t, kube.Update(t.Context(), pcs)) - require.NoError(t, kube.Create(t.Context(), &grovev1alpha1.PodClique{ - ObjectMeta: metav1.ObjectMeta{Name: dynamo.GroveComponentResourceName(ordinary, "frontend"), Namespace: source.Namespace, Generation: 1}, - Spec: grovev1alpha1.PodCliqueSpec{Replicas: 1}, - Status: grovev1alpha1.PodCliqueStatus{ObservedGeneration: ptr.To(int64(1)), Replicas: 1, UpdatedReplicas: 1, ReadyReplicas: 1}, - })) + frontend.Status.ReadyReplicas = 1 + require.NoError(t, kube.Update(t.Context(), frontend)) result, err = program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) require.NoError(t, err) require.Equal(t, v1beta1.RestartPhaseCompleted, result.Status.Restart.Phase) From f62195272cd3bbb1d4847c2c61194917de228c5d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 19:59:26 +0000 Subject: [PATCH 150/196] refactor(operator): remove redundant backend parsing Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index d4532b277bac..6dd2d61ae860 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -332,8 +332,8 @@ func gmsExtraClientContainersError( } func backendFrameworkForGeneratedDCDs(parentDGD *v1beta1.DynamoGraphDeployment) (string, error) { - if backendFramework, err := ParseBackendFramework(parentDGD.Spec.BackendFramework); err == nil { - return string(backendFramework), nil + if parentDGD.Spec.BackendFramework != "" { + return parentDGD.Spec.BackendFramework, nil } var detected BackendFramework @@ -1406,18 +1406,6 @@ const ( BackendFrameworkNoop BackendFramework = "noop" ) -// ParseBackendFramework converts a string to BackendFramework type. -// Returns an error if the framework string is not recognized. -func ParseBackendFramework(framework string) (BackendFramework, error) { - bf := BackendFramework(framework) - switch bf { - case BackendFrameworkVLLM, BackendFrameworkSGLang, BackendFrameworkTRTLLM, BackendFrameworkNoop: - return bf, nil - default: - return "", fmt.Errorf("unsupported backend framework: %s (valid values: vllm, sglang, trtllm, noop)", framework) - } -} - // ContainerGPUCount lazily resolves the main container's scalar or DRA-backed // GPU count. The same resolver can be shared across all roles of a component. type ContainerGPUCount func() (int64, error) @@ -3025,7 +3013,11 @@ func determineBackendFramework( } } - explicitFramework, _ := ParseBackendFramework(explicitBackendFramework) + // Get explicit framework + var explicitFramework BackendFramework + if explicitBackendFramework != "" { + explicitFramework = BackendFramework(explicitBackendFramework) + } // Validate consistency if both detected and explicit exist if detectedFramework != "" && detectedFramework != BackendFrameworkNoop && explicitFramework != "" && detectedFramework != explicitFramework { From 9c9bf58fb83efa5358d25974483d7066330953e4 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 20:54:02 +0000 Subject: [PATCH 151/196] refactor(operator): remove redundant LPX UID annotations Signed-off-by: Andrew Paprotsky --- .../controller/lpx/pipeline_request.go | 2 -- .../lpx/pipeline_request_deadline_test.go | 1 - .../controller/lpx/pipeline_request_test.go | 3 +++ .../internal/controller/lpx/render.go | 15 ++------------ .../internal/controller/lpx/render_test.go | 13 ++++++------ deploy/operator/internal/dynamo/graph_lpx.go | 5 ++--- .../internal/dynamo/lpx/model_projection.go | 4 ---- .../from_dgd_yaml/lpu-cyborg-specdec.yaml | 20 ------------------- .../from_dgd_yaml/lpu-gpu-specdecode.yaml | 10 ---------- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 12 ----------- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 12 ----------- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 10 ---------- .../node-local-v2-specdecode.yaml | 14 ------------- .../node-local-v3-hx-hybrid.yaml | 10 ---------- .../node-local-v3-hx-lpu-only.yaml | 10 ---------- .../node-local-v3-hx-specdecode.yaml | 14 ------------- .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 10 ---------- 17 files changed, 13 insertions(+), 152 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index c287c2c95e01..5b68c2f9d80b 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -85,9 +85,7 @@ func resolvePipelineRequests( deploymentUIDLabel: string(deployment.UID), }, Annotations: map[string]string{ - lpx.DGDUIDAnnotation: string(dgdOwner.UID), lpx.DeploymentNameAnnotation: deployment.Name, - deploymentUIDAnnotation: string(deployment.UID), pipelineRequestModelAnnotation: projection.Model(), lpx.WorkloadDigestAnnotation: projection.Digest().String(), lpxv1alpha1.CompilerSnapshotDigestAnnotation: projection.CompilerSnapshotDigest(), diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 3254f3b39f89..015af8668c3f 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -302,7 +302,6 @@ func newTestPipelineRequest(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev Name: name, Namespace: deployment.Namespace, UID: types.UID(name + "-uid"), ResourceVersion: "1", Generation: generation, CreationTimestamp: metav1.NewTime(created), Labels: map[string]string{deploymentUIDLabel: string(deployment.UID)}, - Annotations: map[string]string{deploymentUIDAnnotation: string(deployment.UID)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(pcs, grovev1alpha1.SchemeGroupVersion.WithKind("PodCliqueSet"))}, }, Spec: lpxv1alpha1.LPUPipelineRequestSpec{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}, diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_test.go index 5229ccd747a7..c951a8bde2a9 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_test.go @@ -201,6 +201,9 @@ func TestNodeLocalSpecDecodePublishesOneRequestAndAgentCliquePerModelProjection( model := request.Annotations[pipelineRequestModelAnnotation] requestByModel[model] = request require.Equal(t, string(deployment.UID), request.Labels[deploymentUIDLabel]) + require.Equal(t, deployment.Name, request.Annotations[lpx.DeploymentNameAnnotation]) + require.NotContains(t, request.Annotations, "lpx.nvidia.com/deployment-uid") + require.NotContains(t, request.Annotations, "scheduling.lpu.nvidia.com/dgd-uid") require.Equal(t, model, request.Spec.NodeLocal.Model) require.Equal(t, desired.plan.LPXScalingGroup, request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name) } diff --git a/deploy/operator/internal/controller/lpx/render.go b/deploy/operator/internal/controller/lpx/render.go index 605df0c9fb99..805f8e0e8356 100644 --- a/deploy/operator/internal/controller/lpx/render.go +++ b/deploy/operator/internal/controller/lpx/render.go @@ -19,14 +19,10 @@ import ( lpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" grovecommon "github.com/ai-dynamo/grove/operator/api/common" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - deploymentUIDAnnotation = dynamo.LPXDeploymentUIDAnnotation - deploymentUIDLabel = deploymentUIDAnnotation -) +const deploymentUIDLabel = "lpx.nvidia.com/deployment-uid" // resolveWorkloads resolves every component group and finalizes its resource names. // deployment and dgd must be non-nil; dgd must have passed admission. @@ -167,18 +163,11 @@ func (r *graphReconciler) renderPodCliqueSet( // stampDeploymentIdentity propagates stable ownership labels and annotations, never DGD revision. func stampDeploymentIdentity(deployment *v1alpha1.LPXGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, resources []client.Object) { - // The initial DGD lookup validated this controller owner; rendering trusts that observation. - dgdOwner := metav1.GetControllerOf(deployment) - identity := map[string]string{ - lpx.DeploymentNameAnnotation: deployment.Name, - deploymentUIDAnnotation: string(deployment.UID), - lpx.DGDUIDAnnotation: string(dgdOwner.UID), - } stamp := func(annotations *map[string]string) { if *annotations == nil { *annotations = make(map[string]string) } - maps.Copy(*annotations, identity) + (*annotations)[lpx.DeploymentNameAnnotation] = deployment.Name } stampOwnerLabel := func(object client.Object) { labels := object.GetLabels() diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 1bda8b3e77bd..abdf534c4f0d 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -218,8 +218,6 @@ func TestLPXRenderingIncludesDiscoveryServices(t *testing.T) { require.Equal(t, map[string]string{ "example.com/description": "serving", lpx.DeploymentNameAnnotation: child.Name, - deploymentUIDAnnotation: string(child.UID), - lpx.DGDUIDAnnotation: string(dgd.UID), }, service.Annotations) require.Equal(t, string(child.UID), service.Labels[deploymentUIDLabel]) require.Equal(t, consts.KubeLabelValueTrue, service.Labels[consts.KubeLabelDynamoDiscoveryEnabled]) @@ -399,7 +397,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { group.Annotations[lpx.WorkloadDigestAnnotation] = mutatedValue } for _, resource := range firstResources { - resource.GetAnnotations()[deploymentUIDAnnotation] = mutatedValue + resource.GetAnnotations()[lpx.DeploymentNameAnnotation] = mutatedValue switch resource := resource.(type) { case *corev1.ConfigMap: resource.Data["rendered-only"] = mutatedValue @@ -426,8 +424,8 @@ func TestLPXRenderingMetadata(t *testing.T) { dgd.Labels = map[string]string{"project": "inference", "unrelated": "ignored"} dgd.Annotations["kai.scheduler/topology"] = "source-topology" dgd.Spec.Annotations = map[string]string{ - "kai.scheduler/topology": "explicit-topology", - lpx.DGDUIDAnnotation: "stale", + "kai.scheduler/topology": "explicit-topology", + lpx.DeploymentNameAnnotation: "stale", } child.Annotations[dynamo.LPXRestartAnnotation] = restartToken dgd.Generation++ @@ -462,8 +460,9 @@ func TestLPXRenderingMetadata(t *testing.T) { annotationMaps = append(annotationMaps, resource.GetAnnotations()) } for _, annotations := range annotationMaps { - require.Equal(t, string(metav1.GetControllerOf(child).UID), annotations[lpx.DGDUIDAnnotation]) - require.Equal(t, string(child.UID), annotations[deploymentUIDAnnotation]) + require.Equal(t, child.Name, annotations[lpx.DeploymentNameAnnotation]) + require.NotContains(t, annotations, "lpx.nvidia.com/deployment-uid") + require.NotContains(t, annotations, "scheduling.lpu.nvidia.com/dgd-uid") require.NotContains(t, annotations, "lpx.nvidia.com/deployment-generation") require.NotContains(t, annotations, "lpx.nvidia.com/input-revision") } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index f804d2edf447..279558bf251e 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -97,9 +97,8 @@ func RenderLPXWorkloadTemplates( } const ( - LPXDeploymentUIDAnnotation = dynamolpx.DeploymentUIDAnnotation - LPXRestartAnnotation = "lpx.nvidia.com/restart-id" - LPXServingLabel = "lpx.nvidia.com/serving" + LPXRestartAnnotation = "lpx.nvidia.com/restart-id" + LPXServingLabel = "lpx.nvidia.com/serving" ) // LPXRestartToken advances only from the DGD's persisted restart selection. diff --git a/deploy/operator/internal/dynamo/lpx/model_projection.go b/deploy/operator/internal/dynamo/lpx/model_projection.go index 126e7efe816d..4f8d86d05711 100644 --- a/deploy/operator/internal/dynamo/lpx/model_projection.go +++ b/deploy/operator/internal/dynamo/lpx/model_projection.go @@ -17,12 +17,8 @@ import ( const ( // DeploymentNameAnnotation routes rendered-object events to their LPXGraphDeployment. DeploymentNameAnnotation = "lpx.nvidia.com/deployment-name" - // DeploymentUIDAnnotation records the owning LPXGraphDeployment UID on rendered objects. - DeploymentUIDAnnotation = "lpx.nvidia.com/deployment-uid" // WorkloadDigestAnnotation records the immutable Dynamo workload projection digest on rendered objects. WorkloadDigestAnnotation = "scheduling.lpu.nvidia.com/dynamo-workload-digest" - // DGDUIDAnnotation records the source DGD UID on rendered objects. - DGDUIDAnnotation = "scheduling.lpu.nvidia.com/dgd-uid" // WorkloadModeAnnotation records the projected LPX workload mode on rendered objects. WorkloadModeAnnotation = "scheduling.lpu.nvidia.com/workload-mode" ) diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml index 13b2b3089ef5..4c2309eb3148 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml @@ -21,8 +21,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: lpu-cyborg-specdec-5f16-draft-lpu-6bcf1f4cf22aaf55 @@ -34,8 +32,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: lpu-cyborg-specdec-5f16-target-decode-32805b2198371ed2 @@ -53,8 +49,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: lpu-cyborg-specdec-5f16-target-lpu-355e8ac6768c54e3 @@ -63,8 +57,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:31a0ee5e56b73e39b7c7b2c93678df6bf858dedaa77350422c3f557fbdb7d5ad labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -78,10 +70,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -164,9 +154,7 @@ spec: roleName: draft-agt - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor labels: @@ -250,10 +238,8 @@ spec: - draft-agt - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -338,9 +324,7 @@ spec: roleName: target-agt - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 32805b2198371ed2ff837e4776e3e59448937c6274efe6602681e42f78f14d41 - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/role: cyborg-worker labels: @@ -456,8 +440,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc cliqueNames: - draft-cond @@ -467,8 +449,6 @@ spec: replicas: 1 - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 cliqueNames: - target-agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml index 4d1417a56a3e..f1eac2d09e7d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml @@ -21,8 +21,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: lpu-gpu-specdecode-bc86-lpu-6bcf1f4cf22aaf55 @@ -31,8 +29,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:26e70c69aa2a43744d6a89c9fffb8c9bbd2711cd6203561d0a97af75e4080798 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -46,10 +42,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -132,9 +126,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor labels: @@ -221,8 +213,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc cliqueNames: - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index bf637f288dbe..ea8af2528a1f 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -24,8 +24,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-lpx-v2-vllm-94cd-decode-54f76f5c988eb748 @@ -162,8 +160,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-lpx-v2-vllm-94cd-lpu-e586b76cc8ccd9a1 @@ -172,8 +168,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:2b28cb9f0acada1e36c2e2f52a29876f852308f21d2d753da6095dc4261589d6 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -187,10 +181,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: e586b76cc8ccd9a1ed2e1b0297032eb1da420881dcfdcb632a4f00f4971308ff scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -398,9 +390,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 scheduling.lpu.nvidia.com/role: cyborg-worker labels: @@ -582,8 +572,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 cliqueNames: - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index f74b76e213a4..d7fcac387307 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -7,8 +7,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-hybrid-9d06-decode-734ce683f6b7ea5f @@ -26,8 +24,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-hybrid-9d06-lpu-355e8ac6768c54e3 @@ -36,8 +32,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:db59c694410ca3a5c6502ab3b9d4de7ec12d4a5c0488fba792685fd35f08fd67 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -51,10 +45,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -213,9 +205,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 scheduling.lpu.nvidia.com/role: cyborg-worker labels: @@ -371,8 +361,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 22222222-2222-4222-8222-222222222222 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 cliqueNames: - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 81d1214c0712..b172cc5a0610 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -21,8 +21,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-lpu-only-33b1-lpu-6bcf1f4cf22aaf55 @@ -31,8 +29,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:561ba3854afee2dd805b99728289b13411f9ebfe379fa60b1262d373c1565f96 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -46,10 +42,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -182,9 +176,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc scheduling.lpu.nvidia.com/role: conductor labels: @@ -359,8 +351,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc cliqueNames: - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index 1af2ad76cd1a..f8db7573467c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -55,8 +55,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v2-specdecode-63df-lpu-b6f18b8ea30b38c5 @@ -65,8 +63,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:eeb1e1a79f8739fa99975c047dbffde2c51cf1d1d5cfedae40ee7a3857a55ab2 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -80,10 +76,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b0943511c591eb7201003374eacf778c4545a959d7be251547399296e35ee1ae scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent @@ -220,10 +214,8 @@ spec: roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d19c12c440463a3ebbc8c86f3fe882a33e7d5f48de493146aaf6d0acfa4fdba7 scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent @@ -360,10 +352,8 @@ spec: roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b06e7ae90a96bd569bf228eb8b1d4b1192c4b0df3af7e33202d1fc1965a5247e scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent @@ -500,9 +490,7 @@ spec: roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 scheduling.lpu.nvidia.com/role: conductor labels: @@ -686,8 +674,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 cliqueNames: - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index debb40b8085d..e2c264e505f0 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -13,8 +13,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-hybrid-be18-lpu-f8b027cf29abf6cd @@ -23,8 +21,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f7d69156466e0ef0f00c871c90da8cd98e6956cb5da9745ec86b462b19dd5d26 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -38,10 +34,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -200,8 +194,6 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 scheduling.lpu.nvidia.com/role: cyborg-worker labels: @@ -339,8 +331,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 cliqueNames: - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index c700f47f2436..dfde2cd563e8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -13,8 +13,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-lpu-only-d43c-lpu-f8b027cf29abf6cd @@ -23,8 +21,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:4de325afa19b861c0592d5037388b4e878bdc0f2991f5d75ef485a933ae7e868 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -38,10 +34,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -188,9 +182,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 scheduling.lpu.nvidia.com/role: conductor labels: @@ -384,8 +376,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 33333333-3333-4333-8333-333333333333 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 cliqueNames: - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 64cb0567f27f..447156f30198 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -34,8 +34,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: node-local-v3-hx-specdecode-1f7a-lpu-19888d3182c23d08 @@ -44,8 +42,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:34698082a40a6249bd1416b3f21bf429f2aac81064563dd6ebf40bcbd61bbd6d labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -59,10 +55,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6b42344f7916f4778ffdef73249a7395bbc7a082466cc457953af5de5544c80f scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent @@ -213,10 +207,8 @@ spec: roleName: agt0 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b94a35c4a019e3213f4497e105522208d2a2707e8469ec13c22d46b99d53c227 scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent @@ -367,10 +359,8 @@ spec: roleName: agt1 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8440780b5cdc30ae4bbcf80fda3cce06027325884a8b69751577112153ab2cf5 scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent @@ -521,9 +511,7 @@ spec: roleName: agt2 - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 scheduling.lpu.nvidia.com/role: conductor labels: @@ -727,8 +715,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 44444444-4444-4444-8444-444444444444 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 cliqueNames: - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index adbe62b41146..17ee88d8b359 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -96,8 +96,6 @@ immutable: true metadata: annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid name: dgd-single-v2-3a98-lpu-1ecba46d5a3c4e27 @@ -106,8 +104,6 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a917077bc269f6b8d4770d022b50dc6208c4b2cc64800c813422b3768321613f labels: lpx.nvidia.com/deployment-uid: lpx-render-uid @@ -121,10 +117,8 @@ spec: cliques: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent @@ -261,9 +255,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 scheduling.lpu.nvidia.com/role: conductor labels: @@ -457,8 +449,6 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - lpx.nvidia.com/deployment-uid: lpx-render-uid - scheduling.lpu.nvidia.com/dgd-uid: 11111111-1111-4111-8111-111111111111 scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 cliqueNames: - cond From a6ca2cf1383330dbd21931f2099fde4f37363e98 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 20:54:21 +0000 Subject: [PATCH 152/196] refactor(operator): trust validated LPX deadline targets Signed-off-by: Andrew Paprotsky --- .../lpx/pipeline_request_deadline.go | 45 +++++-------------- .../lpx/pipeline_request_deadline_test.go | 3 +- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index b408d389e0f2..40bc801bc905 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -7,9 +7,7 @@ package lpx import ( "context" - "fmt" "maps" - "math" "slices" "time" @@ -164,9 +162,9 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( manageReplicas bool, ) error { // Use the observed capacity to decide which ordinals are safe to remove. - replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, pcsg.Spec.Replicas) - if err != nil || len(removed) == 0 { - return err + replicas, removed := expiredPipelineRequestSuffix(requests, expired, pcsg.Spec.Replicas) + if len(removed) == 0 { + return nil } // Persist the lower PCSG count before asynchronous LPR/pod cleanup starts. @@ -185,27 +183,18 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( // evidence for the next reconciliation. An interior failure blocks all // scale-down so a healthy higher ordinal is never removed. // For four replicas, expiry at {2,3} removes that suffix; {0,3} removes nothing. +// All requests must already be validated and target the same scaling group. func expiredPipelineRequestSuffix( requests []*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, replicas int32, -) (int32, []*lpxv1alpha1.LPUPipelineRequest, error) { +) (int32, []*lpxv1alpha1.LPUPipelineRequest) { if len(expired) == 0 { - return replicas, nil, nil - } - groupName, _, err := pipelineRequestScalingGroupTarget(expired[0]) - if err != nil { - return 0, nil, err + return replicas, nil } failedReplicas := make(map[int64]struct{}, len(expired)) for _, request := range expired { - name, replica, err := pipelineRequestScalingGroupTarget(request) - if err != nil { - return 0, nil, err - } - if name != groupName { - return 0, nil, fmt.Errorf("expired LPX requests target different scaling groups") - } + replica := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex failedReplicas[replica] = struct{}{} } @@ -221,7 +210,7 @@ func expiredPipelineRequestSuffix( // Any hole blocks all cleanup, even a separate expired trailing suffix. for replica := range failedReplicas { if replica < targetReplicas { - return replicas, nil, nil + return replicas, nil } } @@ -232,13 +221,7 @@ func expiredPipelineRequestSuffix( siblings := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) for _, request := range requests { - name, replica, err := pipelineRequestScalingGroupTarget(request) - if err != nil { - return 0, nil, err - } - if name != groupName { - return 0, nil, fmt.Errorf("LPX requests target different scaling groups") - } + replica := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex // Include LPRs beyond the live count to finish cleanup after an earlier scale write. if replica < targetReplicas { @@ -250,13 +233,5 @@ func expiredPipelineRequestSuffix( siblings = append(siblings, request) } } - return int32(targetReplicas), append(siblings, failed...), nil -} - -func pipelineRequestScalingGroupTarget(request *lpxv1alpha1.LPUPipelineRequest) (string, int64, error) { - target := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef - if target == nil || target.Name == "" || target.ReplicaIndex < 0 || target.ReplicaIndex >= math.MaxInt32 { - return "", 0, fmt.Errorf("LPX request %q lacks a valid scaling group target", request.Name) - } - return target.Name, target.ReplicaIndex, nil + return int32(targetReplicas), append(siblings, failed...) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 015af8668c3f..2aadfdce8380 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -282,8 +282,7 @@ func TestExpiredPipelineRequestSuffix(t *testing.T) { expired = append(expired, requests[i]) } } - replicas, removed, err := expiredPipelineRequestSuffix(requests, expired, tc.replicas) - require.NoError(t, err) + replicas, removed := expiredPipelineRequestSuffix(requests, expired, tc.replicas) require.Equal(t, tc.wantReplicas, replicas) var names []string for _, request := range removed { From 5d431b6b4fa9e8e5256997bf2410369687754630 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 20:54:22 +0000 Subject: [PATCH 153/196] refactor(operator): omit unused LPX backend fingerprint input Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph_lpx.go | 2 -- deploy/operator/internal/dynamo/graph_lpx_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 279558bf251e..4087b82ccbb7 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -128,7 +128,6 @@ type lpxInputRevisionPayload struct { PropagatedAnnotations map[string]string Environment []corev1.EnvVar `json:"Env"` PriorityClass string - BackendFramework string TopologyConstraint *v1beta1.SpecTopologyConstraint RestartToken string `json:"Restart"` ProviderOverride *v1beta1.ProviderOverride @@ -178,7 +177,6 @@ func LPXInputRevision(dgd *v1beta1.DynamoGraphDeployment, restart string) (strin PropagatedAnnotations: annotations, Environment: dgd.Spec.Env, PriorityClass: dgd.Spec.PriorityClassName, - BackendFramework: dgd.Spec.BackendFramework, TopologyConstraint: dgd.Spec.TopologyConstraint, RestartToken: restart, ProviderOverride: dgd.Spec.ProviderOverride, diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index f05ce009728f..8bf3b6bac6ad 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -304,7 +304,6 @@ func TestLPXInputRevision(t *testing.T) { {"shared/annotations", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Annotations = map[string]string{"workload": "new"} }}, {"shared/environment", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Env = []corev1.EnvVar{{Name: "SHARED", Value: "new"}} }}, {"shared/priority", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.PriorityClassName = "inference" }}, - {"shared/backend", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.BackendFramework = "vllm" }}, {"shared/topology", true, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.TopologyConstraint = &v1beta1.SpecTopologyConstraint{ClusterTopologyName: "cluster", PackDomain: "rack"} }}, @@ -315,6 +314,7 @@ func TestLPXInputRevision(t *testing.T) { d.Spec.Restart = &v1beta1.Restart{ID: "restart-selected"} d.Status.Restart = &v1beta1.RestartStatus{ObservedID: d.Spec.Restart.ID, Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{"decode"}} }}, + {"ignored/backend", false, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.BackendFramework = "vllm" }}, {"ignored/ordinary-replicas", false, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Components[0].Replicas = ptr.To(int32(9)) }}, {"ignored/ordinary-image", false, func(d *v1beta1.DynamoGraphDeployment) { d.Spec.Components[0].PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "prefill:next"}}}} From ba1f4e8433a5fa882db689639ec250d756280939 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 21:19:47 +0000 Subject: [PATCH 154/196] refactor(operator): reuse component labels for LPX readiness Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/controller_test.go | 6 +++--- deploy/operator/internal/controller/lpx/render_test.go | 1 - deploy/operator/internal/dynamo/graph_lpx.go | 3 --- deploy/operator/internal/dynamo/grove_lpx.go | 4 ++-- deploy/operator/internal/dynamo/lpx/render.go | 3 --- .../lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml | 4 ---- .../lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml | 2 -- .../dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 2 -- .../lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml | 2 -- .../lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v2-specdecode.yaml | 4 ---- .../lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml | 2 -- .../testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml | 4 ---- .../dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml | 2 -- 15 files changed, 5 insertions(+), 38 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 220db404b853..2b4d4b78eb39 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -92,7 +92,7 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(object), object)) if clique, ok := object.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker { want := int32(1) - if clique.Labels[lpx.StageLabel] == second.ComponentName { + if clique.Labels[consts.KubeLabelDynamoComponent] == second.ComponentName { want = 3 } require.Equal(t, want, clique.Spec.Replicas) @@ -160,7 +160,7 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { if _, ok := original.(*grovev1alpha1.PodCliqueSet); ok { continue } - if clique, ok := original.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker && clique.Labels[lpx.StageLabel] == "lpx" { + if clique, ok := original.(*grovev1alpha1.PodClique); ok && clique.Annotations[lpxv1alpha1.PodRoleAnnotation] == lpxv1alpha1.PodRoleCyborgWorker && clique.Labels[consts.KubeLabelDynamoComponent] == "lpx" { continue } actual := original.DeepCopyObject().(client.Object) @@ -2307,7 +2307,7 @@ func TestSpecDecodeStatusCountsCompleteDraftInstances(t *testing.T) { } } firstDraft := pclqs[selected.plan.Agents[0].CliqueName] - require.Equal(t, "draft", firstDraft.Labels[lpx.StageLabel]) + require.Equal(t, "draft", firstDraft.Labels[consts.KubeLabelDynamoComponent]) t.Log("Make only the first draft incomplete while retaining a fully ready second draft and target") switch scenario { diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index abdf534c4f0d..37b43652bb9c 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -99,7 +99,6 @@ func TestGenerateGrovePodCliqueSet_FromDGDYaml(t *testing.T) { component := dynamoDeployment.GetComponentByName(clique.Labels[consts.KubeLabelDynamoComponent]) require.NotNil(t, component) require.True(t, component.IsLPX()) - require.Equal(t, component.ComponentName, clique.Labels[lpx.StageLabel]) } t.Log("Render conventional components independently from their ordinary-only input") diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 4087b82ccbb7..5158e6d539ce 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -270,7 +270,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn if err != nil { return nil, fmt.Errorf("rendering %s.agent: rendering selected LPX base pod: %w", component.ComponentName, err) } - lpuTemplate.Labels[dynamolpx.StageLabel] = component.ComponentName input.Stages[component.ComponentName] = *lpuTemplate conductor := component.ComponentRole(v1beta1.ComponentRoleLPXConductor) if component != p.component { @@ -285,7 +284,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn if err != nil { return nil, fmt.Errorf("rendering %s.conductor: rendering selected LPX base pod: %w", component.ComponentName, err) } - input.Conductor.Labels[dynamolpx.StageLabel] = component.ComponentName continue } @@ -318,7 +316,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn } clique.Labels[commonconsts.KubeLabelDynamoComponentType] = string(v1beta1.ComponentTypeLPX) - clique.Labels[dynamolpx.StageLabel] = component.ComponentName input.Cyborg = clique } return input, nil diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 384e327dcf3d..7676e71c01fe 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -9,7 +9,7 @@ import ( "slices" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" grovecommon "github.com/ai-dynamo/grove/operator/api/common" groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" @@ -83,7 +83,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD continue } name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(replica)}, template.Name) - memberName := template.Labels[lpx.StageLabel] + memberName := template.Labels[commonconsts.KubeLabelDynamoComponent] // Revision checks precede the pure per-clique readiness calculation. pclq := pclqs[name] diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 2eb440e2d9f4..4de4bc65aa96 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -25,9 +25,6 @@ import ( // admission metadata and transport overhead. const MaxRenderedPodCliqueSetBytes = 1 << 20 -// StageLabel records the authored LPX component association. -const StageLabel = "lpx.nvidia.com/stage" - // RenderInput contains the fresh stage templates and runtime settings consumed by rendering. type RenderInput struct { // MinAvailable is the minimum number of complete workload replicas in the gang. diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml index 4c2309eb3148..b094af7b5d6e 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml @@ -77,7 +77,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec @@ -159,7 +158,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec @@ -245,7 +243,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid labels: - lpx.nvidia.com/stage: target nvidia.com/dynamo-component: target nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec @@ -329,7 +326,6 @@ spec: scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: target nvidia.com/dynamo-component: target nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml index f1eac2d09e7d..11d3b7929189 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml @@ -49,7 +49,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode @@ -131,7 +130,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index ea8af2528a1f..98e58139edda 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -188,7 +188,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm @@ -395,7 +394,6 @@ spec: scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index d7fcac387307..8f1048ee14a8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -52,7 +52,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid @@ -210,7 +209,6 @@ spec: scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index b172cc5a0610..80c820565585 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -49,7 +49,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only @@ -181,7 +180,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index f8db7573467c..f521b1ef5ecd 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -83,7 +83,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -221,7 +220,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -359,7 +357,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -495,7 +492,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index e2c264e505f0..c3bc3989ca06 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -41,7 +41,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxStrictHybrid labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid @@ -198,7 +197,6 @@ spec: scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index dfde2cd563e8..4c5f652a52a8 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -41,7 +41,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only @@ -187,7 +186,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index 447156f30198..af525da9d6d2 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -62,7 +62,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -214,7 +213,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/stage: draft nvidia.com/dynamo-component: draft nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -366,7 +364,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -516,7 +513,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index 17ee88d8b359..c03f196889bb 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -124,7 +124,6 @@ spec: scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly labels: - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 @@ -260,7 +259,6 @@ spec: scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" - lpx.nvidia.com/stage: lpu nvidia.com/dynamo-component: lpu nvidia.com/dynamo-component-type: lpx nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 From e5cf4b64de36d1b3acec2aa2e7b204dbd9f7cb50 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Thu, 24 Sep 2026 21:20:00 +0000 Subject: [PATCH 155/196] refactor(operator): validate prop-sync chain length at manifest boundary Signed-off-by: Andrew Paprotsky --- .../operator/internal/dynamo/lpx/model_manifest_v2_test.go | 6 ++++++ deploy/operator/internal/dynamo/lpx/selected_prop_sync.go | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go index e669f8e58117..c16a5d010e7d 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -51,6 +51,8 @@ func TestNormalizeBuildSnapshotRejectsInvalidHXArtifacts(t *testing.T) { {name: "duplicate LPU partition ID", wantErr: "repeats LPU partition ID 1"}, {name: "topology metadata node-count mismatch", wantErr: "deployment.numLpuNodes = 2, but partition extents require 1 LPU nodes"}, {name: "partial build", wantErr: "partSelect builds are not supported"}, + {name: "empty selected prop-sync chain", wantErr: "deployment.selectedPropSyncChains[0] must contain at least two partitionIds"}, + {name: "singleton selected prop-sync chain", wantErr: "deployment.selectedPropSyncChains[0] must contain at least two partitionIds"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -65,6 +67,10 @@ func TestNormalizeBuildSnapshotRejectsInvalidHXArtifacts(t *testing.T) { fixture.partitions[0].partitionShape = []uint32{16, 1, 1, 1} case "partial build": fixture.partSelect = true + case "empty selected prop-sync chain": + fixture.selectedPropSyncChains = [][]uint32{{}} + case "singleton selected prop-sync chain": + fixture.selectedPropSyncChains = [][]uint32{{1}} } snapshot := acquireTestSnapshot(t, writeCompilerFixture(t, fixture)) diff --git a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go index 590e6f5d8324..2d3649d95ec0 100644 --- a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go +++ b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go @@ -27,9 +27,6 @@ func validateSelectedPropSyncGraph( // Reject malformed references across every chain before evaluating relationships between valid members. for chainIndex, chain := range chains { - if len(chain) < 2 { - return nil, fmt.Errorf("%s %d must contain at least two partition IDs", subject, chainIndex) - } for _, partitionID := range chain { if _, present := partitionPositions[partitionID]; !present { return nil, fmt.Errorf("%s %d references missing partition ID %d", subject, chainIndex, partitionID) @@ -74,9 +71,6 @@ func (b *Build) consumeRuntimeSelectedPropSyncChain() error { return fmt.Errorf("LPU-only runtime requires exactly one selected prop-sync chain, got %d", len(b.SelectedPropSyncChains)) } chain := b.SelectedPropSyncChains[0] - if len(chain) < 2 { - return fmt.Errorf("LPU-only selected prop-sync chain %s must contain at least two partition ids", formatPropSyncChain(chain)) - } // Locate the selected chain in normalized physical partition order. firstSelected := sort.Search(len(b.Partitions), func(index int) bool { From 411e4ab79c9b20614aa2541e6e26f383adaa099c Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 05:33:17 +0000 Subject: [PATCH 156/196] refactor(operator): remove redundant LPX scheduler assignment Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/pod_roles.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles.go b/deploy/operator/internal/dynamo/lpx/pod_roles.go index 253c5ece64af..3fa6595c1dcb 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles.go @@ -8,7 +8,6 @@ package lpx import ( "strings" - "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" corev1 "k8s.io/api/core/v1" @@ -30,7 +29,6 @@ func configureAgentScheduling( ) { // Preserve Agent placement while replacing authored LPU resources with the selected device. stripLPUResources(agent) - agent.SchedulerName = v1alpha1.LPXSchedulerName container := common.FindContainerByName(agent.Containers, commonconsts.MainContainerName) // Model projection has already restricted the target family to XT or HX. name, amount := v2LPUResourceName, resource.MustParse("8") From bdc3051222b729614398b13c10e4d42a805caa13 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 06:08:00 +0000 Subject: [PATCH 157/196] refactor(operator): remove redundant LPX code Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph_lpx.go | 1 - deploy/operator/internal/dynamo/grove_lpx.go | 47 ++++++++----------- .../dynamo/lpx/model_manifest_common.go | 2 +- .../internal/dynamo/lpx/model_registry.go | 4 +- .../internal/dynamo/lpx/selected_prop_sync.go | 2 +- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 5158e6d539ce..52c19bd4278a 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -296,7 +296,6 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn role := lpxRoleComponent(component, template, p.dynamoDeployment, p.discoveryBackend) role.ComponentType = v1beta1.ComponentTypeDecode role.Replicas = ptr.To(replicas) - role.MinAvailable = nil defaults := &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} gpu := p gpu.component = role diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 7676e71c01fe..792db6d22905 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -38,15 +38,15 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD } } verifiedAvailable := int32(0) - result := func(ready bool, classification, message string) GroveReadiness { + result := func(ready bool, message string) GroveReadiness { if status.AvailableReplicas != nil { status.AvailableReplicas = ptr.To(min(*status.AvailableReplicas, verifiedAvailable)) } statuses[component.ComponentName] = status - return GroveReadiness{Ready: ready, Classification: classification, Message: message, ComponentStatuses: statuses} + return GroveReadiness{Ready: ready, Message: message, ComponentStatuses: statuses} } pending := func(message string) GroveReadiness { - return result(false, v1beta1.DGDReadyReasonSomeResourcesNotReady, message) + return result(false, message) } if pcs == nil { return pending("Waiting for the exact LPX PodCliqueSet") @@ -67,15 +67,10 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD status.ScheduledReplicas = ptr.To(pcsg.Status.ScheduledReplicas) replicas := ptr.Deref(component.Replicas, pcsg.Spec.Replicas) if pcsg.Spec.Replicas != replicas || pcsg.Status.CurrentPodCliqueSetGenerationHash == nil || *pcsg.Status.CurrentPodCliqueSetGenerationHash != *hash { - return result(false, v1beta1.DGDReadyReasonUpdating, "LPX scaling group has not applied the desired revision and capacity") + return pending("LPX scaling group has not applied the desired revision and capacity") } // Observe every member before returning so partial draft readiness remains visible. - unreadyClassification, unreadyMessage := "", "" - noteUnready := func(classification, message string) { - if unreadyMessage == "" { - unreadyClassification, unreadyMessage = classification, message - } - } + unreadyMessage := "" for replica := range replicas { replicaReady := true for _, template := range pcs.Spec.Template.Cliques { @@ -90,9 +85,9 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD readiness := groveComponentReadiness{} switch { case pclq == nil: - readiness = readiness.withResult(false, fmt.Sprintf("Waiting for LPX role %s", name), v1beta1.DGDReadyReasonSomeResourcesNotReady) + readiness.reason = fmt.Sprintf("Waiting for LPX role %s", name) case pclq.Status.CurrentPodCliqueSetGenerationHash == nil || *pclq.Status.CurrentPodCliqueSetGenerationHash != *hash: - readiness = readiness.withResult(false, fmt.Sprintf("LPX role %s has not applied the desired revision", name), v1beta1.DGDReadyReasonUpdating) + readiness.reason = fmt.Sprintf("LPX role %s has not applied the desired revision", name) default: readiness = observeLPXRole(ctx, pclq) } @@ -108,7 +103,9 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD statuses[memberName] = draft } if !readiness.ready { - noteUnready(readiness.classification, readiness.reason) + if unreadyMessage == "" { + unreadyMessage = readiness.reason + } replicaReady = false } } @@ -117,13 +114,9 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD } } if unreadyMessage != "" { - return result(false, unreadyClassification, unreadyMessage) - } - ready, message, classification := pcsgStatusReady(pcsg, replicas) - if ready { - classification = v1beta1.DGDReadyReasonAllResourcesReady + return pending(unreadyMessage) } - return result(ready, classification, message) + return result(pcsgStatusReady(pcsg, replicas)) } // observeLPXRole calculates readiness and complete-instance counts from a non-nil @@ -135,7 +128,7 @@ func observeLPXRole(ctx context.Context, pclq *grovev1alpha1.PodClique) groveCom // A model instance is counted only after the complete build width is observed. readiness := podCliqueReadiness(pclq, log.FromContext(ctx)) - role = role.withResult(readiness.ready, readiness.reason, readiness.classification) + role.ready, role.reason = readiness.ready, readiness.reason if pclq.Status.ObservedGeneration == nil || *pclq.Status.ObservedGeneration != pclq.Generation { return role } @@ -156,27 +149,27 @@ func observeLPXRole(ctx context.Context, pclq *grovev1alpha1.PodClique) groveCom return role } -func pcsgStatusReady(pcsg *grovev1alpha1.PodCliqueScalingGroup, desiredReplicas int32) (bool, string, string) { +func pcsgStatusReady(pcsg *grovev1alpha1.PodCliqueScalingGroup, desiredReplicas int32) (bool, string) { if pcsg.Status.Replicas == desiredReplicas && pcsg.Status.UpdatedReplicas == desiredReplicas && pcsg.Status.AvailableReplicas == desiredReplicas { - return true, "", "" + return true, "" } minAvailable := meta.FindStatusCondition(pcsg.Status.Conditions, groveconstants.ConditionTypeMinAvailableBreached) if minAvailable != nil && minAvailable.Status == metav1.ConditionFalse && (minAvailable.Reason == groveconstants.ConditionReasonScheduledReplicasBelowMinAvailable || minAvailable.Reason == legacyConditionReasonInsufficientScheduledPCSGReplicas) { - return false, fmt.Sprintf("min-available breached (%s): %s", minAvailable.Reason, minAvailable.Message), v1beta1.DGDReadyReasonInsufficientCapacity + return false, fmt.Sprintf("min-available breached (%s): %s", minAvailable.Reason, minAvailable.Message) } if scheduled := pcsg.Status.ScheduledReplicas; scheduled > 0 && scheduled < desiredReplicas { - return false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduled, desiredReplicas), v1beta1.DGDReadyReasonInsufficientCapacity + return false, fmt.Sprintf("insufficient scheduled replicas: scheduled=%d/%d", scheduled, desiredReplicas) } if pcsg.Status.UpdatedReplicas != desiredReplicas { - return false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, pcsg.Status.UpdatedReplicas), v1beta1.DGDReadyReasonUpdating + return false, fmt.Sprintf("desired=%d, updated=%d", desiredReplicas, pcsg.Status.UpdatedReplicas) } if pcsg.Status.Replicas != desiredReplicas { - return false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, pcsg.Status.Replicas), v1beta1.DGDReadyReasonUpdating + return false, fmt.Sprintf("performing rolling update: desired=%d, replicas=%d", desiredReplicas, pcsg.Status.Replicas) } - return false, fmt.Sprintf("scheduled but available=%d/%d", pcsg.Status.AvailableReplicas, desiredReplicas), v1beta1.DGDReadyReasonPodsNotReady + return false, fmt.Sprintf("scheduled but available=%d/%d", pcsg.Status.AvailableReplicas, desiredReplicas) } diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go index f57432ddc05f..f7268e866a37 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_common.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_common.go @@ -215,7 +215,7 @@ func validateManifestPartitionNodeCount( // partSelect artifacts contain only the selected partitions, while numLpuNodes // describes the complete deployment geometry. - if partialSelection && (want >= packagedNodes || want >= hostEmbeddingNodes) { + if partialSelection && want >= hostEmbeddingNodes { return nil } diff --git a/deploy/operator/internal/dynamo/lpx/model_registry.go b/deploy/operator/internal/dynamo/lpx/model_registry.go index bb36311726da..9bf50c2b2692 100644 --- a/deploy/operator/internal/dynamo/lpx/model_registry.go +++ b/deploy/operator/internal/dynamo/lpx/model_registry.go @@ -144,14 +144,14 @@ func (r *defaultModelRegistry) BuildURL(id string) (*url.URL, error) { return nil, fmt.Errorf("build ID %q must remain within the configured model registry", id) } relativeID = filepath.Clean(relativeID) - if relativeID == "" || relativeID == "." { + if relativeID == "." { return nil, fmt.Errorf("empty ref") } if r.registryURL.Scheme == BuildSchemeFile { return &url.URL{ Scheme: BuildSchemeFile, - Path: filepath.Clean(filepath.Join(r.registryURL.Path, relativeID)), + Path: filepath.Join(r.registryURL.Path, relativeID), }, nil } diff --git a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go index 2d3649d95ec0..e7f3545f6d39 100644 --- a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go +++ b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go @@ -64,7 +64,7 @@ func validateSelectedPropSyncGraph( } func (b *Build) consumeRuntimeSelectedPropSyncChain() error { - if b == nil || len(b.SelectedPropSyncChains) == 0 { + if len(b.SelectedPropSyncChains) == 0 { return nil } if len(b.SelectedPropSyncChains) != 1 { From 248dd27e05857dcca6a32c25e83cc8545e9115c7 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 08:58:40 +0000 Subject: [PATCH 158/196] refactor(operator): avoid redundant LPX copies and writes Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/pipeline_request.go | 5 +++-- deploy/operator/internal/dynamo/graph_lpx.go | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index 5b68c2f9d80b..8c421e0fd7be 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -39,8 +39,9 @@ func (r *graphReconciler) getPipelineRequests(ctx context.Context, pcs *grovev1a return nil, err } requests := make(map[string]*lpxv1alpha1.LPUPipelineRequest) - for _, request := range requestList.Items { - requests[request.Name] = &request + for i := range requestList.Items { + request := &requestList.Items[i] + requests[request.Name] = request } return requests, nil } diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 52c19bd4278a..851edb35dfe7 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -295,12 +295,10 @@ func renderLPXComponents(p cliqueParams, workload *dynamolpx.Workload, plan *dyn template, replicas := conductor.PodTemplate, ptr.Deref(conductor.Replicas, minimumReplicas) role := lpxRoleComponent(component, template, p.dynamoDeployment, p.discoveryBackend) role.ComponentType = v1beta1.ComponentTypeDecode - role.Replicas = ptr.To(replicas) defaults := &podTemplateRuntimeDefaults{ComponentDefaults: NewWorkerDefaults()} gpu := p gpu.component = role - gpu.componentName = component.ComponentName - gpu.r = ServiceRole{Name: plan.CyborgTemplate, Role: RoleMain, Replicas: *role.Replicas} + gpu.r = ServiceRole{Name: plan.CyborgTemplate, Role: RoleMain, Replicas: replicas} gpuTemplate, err := renderSelectedLPXRole(role, p.dynamoDeployment, alphaComponent, p.operatorConfig, p.secretsRetriever, p.discoveryContext, defaults) if err != nil { @@ -330,7 +328,6 @@ func lpxRoleComponent(source *v1beta1.DynamoComponentDeploymentSharedSpec, templ // Bind source metadata on the independently owned role template. propagateDGDAnnotations(dgd.Annotations, role) - role.PodTemplate.Labels[commonconsts.KubeLabelDynamoNamespace] = GetDynamoNamespace(dgd, source) if backend != "" { role.PodTemplate.Annotations[commonconsts.KubeAnnotationDynamoDiscoveryBackend] = string(backend) } From 015a6ebfc35f812fe0ffac3afac39a47ae9b5e4c Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 10:02:32 +0000 Subject: [PATCH 159/196] refactor(operator): avoid copying converted ports twice Signed-off-by: Andrew Paprotsky --- deploy/operator/api/v1alpha1/shared_spec_conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operator/api/v1alpha1/shared_spec_conversion.go b/deploy/operator/api/v1alpha1/shared_spec_conversion.go index 184c44bf4fd3..647a8c4dbd33 100644 --- a/deploy/operator/api/v1alpha1/shared_spec_conversion.go +++ b/deploy/operator/api/v1alpha1/shared_spec_conversion.go @@ -1502,7 +1502,7 @@ func mergeExtraPodSpecMainContainer(src *DynamoComponentDeploymentSharedSpec, ma // than allowing mergo to replace the VolumeMounts slice. mainBase.VolumeMounts = append(slices.Clone(main.VolumeMounts), dedicatedVolumeMounts...) if main.Ports != nil { - mainBase.Ports = slices.Clone(main.Ports) + mainBase.Ports = main.Ports } // StartupProbe has no dedicated v1alpha1 field; take it verbatim. if main.StartupProbe != nil { From 7c3664723df2f71e3841dbfd33405b906ebcc221 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 10:44:15 +0000 Subject: [PATCH 160/196] refactor(operator): reuse the owned LPX security context Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph_lpx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operator/internal/dynamo/graph_lpx.go b/deploy/operator/internal/dynamo/graph_lpx.go index 851edb35dfe7..66bf13b62088 100644 --- a/deploy/operator/internal/dynamo/graph_lpx.go +++ b/deploy/operator/internal/dynamo/graph_lpx.go @@ -398,7 +398,7 @@ func renderSelectedLPXRole( } // The shared renderer's filesystem-group fallback is not part of the LPX template contract. - basePodSpec.SecurityContext = component.PodTemplate.Spec.SecurityContext.DeepCopy() + basePodSpec.SecurityContext = component.PodTemplate.Spec.SecurityContext // LPX supplies the generated config volume after merging; discard only inferred PVCs. explicitVolumes := component.PodTemplate.Spec.Volumes From 74c1314a9ed836e69457ba720c32be688da88103 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 10:44:16 +0000 Subject: [PATCH 161/196] refactor(operator): ignore unused LPX template hash updates Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/watch.go | 2 -- deploy/operator/internal/controller/lpx/watch_test.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index d0f1aa9344f4..555a9e2c1cee 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -18,7 +18,6 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -172,7 +171,6 @@ func podCliquePredicate() predicate.Funcs { return false } return commoncontroller.PodCliqueStatusChangeIsSignificant(oldClique, newClique) || - !ptr.Equal(oldClique.Status.CurrentPodTemplateHash, newClique.Status.CurrentPodTemplateHash) || oldClique.Generation != newClique.Generation || !maps.Equal(oldClique.Annotations, newClique.Annotations) || !maps.Equal(oldClique.Labels, newClique.Labels) || diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 210747adaf09..0f4699bb5397 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -160,7 +160,7 @@ func TestGroveEventPredicates(t *testing.T) { } func TestPodCliquePredicateRoles(t *testing.T) { - t.Log("Observe compiled LPX roles and template-hash convergence") + t.Log("Observe compiled LPX roles and ignore unused template-hash updates") filter := podCliquePredicate() for _, test := range []struct { role string @@ -179,7 +179,7 @@ func TestPodCliquePredicateRoles(t *testing.T) { require.Equal(t, test.want, filter.Create(event.CreateEvent{Object: clique})) current := clique.DeepCopy() current.Status.CurrentPodTemplateHash = ptr.To("current-template") - require.Equal(t, test.want, filter.Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: current})) + require.False(t, filter.Update(event.UpdateEvent{ObjectOld: clique, ObjectNew: current})) }) } } From 3d91e13b260c74b27a5c6023b8e91a4f724e59dc Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 11:09:59 +0000 Subject: [PATCH 162/196] refactor(operator): remove redundant PCS annotation allocation Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/graph.go | 3 --- deploy/operator/internal/dynamo/graph_test.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 6dd2d61ae860..baa983143735 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -2706,9 +2706,6 @@ func newGrovePodCliqueSet( } gangSet.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name gangSet.Annotations = maps.Clone(dynamoDeployment.Spec.Annotations) - if gangSet.Annotations == nil { - gangSet.Annotations = make(map[string]string) - } // Volcano queue selection is consumed by Grove from the PodCliqueSet annotation. // KAI-Scheduler is injected later on each clique via schedulerName and queue label. injectVolcanoQueueAnnotation(gangSet, dynamoDeployment.Annotations, runtimeConfig) diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index 15caa59d02f6..e2683323b334 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -1995,7 +1995,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, - Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, @@ -2608,7 +2607,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, - Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, @@ -3632,7 +3630,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) { Labels: map[string]string{ commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment", }, - Annotations: make(map[string]string), }, Spec: grovev1alpha1.PodCliqueSetSpec{ Replicas: 1, From 8d59a63b9650b37fdcab36045663071e5097e044 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 11:48:09 +0000 Subject: [PATCH 163/196] refactor(operator): reuse LPX conductor labels and count Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/render.go | 2 +- .../internal/webhook/validation/dynamographdeployment.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 4de4bc65aa96..8589c9c3e651 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -128,7 +128,7 @@ func RenderNodeLocal( annotations[v1alpha1.AnnotationExtraResourcesHash] = configHash conductor = &grovev1alpha1.PodCliqueTemplateSpec{ Name: conductorTemplateName, - Labels: maps.Clone(conductorTemplate.Labels), + Labels: conductorTemplate.Labels, Annotations: annotations, Spec: grovev1alpha1.PodCliqueSpec{ RoleName: conductorTemplateName, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 1d180ace3578..c8b19ae801c4 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -432,7 +432,6 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( return conductorComponentsPath.Key(spec.Components[index].ComponentName) } } - conductorCount := 0 for index := range spec.Components { component := &spec.Components[index] if !component.IsLPX() { @@ -442,13 +441,12 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( if role.Name != nvidiacomv1beta1.ComponentRoleLPXConductor { continue } - conductorCount++ if role.PodTemplate == nil { allErrs = append(allErrs, field.Required(componentPath(index).Child("roles").Index(roleIndex).Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) } } } - if hasLPXComponent && conductorCount != lpxComponentCount && !(lpxComponentCount == 2 && conductorCount == 1) { + if hasLPXComponent && independentEngineCount != lpxComponentCount && !(lpxComponentCount == 2 && independentEngineCount == 1) { allErrs = append(allErrs, field.Forbidden(conductorComponentsPath, "LPX components must each declare a conductor role or form a shared draft and target pair")) } From d9b3c5bd132eca89aa91660e16c7fac8044d4508 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 15:35:29 +0000 Subject: [PATCH 164/196] refactor(operator): remove redundant LPX checks Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/pipeline_request_deadline.go | 2 +- deploy/operator/internal/dynamo/lpx/pod_roles_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 40bc801bc905..87d6db9ff51c 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -54,7 +54,7 @@ func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineReques // pipelineRequestSchedulingStartedAt returns the scheduler's nonzero cycle start, // or creation time while waiting for the scheduler. request must be non-nil. func pipelineRequestSchedulingStartedAt(request *lpxv1alpha1.LPUPipelineRequest) time.Time { - if request.Status == nil || request.Status.SchedulingStartedAt == nil || request.Status.SchedulingStartedAt.IsZero() { + if request.Status == nil || request.Status.SchedulingStartedAt.IsZero() { return request.CreationTimestamp.Time } return request.Status.SchedulingStartedAt.Time diff --git a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go index b2be3910870a..c9568b0bbb2b 100644 --- a/deploy/operator/internal/dynamo/lpx/pod_roles_test.go +++ b/deploy/operator/internal/dynamo/lpx/pod_roles_test.go @@ -79,8 +79,7 @@ func TestSelectedRolePodSpecsMaterializeFamilyResourceOnlyOnAgentMainContainer(t stripLPUResources(conductor) configureAgentScheduling(agent, test.family) - t.Log("Verify the input is immutable and affinity follows its owning role") - require.Equal(t, lpuResources, base.Containers[0].Resources.Limits, "input PodSpec must remain unchanged") + t.Log("Verify Agent affinity is preserved") require.Equal(t, base.Affinity, agent.Affinity) t.Log("Remove generic and wrong-family LPU resources from both roles") From 0a7877623dab306bed4d7904fe2bde53d480275e Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 16:00:50 +0000 Subject: [PATCH 165/196] refactor(operator): remove unused LPX dependencies and copies Signed-off-by: Andrew Paprotsky --- deploy/operator/Makefile | 2 +- deploy/operator/internal/controller/dgd_grove_lpx_status.go | 3 ++- deploy/operator/internal/dynamo/lpx/render.go | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index a79a9c1b66f3..3b0004ac122c 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -309,7 +309,7 @@ test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$$($(ENVTEST_ASSETS_CMD))" go test $$(go list ./... | grep -vE '/cmd($$|/)') ./cmd/crd-apply -coverprofile cover.out .PHONY: envtest -envtest: setup-envtest yq ## Run envtest-backed operator tests. +envtest: setup-envtest ## Run envtest-backed operator tests. KUBEBUILDER_ASSETS="$$($(ENVTEST_ASSETS_CMD))" go test $(ENVTEST_PACKAGES) .PHONY: setup-clustertest diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index b973af15d9a7..a343cd3f2706 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -13,6 +13,7 @@ import ( // mergeLPXChildStatus composes a current LGD observation into the ordinary // Grove result. The outer DGD controller remains the only status writer. +// The result reuses the handoff-owned child's component status. func mergeLPXChildStatus( source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, @@ -38,7 +39,7 @@ func mergeLPXChildStatus( if observed { for _, component := range components { if observedStatus, found := child.Status.Components[component.ComponentName]; found { - ordinary.ComponentStatus[component.ComponentName] = *observedStatus.ComponentReplicaStatus.DeepCopy() + ordinary.ComponentStatus[component.ComponentName] = observedStatus.ComponentReplicaStatus } } } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 8589c9c3e651..32435d13766d 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -175,7 +175,7 @@ func RenderNodeLocal( podSpec = *podSpec.DeepCopy() } - annotations := roleAnnotations(template.Annotations, lpxv1alpha1.PodRoleAgent, projection.Digest().String()) + annotations := roleAnnotations(maps.Clone(template.Annotations), lpxv1alpha1.PodRoleAgent, projection.Digest().String()) annotations[v1alpha1.AnnotationExtraResourcesHash] = configHash annotations[lpxv1alpha1.PodModelAnnotation] = projection.model annotations[lpxv1alpha1.CompilerSnapshotDigestAnnotation] = projection.CompilerSnapshotDigest() @@ -270,12 +270,13 @@ func configureLPURolePods(agentPodSpec, conductorPodSpec *corev1.PodSpec, worklo return nil } +// roleAnnotations consumes base, allocating it when nil. func roleAnnotations( base map[string]string, role string, workloadDigest string, ) map[string]string { - annotations := workloadAnnotations(maps.Clone(base), workloadDigest) + annotations := workloadAnnotations(base, workloadDigest) // Remove controller-owned role metadata before stamping canonical values. for _, key := range []string{ WorkloadModeAnnotation, From 2c1286c5d822a3df1d12952575fe1bf6c23263d3 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 16:24:35 +0000 Subject: [PATCH 166/196] refactor(operator): reuse the validated LPX restart token Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_grove_lpx_restart.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index b10a62f9ce37..b42ae1e38801 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -103,8 +103,7 @@ func (r *lpxRestartProgressResolver) observeRestart( } // Restart progress is resolved before handoff; an old Ready child cannot cover a newer DGD input. - restart := dynamo.LPXRestartToken(source, child.Annotations[dynamo.LPXRestartAnnotation]) - revision, err := dynamo.LPXInputRevision(source, restart) + revision, err := dynamo.LPXInputRevision(source, source.Spec.Restart.ID) if err != nil || child.Spec.InputRevision != revision { return nil } From 16214006a84aaedc597f3d3f8e60c76417c0c91a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 16:50:58 +0000 Subject: [PATCH 167/196] refactor(operator): simplify Cyborg startup dependencies Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/cyborg_render.go | 3 ++- deploy/operator/internal/dynamo/lpx/render.go | 9 --------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 4622928f2b2a..8b5f33a2bb82 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -7,6 +7,7 @@ package lpx import ( "fmt" + "slices" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" @@ -54,6 +55,6 @@ func configureHybridCyborg( } cyborg.Annotations[v1alpha1.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) } - cyborg.Spec.StartsAfter = appendUnique(cyborg.Spec.StartsAfter, agentTemplateNames...) + cyborg.Spec.StartsAfter = slices.Clone(agentTemplateNames) return nil } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 32435d13766d..b20e236dd3e9 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -298,12 +298,3 @@ func workloadAnnotations(base map[string]string, digest string) map[string]strin base[WorkloadDigestAnnotation] = digest return base } - -func appendUnique(existing []string, values ...string) []string { - for _, value := range values { - if !slices.Contains(existing, value) { - existing = append(existing, value) - } - } - return existing -} From 16c1eb10afee02602e2297d46f17ca37870b7742 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 16:50:59 +0000 Subject: [PATCH 168/196] fix(operator): skip download bookkeeping for local LPX builds Signed-off-by: Andrew Paprotsky --- .../dgd_lpx_handoff_envtest_test.go | 2 +- .../internal/controller/lpx/controller.go | 5 ++- .../controller/lpx/controller_test.go | 4 ++- .../internal/controller/lpx/modeldownload.go | 5 +++ .../controller/lpx/modeldownload_test.go | 32 +++++++++++++++++++ 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 0ba7060d052f..a2a32c9f3828 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -233,7 +233,7 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { assert.Equal(c, metav1.ConditionFalse, ready.Status) assert.Equal(c, failed.Reason, ready.Reason) assert.Equal(c, failed.Message, ready.Message) - assert.NotNil(c, child.Status.ModelDownload) + assert.Nil(c, child.Status.ModelDownload) }, 20*time.Second, 50*time.Millisecond) t.Log("The rejected publication created neither Grove workloads nor scheduler requests") diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 325eacaa22fe..0f93c9a91fad 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -464,12 +464,11 @@ func (r *graphReconciler) reconcileReadiness( setReadyCondition(deployment, v1beta1.DGDStateSuccessful, readiness.Message) - delay := modelDownloadRefreshInterval if download := deployment.Status.ModelDownload; download != nil && download.LastCheckedAt != nil { - delay = max(modelDownloadRequeueAfter, time.Until(download.LastCheckedAt.Add(modelDownloadRefreshInterval))) + return ctrl.Result{RequeueAfter: max(modelDownloadRequeueAfter, time.Until(download.LastCheckedAt.Add(modelDownloadRefreshInterval)))} } - return ctrl.Result{RequeueAfter: delay} + return ctrl.Result{} } // deleteUnusedConfigMaps runs only after readiness so existing pods keep their diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 2b4d4b78eb39..6d2e1c6ab13d 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -102,10 +102,12 @@ func TestIndependentLPXRoleScalingAndReadiness(t *testing.T) { require.NoError(t, r.Update(t.Context(), clique)) } } - _, err = r.Reconcile(t.Context(), key) + result, err := r.Reconcile(t.Context(), key) require.NoError(t, err) + require.Zero(t, result, "ready local builds need no download refresh") require.NoError(t, r.Get(t.Context(), key.NamespacedName, child)) require.True(t, meta.IsStatusConditionTrue(child.Status.Conditions, v1alpha1.LPXReadyCondition)) + require.Nil(t, child.Status.ModelDownload) for _, replicas := range []*int32{ptr.To(int32(3)), ptr.To(int32(1)), nil} { t.Log("Change only the first workload's Cyborg count, then wait for its workers") diff --git a/deploy/operator/internal/controller/lpx/modeldownload.go b/deploy/operator/internal/controller/lpx/modeldownload.go index ded524968570..a6dba94e3e95 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload.go +++ b/deploy/operator/internal/controller/lpx/modeldownload.go @@ -96,6 +96,11 @@ func (r *graphReconciler) reconcileModelDownloads( return ctrl.Result{RequeueAfter: modelDownloadRequeueAfter}, nil } + if len(downloaded) == 0 { + deployment.Status.ModelDownload = nil + return ctrl.Result{}, nil + } + if len(existingBuilds) == 0 { lastCheckedAt = new(metav1.Now()) } diff --git a/deploy/operator/internal/controller/lpx/modeldownload_test.go b/deploy/operator/internal/controller/lpx/modeldownload_test.go index bb239d04d23d..48516113e286 100644 --- a/deploy/operator/internal/controller/lpx/modeldownload_test.go +++ b/deploy/operator/internal/controller/lpx/modeldownload_test.go @@ -133,6 +133,38 @@ func TestEnsureModelsDownloadedSharesDeadlineAcrossBuilds(t *testing.T) { } } +func TestLocalLPXModelDownloads(t *testing.T) { + t.Log("Local builds have no remote-download status or calls") + dgd := newModelDownloadDGD(modelDownloadTestLocalBuildID) + child := &v1alpha1.LPXGraphDeployment{} + registry := newModelDownloadRegistry(t, map[string]bool{modelDownloadTestBuildID: true}, nil) + r := &graphReconciler{modelRegistry: registry} + for range 2 { + result, err := r.reconcileModelDownloads(t.Context(), child, dgd) + require.NoError(t, err) + require.Zero(t, result) + require.Nil(t, child.Status.ModelDownload) + } + require.Empty(t, registry.calls) + + t.Log("Selecting a remote build still checks its download") + dgd.Spec.Components[0].LPX.BuildID = modelDownloadTestBuildID + result, err := r.reconcileModelDownloads(t.Context(), child, dgd) + require.NoError(t, err) + require.Zero(t, result) + require.Equal(t, []string{modelDownloadTestBuildID}, registry.calls) + require.Equal(t, registry.calls, child.Status.ModelDownload.Builds) + require.NotNil(t, child.Status.ModelDownload.LastCheckedAt) + + t.Log("Returning to a local build clears the remote cache without another download check") + dgd.Spec.Components[0].LPX.BuildID = modelDownloadTestLocalBuildID + result, err = r.reconcileModelDownloads(t.Context(), child, dgd) + require.NoError(t, err) + require.Zero(t, result) + require.Nil(t, child.Status.ModelDownload) + require.Equal(t, []string{modelDownloadTestBuildID}, registry.calls) +} + func TestRunningLPXModelDownloadRefresh(t *testing.T) { tests := []struct { name string From ece8141894f1789925046447927344604f01a0ff Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 17:11:51 +0000 Subject: [PATCH 169/196] refactor(operator): preserve existing LPX scheduling failures Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/status.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/status.go b/deploy/operator/internal/controller/lpx/status.go index 17d40cf1829b..452f55b5b822 100644 --- a/deploy/operator/internal/controller/lpx/status.go +++ b/deploy/operator/internal/controller/lpx/status.go @@ -194,23 +194,18 @@ func schedulingFailureCoversPipelineRequests( // renew starts a new durable fence; otherwise the original failure time and // generation survive retries. Reconcile persists this before a later pass deletes LPRs. func setSchedulingFailedCondition(deployment *v1alpha1.LPXGraphDeployment, renew bool) { - observedGeneration := deployment.Generation - prior := meta.FindStatusCondition(deployment.Status.Conditions, schedulingFailedCondition) - if prior != nil && prior.Status == metav1.ConditionTrue && !renew { - observedGeneration = prior.ObservedGeneration - } - - // Even True-to-True renewal needs a fresh timestamp to cover the newly expired cycle. - if renew { - meta.RemoveStatusCondition(&deployment.Status.Conditions, schedulingFailedCondition) - } - // Aggregate failure and retry authorization have independent condition lifetimes. const message = "An LPX request exceeded its scheduling deadline" setReadyCondition(deployment, v1beta1.DGDStateFailed, message) + if !renew { + return + } + + // Even True-to-True renewal needs a fresh timestamp to cover the newly expired cycle. + meta.RemoveStatusCondition(&deployment.Status.Conditions, schedulingFailedCondition) meta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ Type: schedulingFailedCondition, Status: metav1.ConditionTrue, - ObservedGeneration: observedGeneration, Reason: pipelineRequestDeadlineExceededReason, Message: message, + ObservedGeneration: deployment.Generation, Reason: pipelineRequestDeadlineExceededReason, Message: message, }) } From e33cacd95bcec42a827ce3d7b199735095984b67 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 17:11:51 +0000 Subject: [PATCH 170/196] refactor(operator): simplify HX node-count compatibility Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/lpx/model_manifest_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go index fed64257252f..af1b497c7fb1 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2.go @@ -214,7 +214,7 @@ func addLPUArtifactsFromManifestV2( // Nonempty paths mark LPU artifacts; only metadata-less HX partitions permit the historical doubled node count. if partition.PartPath != "" { partitions = append(partitions, partition) - hxDoubleNodeCount = hxDoubleNodeCount && (len(partition.HXExtent) == 0 || compatible) + hxDoubleNodeCount = hxDoubleNodeCount && compatible } } // An LPU-only build cannot silently discard packaged CUDA or CPU partitions. From f3f4a5322848c4d0703732191b3f9442421fb1fe Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 18:01:24 +0000 Subject: [PATCH 171/196] refactor(operator): retain HX runtime partitions Signed-off-by: Andrew Paprotsky --- .../operator/internal/dynamo/lpx/lpu_configmap.go | 13 +++---------- .../internal/dynamo/lpx/workload_projection_v3.go | 3 +-- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index f1e015f7a733..6b3266155cf0 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -108,19 +108,12 @@ func resolvedPartitionData(projections []*ModelProjection) map[string]string { // Accumulate each projection's runtime partitions into the surviving columns. for _, projection := range projections { - // Render configured V2 runtime metadata without reapplying scheduler-shape validation to collapsed partitions. - partitions := projection.partitions - v2Runtime := projection.configuredBuild.Family == BuildFamilyXT && - len(projection.configuredBuild.Partitions) != 0 - if v2Runtime { - partitions = projection.configuredBuild.Partitions - } - + // Render runtime partitions, including XT's collapsed prop-sync chains. offset := int64(0) - for index, partition := range partitions { + for index, partition := range projection.configuredBuild.Partitions { var nodes string var endpointCount int64 - if v2Runtime { + if projection.configuredBuild.Family == BuildFamilyXT { nodeCount := partition.effectiveNodeCount() nodes, endpointCount = strconv.Itoa(nodeCount), int64(nodeCount) } else { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 49d1384ce9fa..36e2e0fd7360 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -27,8 +27,7 @@ func appendV3ModelProjections(dst []*ModelProjection, intent ModelProjectionInpu ioFPGACount, ioFanoutFactor := runtimeBuild.IOFPGACount, runtimeBuild.IOFanoutFactor - // Keep physical artifacts in the scheduler projection; the runtime uses resolved partition metadata. - runtimeBuild.Partitions = nil + // Selected chains are represented by the allocation metadata and connectors below. runtimeBuild.SelectedPropSyncChains = nil allocationMetadata, connectors, err := projectV3PropSync(manifestPartitions, selectedPropSyncChains, intent.Pipeline) From a33c27069a01f52af212adbabc6de8315ac64c96 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 19:21:29 +0000 Subject: [PATCH 172/196] refactor(operator): skip unused LPX reconciliation work Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_workload_renderer.go | 5 ----- .../dgd_grove_workloads_reconciler.go | 18 +++++++++--------- .../controller/dgd_lpx_handoff_test.go | 11 ++++------- .../controller/lpx/pipeline_request.go | 8 ++++---- deploy/operator/internal/dynamo/lpx/render.go | 5 +---- 5 files changed, 18 insertions(+), 29 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index 1c77dfe17ad5..a62fa65ddb8f 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -84,11 +84,6 @@ func (r *groveWorkloadRenderer) Render( return nil, fmt.Errorf("cannot render Grove PodCliqueSet without a Kubernetes reader") } - // An LPX-only graph has no ordinary PodCliqueSet to observe or render. - if len(ordinary.Spec.Components) == 0 { - return &grovePodCliqueSetRender{renderDeployment: ordinary}, nil - } - existingPodCliqueSet := &grovev1alpha1.PodCliqueSet{} key := types.NamespacedName{ Name: dynamo.PCSNameForDGD(ordinary.Name, ordinary.Spec.Components), diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index aed895aeab4b..171cd6a682ea 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -76,6 +76,15 @@ func (r *groveWorkloadsReconciler) Reconcile( restartState *dynamo.RestartState, checkpointInfos map[string]*checkpoint.CheckpointInfo, ) (ReconcileResult, error) { + // An LPX-only graph still reconciles its stable resources. + if len(ordinary.Spec.Components) == 0 { + stableResources, err := r.stableResources.Reconcile(ctx, source, ordinary) + if err != nil { + return ReconcileResult{}, err + } + return checkResourcesReadiness(stableResources), nil + } + logger := log.FromContext(ctx) workerHashTransition, err := r.rollout.planUnsupportedWorkerHashTransition(source) @@ -95,15 +104,6 @@ func (r *groveWorkloadsReconciler) Reconcile( return ReconcileResult{}, fmt.Errorf("failed to generate the Grove GangSet: %w", err) } - // An LPX-only graph still reconciles its stable resources. - if renderedPodCliqueSet.desired == nil { - stableResources, err := r.stableResources.Reconcile(ctx, source, renderedPodCliqueSet.renderDeployment) - if err != nil { - return ReconcileResult{}, err - } - return checkResourcesReadiness(stableResources), nil - } - // Converge the ordinary PCS before rollout or readiness observation. syncedPodCliqueSet, pcsWasWritten, err := r.reconcilePodCliqueSet(ctx, source, renderedPodCliqueSet) if err != nil { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index c7733963e346..66fc69199b62 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -122,13 +122,10 @@ func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing kube := builder.Build() t.Log("Render and observe restarts against that same ordinary PCS") - renderer := newGroveWorkloadRenderer(kube, &configv1alpha1.OperatorConfiguration{}, &commoncontroller.RuntimeConfig{}, nil) - rendered, err := renderer.Render(t.Context(), source, ordinary, nil, nil, false) - require.NoError(t, err) - if test.lpxOnly { - require.Nil(t, rendered.existing) - require.Nil(t, rendered.desired) - } else { + if !test.lpxOnly { + renderer := newGroveWorkloadRenderer(kube, &configv1alpha1.OperatorConfiguration{}, &commoncontroller.RuntimeConfig{}, nil) + rendered, err := renderer.Render(t.Context(), source, ordinary, nil, nil, false) + require.NoError(t, err) require.NotNil(t, rendered.existing) require.Equal(t, pcsName, rendered.existing.Name) require.Equal(t, pcsName, rendered.desired.Name) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request.go b/deploy/operator/internal/controller/lpx/pipeline_request.go index 8c421e0fd7be..88ee3ce7dc3f 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request.go @@ -81,10 +81,6 @@ func resolvePipelineRequests( request := &lpxv1alpha1.LPUPipelineRequest{ ObjectMeta: metav1.ObjectMeta{ Name: pipelineRequestName(deployment.Name, digest), Namespace: deployment.Namespace, - Labels: map[string]string{ - consts.KubeLabelDynamoGraphDeploymentName: dgdOwner.Name, - deploymentUIDLabel: string(deployment.UID), - }, Annotations: map[string]string{ lpx.DeploymentNameAnnotation: deployment.Name, pipelineRequestModelAnnotation: projection.Model(), @@ -102,6 +98,10 @@ func resolvePipelineRequests( requests[request.Name] = currentRequest } else { + request.Labels = map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgdOwner.Name, + deploymentUIDLabel: string(deployment.UID), + } requests[request.Name] = request missing = append(missing, request) } diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index b20e236dd3e9..2f3eab390cc0 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -73,10 +73,7 @@ func RenderNodeLocal( agentTemplateNames = append(agentTemplateNames, agent.TemplateName) } conductorTemplateName := plan.ConductorTemplate - allocation := strings.Join(agentTemplateNames, ",") - if projections[0].pipeline == PipelineSpecDecode { - allocation = strings.Join(agentTemplateNames, ":") - } + allocation := strings.Join(agentTemplateNames, ":") // The serving component owns conductor metadata and storage independently of model order. conductorStage := workload.ServingComponentName() From 279be262f434436cb234c51d07e62ad77346680f Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Fri, 25 Sep 2026 21:17:43 +0000 Subject: [PATCH 173/196] refactor(operator): simplify LPX status projection Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_status.go | 18 ++++++++---------- .../controller/dgd_lpx_handoff_test.go | 7 +++++++ .../dynamo/lpx/workload_projection_v3_test.go | 3 +-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index a343cd3f2706..2343ec71d608 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -26,22 +26,20 @@ func mergeLPXChildStatus( if ordinary.ComponentStatus == nil { ordinary.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) } - for _, component := range components { - kind := v1beta1.ComponentKindPodCliqueScalingGroup - if component.ComponentRole(v1beta1.ComponentRoleLPXConductor) == nil { - kind = v1beta1.ComponentKindPodClique - } - ordinary.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ComponentKind: kind} - } - current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() observed := current && child.Status.ObservedGeneration == child.Generation - if observed { - for _, component := range components { + for _, component := range components { + if observed { if observedStatus, found := child.Status.Components[component.ComponentName]; found { ordinary.ComponentStatus[component.ComponentName] = observedStatus.ComponentReplicaStatus + continue } } + kind := v1beta1.ComponentKindPodCliqueScalingGroup + if component.ComponentRole(v1beta1.ComponentRoleLPXConductor) == nil { + kind = v1beta1.ComponentKindPodClique + } + ordinary.ComponentStatus[component.ComponentName] = v1beta1.ComponentReplicaStatus{ComponentKind: kind} } // A current failure is actionable even if reconciliation did not finish. diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 66fc69199b62..8d44adcd0d59 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -515,6 +515,13 @@ func TestLPXHandoffOrdinaryScalingPreservesReadiness(t *testing.T) { beforeChild := child.DeepCopy() handoff := &dgdLPXHandoff{client: kube} + t.Log("Retain observed conductor status when the child's draft status is missing") + partial := child.DeepCopy() + delete(partial.Status.Components, "draft") + partialResult := mergeLPXChildStatus(source, partial, ReconcileResult{State: v1beta1.DGDStateSuccessful}) + require.Equal(t, v1beta1.ComponentReplicaStatus{ComponentKind: v1beta1.ComponentKindPodClique}, partialResult.ComponentStatus["draft"]) + require.Equal(t, child.Status.Components["lpx"].ComponentReplicaStatus, partialResult.ComponentStatus["lpx"]) + t.Log("Reorder the pair and make five frontend-only edits without changing the child") source.Spec.Components[0], source.Spec.Components[1] = source.Spec.Components[1], source.Spec.Components[0] for edit := range 5 { diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go index 194d06e1ed78..3404349f4d07 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3_test.go @@ -62,14 +62,13 @@ func TestProjectModelV3HybridBuildProjectsSelectedPropSyncWithoutGlobalCoupling( require.Equal(t, partition.ID, spec.NodeLocal.PartitionMappings[index].PartitionID) } - t.Log("Mutating returned fields must not change the projection or caller-owned reference") + t.Log("Mutating returned fields must not change the projection") before := spec.DeepCopy() (*spec.Partitions[0].Extent)[0] = 0 (*spec.PropSyncConnectors[0].Requirement.Connections)[0].FromLogicalDevice = 99 (*spec.PropSyncConnectors[0].Requirement.AcceptableLaneMultiplicities)[0] = 99 spec.AllocationMetadata.Raw[0] = ' ' spec.CyborgPodCliqueRef.Name = "changed" - require.Equal(t, before.CyborgPodCliqueRef, cyborgRef) require.Equal(t, *before, projection.RequestSpec(&MaterializationPlan{CyborgClique: cyborgRef.Name}, "agents")) require.Nil(t, projection.RequestSpec(&MaterializationPlan{}, "agents").CyborgPodCliqueRef) From b7e7d5941136d896e9e66aabeb3ce7743e46a8f9 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 06:52:19 +0000 Subject: [PATCH 174/196] refactor(operator): avoid redundant LPX derived work Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller_test.go | 2 +- .../controller/lpx/pipeline_request_deadline.go | 10 ++++------ .../lpx/pipeline_request_deadline_test.go | 12 ++++++++---- .../operator/internal/controller/lpx/render_test.go | 5 +++-- .../operator/internal/dynamo/lpx/materialization.go | 13 ++++++------- .../internal/dynamo/lpx/materialization_test.go | 6 ++++-- deploy/operator/internal/dynamo/lpx/render_test.go | 2 +- .../internal/dynamo/lpx/resolve_workload_test.go | 2 +- 8 files changed, 28 insertions(+), 24 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 6d2e1c6ab13d..76bb214e1ae3 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -2652,7 +2652,7 @@ func resolveLPXTestWorkload(t *testing.T, registry lpx.ModelRegistry, ctx contex require.NoError(t, err) _, missing, changed := resolvePipelineRequests(deployment, nil, workload, plan) require.False(t, changed) - fixture := &lpxTestWorkload{workload: workload, plan: plan} + fixture := &lpxTestWorkload{workload: workload, plan: plan.ForReplica(0)} for _, request := range missing { fixture.requests = append(fixture.requests, *request) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 87d6db9ff51c..120010ba64e9 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -183,7 +183,8 @@ func (r *graphReconciler) reconcileExpiredPipelineRequests( // evidence for the next reconciliation. An interior failure blocks all // scale-down so a healthy higher ordinal is never removed. // For four replicas, expiry at {2,3} removes that suffix; {0,3} removes nothing. -// All requests must already be validated and target the same scaling group. +// All requests must already be validated and target the same scaling group; +// expired is a subset of requests. func expiredPipelineRequestSuffix( requests []*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, @@ -219,7 +220,6 @@ func expiredPipelineRequestSuffix( expiredNames[request.Name] = struct{}{} } siblings := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(requests)) - failed := make([]*lpxv1alpha1.LPUPipelineRequest, 0, len(expired)) for _, request := range requests { replica := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.ReplicaIndex @@ -227,11 +227,9 @@ func expiredPipelineRequestSuffix( if replica < targetReplicas { continue } - if _, expired := expiredNames[request.Name]; expired { - failed = append(failed, request) - } else { + if _, expired := expiredNames[request.Name]; !expired { siblings = append(siblings, request) } } - return int32(targetReplicas), append(siblings, failed...) + return int32(targetReplicas), append(siblings, expired...) } diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 2aadfdce8380..5ad6fbb61d98 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -267,6 +267,7 @@ func TestExpiredPipelineRequestSuffix(t *testing.T) { }{ {name: "no expiry", replicas: 2, ordinals: []int64{0, 1}, wantReplicas: 2}, {name: "siblings before expired model", replicas: 1, ordinals: []int64{0, 0}, expired: []int{0}, wantRemoved: []string{"request-1", "request-0"}}, + {name: "independently ordered expired models", replicas: 2, ordinals: []int64{0, 0, 1, 1}, expired: []int{2, 0}, wantRemoved: []string{"request-1", "request-3", "request-0", "request-2"}}, {name: "scale already lowered", replicas: 0, ordinals: []int64{0, 0}, expired: []int{0}, wantRemoved: []string{"request-1", "request-0"}}, {name: "finish requests beyond live count", replicas: 1, ordinals: []int64{0, 1, 2}, expired: []int{1}, wantReplicas: 1, wantRemoved: []string{"request-2", "request-1"}}, {name: "hole still blocks cleanup beyond live count", replicas: 2, ordinals: []int64{0, 1, 2}, expired: []int{0, 2}, wantReplicas: 2}, @@ -278,9 +279,9 @@ func TestExpiredPipelineRequestSuffix(t *testing.T) { for i, ordinal := range tc.ordinals { requests[i] = &lpxv1alpha1.LPUPipelineRequest{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("request-%d", i)}} requests[i].Spec.MaterializationTarget.PodCliqueScalingGroupRef = &lpxv1alpha1.PodCliqueScalingGroupReference{Name: "engines", ReplicaIndex: ordinal} - if slices.Contains(tc.expired, i) { - expired = append(expired, requests[i]) - } + } + for _, index := range tc.expired { + expired = append(expired, requests[index]) } replicas, removed := expiredPipelineRequestSuffix(requests, expired, tc.replicas) require.Equal(t, tc.wantReplicas, replicas) @@ -288,7 +289,10 @@ func TestExpiredPipelineRequestSuffix(t *testing.T) { for _, request := range removed { names = append(names, request.Name) } - require.Equal(t, tc.wantRemoved, names) + require.ElementsMatch(t, tc.wantRemoved, names) + if len(removed) > 0 { + require.ElementsMatch(t, expired, removed[len(removed)-len(expired):]) + } }) } } diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 37b43652bb9c..9db77e75c8b9 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -369,7 +369,8 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { workloads, plans, err := r.resolveWorkloads(t.Context(), child, dgd) require.NoError(t, err) plan := plans["lpu"] - planBefore := plan.ForReplica(plan.ReplicaIndex) + planBefore := *plan + planBefore.Agents = slices.Clone(plan.Agents) t.Log("Render twice and require identical independently owned output") first, firstResources, err := r.renderPodCliqueSet(t.Context(), child, dgd, workloads, plans) @@ -408,7 +409,7 @@ func TestLPXRenderingPreservesInputs(t *testing.T) { require.Equal(t, resourcesBefore, secondResources) require.Equal(t, before, dgd) require.Equal(t, childBefore, child) - require.Equal(t, planBefore, plan) + require.Equal(t, planBefore, *plan) }) } } diff --git a/deploy/operator/internal/dynamo/lpx/materialization.go b/deploy/operator/internal/dynamo/lpx/materialization.go index 212d64d81478..69b659a15695 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization.go +++ b/deploy/operator/internal/dynamo/lpx/materialization.go @@ -65,8 +65,8 @@ type MaterializationPlan struct { Replicas int32 } -// PlanNodeLocalMaterialization derives the exact identities shared by graph -// rendering and lifecycle observation without mutating the workload. +// PlanNodeLocalMaterialization derives template and scaling-group identities. +// Concrete child names are supplied by ForReplica; the workload is not mutated. func (w *Workload) PlanNodeLocalMaterialization(pcsName string) (*MaterializationPlan, error) { // Leave room for the fixed scaling group and every LPX role in Grove's name budget. if strings.TrimSpace(pcsName) == "" { @@ -103,7 +103,6 @@ func (w *Workload) PlanNodeLocalMaterialization(pcsName string) (*Materializatio ), Replicas: w.scalingGroupReplicas, } - plan = plan.ForReplica(0) return plan, plan.ValidateReplicaCount() } @@ -176,7 +175,7 @@ func materializedPodHostname(cliqueName string, podIndex int) string { // WithGroup scopes a workload's resource names within a multi-workload PCS. // Sole workloads retain the unscoped plan's established names. -// The receiver is a validated non-nil plan and groupName is an admitted, +// The receiver is a validated non-nil base plan and groupName is an admitted, // non-empty conductor component name from ComponentGroups. // The receiver is not mutated; names that cannot fit Grove's budget are rejected. func (p *MaterializationPlan) WithGroup(groupName string) (*MaterializationPlan, error) { @@ -197,7 +196,8 @@ func (p *MaterializationPlan) WithGroup(groupName string) (*MaterializationPlan, if err != nil { return nil, err } - out := p.ForReplica(0) + out := *p + out.Agents = slices.Clone(p.Agents) out.ResourcePrefix = p.PodCliqueSetName + "-" + resourceName out.ScalingGroupTemplate = name out.LPXScalingGroup = grovecommon.GeneratePodCliqueScalingGroupName(grovecommon.ResourceNameReplica{Name: p.PodCliqueSetName, Replica: 0}, name) @@ -212,8 +212,7 @@ func (p *MaterializationPlan) WithGroup(groupName string) (*MaterializationPlan, for i := range out.Agents { out.Agents[i].TemplateName = name + "-" + out.Agents[i].TemplateName } - out = out.ForReplica(0) - return out, out.ValidateReplicaCount() + return &out, out.ValidateReplicaCount() } // boundedGroupName preserves admitted component names when they fit. Hashes retain diff --git a/deploy/operator/internal/dynamo/lpx/materialization_test.go b/deploy/operator/internal/dynamo/lpx/materialization_test.go index b6a5d8f9b36d..4ca2de9191a5 100644 --- a/deploy/operator/internal/dynamo/lpx/materialization_test.go +++ b/deploy/operator/internal/dynamo/lpx/materialization_test.go @@ -7,6 +7,7 @@ package lpx import ( "math" + "slices" "strings" "testing" @@ -110,7 +111,8 @@ func TestWorkloadNames(t *testing.T) { t.Log("Keep shared-runtime resource names and the PCS identity") plan, err := workload.PlanNodeLocalMaterialization(pcsName) require.NoError(t, err) - before := plan.ForReplica(0) + before := *plan + before.Agents = slices.Clone(plan.Agents) t.Log("Use readable component names while bounding and distinguishing shortened names") names := make(map[string]bool) @@ -148,7 +150,7 @@ func TestWorkloadNames(t *testing.T) { require.Empty(t, validation.IsDNS1123Label(materializedPodHostname(agent.CliqueName, agent.Replicas-1))) } } - require.Equal(t, before, plan) + require.Equal(t, before, *plan) }) } diff --git a/deploy/operator/internal/dynamo/lpx/render_test.go b/deploy/operator/internal/dynamo/lpx/render_test.go index 56271a6a03ce..8a1703009062 100644 --- a/deploy/operator/internal/dynamo/lpx/render_test.go +++ b/deploy/operator/internal/dynamo/lpx/render_test.go @@ -330,7 +330,7 @@ func TestRenderSpecDecodeRoleOwnershipAndTemplateSettings(t *testing.T) { require.Equal(t, []string{"agt0", "agt1", "agt2"}, conductor.Spec.StartsAfter) require.NotContains(t, configMap.Data, "datacenter.toml") nodeNameTemplate := testContainerEnvValue(conductor.Spec.PodSpec.Containers[0].Env, "NOVA_NODE_NAME_TEMPLATE") - for _, agent := range plan.Agents { + for _, agent := range plan.ForReplica(0).Agents { hostname := strings.NewReplacer( "${GROVE_PCSG_NAME}", plan.LPXScalingGroup, "${GROVE_PCSG_INDEX}", "0", diff --git a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go index c24418675e9f..edd7b1beaac8 100644 --- a/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go +++ b/deploy/operator/internal/dynamo/lpx/resolve_workload_test.go @@ -131,7 +131,7 @@ func TestResolveWorkloadDerivesRuntimeShapeFromCompilationMode(t *testing.T) { require.Equal(t, "cond", plan.CyborgTemplate) require.EqualValues(t, 2, plan.Replicas) replica := plan.ForReplica(1) - require.NotEqual(t, plan.Agents[0].CliqueName, replica.Agents[0].CliqueName) + require.NotEqual(t, plan.ForReplica(0).Agents[0].CliqueName, replica.Agents[0].CliqueName) t.Log("A scheduling deadline does not change hybrid launch") dgd.Spec.Components[0].LPX.Scheduling = &v1beta1.SchedulingSpec{AttemptDeadlineSeconds: ptr.To(int64(30))} From f7d038a654f619cac205092b7fff9b50a2da18a0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 08:27:44 +0000 Subject: [PATCH 175/196] refactor(operator): skip no-op LPX reconciliation Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_program.go | 6 ++---- .../internal/controller/lpx/controller.go | 8 +++++++- .../controller/lpx/controller_test.go | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 19bf536dfd08..8426b850b4d9 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -23,7 +23,6 @@ import ( "slices" "time" - nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -171,15 +170,14 @@ func (p *groveProgram) Reconcile( } // Keep LPX creation and updates after ordinary reconciliation and restart selection. - var child *nvidiacomv1alpha1.LPXGraphDeployment if req.DGD.HasLPXComponent() { - child, err = p.lpx.Reconcile(ctx, req.DGD) + child, err := p.lpx.Reconcile(ctx, req.DGD) if err != nil { return programResult, fmt.Errorf("reconcile LPX child: %w", err) } + result = mergeLPXChildStatus(req.DGD, child, result) } - result = mergeLPXChildStatus(req.DGD, child, result) result = applyCheckpointStartupReadiness(result, checkpoints.Infos) if result.State != nvidiacomv1beta1.DGDStatePending || result.Reason != reasonWaitingForCheckpoint { diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 0f93c9a91fad..48c0794f7eb6 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -294,7 +294,7 @@ func (r *graphReconciler) reconcileWorkloads( } // Failure blocks publication while each workload independently cleans up expired suffixes. - if len(expiredRequests) > 0 || isSchedulingFailedConditionCurrent(deployment) { + if len(expiredRequests) > 0 { retained := slices.DeleteFunc(slices.Collect(maps.Values(requests)), func(request *lpxv1alpha1.LPUPipelineRequest) bool { _, desired := desiredRequests[request.Name] return !desired @@ -302,6 +302,12 @@ func (r *graphReconciler) reconcileWorkloads( return r.reconcileSchedulingFailure(ctx, deployment, pcsgs, explicitReplicas, retained, expiredRequests) } + // Keep publication blocked after the failed requests have been removed. + if isSchedulingFailedConditionCurrent(deployment) { + setSchedulingFailedCondition(deployment, false) + return ctrl.Result{}, nil + } + acknowledgeSchedulingRetry(deployment) // Terminating requests must settle before synchronization or scale-out. diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 76bb214e1ae3..0c9fbbca3ca1 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -513,11 +513,28 @@ func TestPipelineRequestDeadlineContinuesDuringRequestDeletion(t *testing.T) { tail = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, tail.Name) require.False(t, tail.DeletionTimestamp.IsZero()) + t.Log("Restore the recorded failure after a transient error without losing the surviving deadline") + require.NoError(t, r.Get(ctx, reconcileRequest.NamespacedName, child)) + failure := meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition).DeepCopy() + r.modelRegistry = &snapshotFailureRegistry{ModelRegistry: registry, err: errors.New("registry unavailable")} + _, err := r.Reconcile(ctx, reconcileRequest) + require.ErrorContains(t, err, "registry unavailable") + r.modelRegistry = registry + result, err := r.Reconcile(ctx, reconcileRequest) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.LessOrEqual(t, result.RequeueAfter, 30*time.Second) + require.NoError(t, r.Get(ctx, reconcileRequest.NamespacedName, child)) + require.Equal(t, failure, meta.FindStatusCondition(child.Status.Conditions, schedulingFailedCondition)) + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + require.Equal(t, v1alpha1.LPXReadyReasonFailed, ready.Reason) + require.Equal(t, failure.Message, ready.Message) + t.Log("Continue enforcing the surviving request's independent deadline") prefix = getTestPipelineRequest(t, ctx, r.Client, child.Namespace, prefix.Name) prefix.Status = newTestPipelineRequest(child, pcs, prefix.Name, time.Now().Add(-time.Minute), lpxv1alpha1.RequestPhasePending).Status require.NoError(t, r.Update(ctx, prefix)) - _, err := r.Reconcile(ctx, reconcileRequest) + _, err = r.Reconcile(ctx, reconcileRequest) require.NoError(t, err) requirePipelineRequestNotFound(t, ctx, r.Client, child.Namespace, prefix.Name) } From 301e6ac20929061b3dccbc964422b8d935951bff Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 13:14:49 +0000 Subject: [PATCH 176/196] refactor(operator): skip unused LPX evaluation work Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_watch_setup.go | 10 +++- .../dynamographdeployment_controller_test.go | 14 +++--- .../internal/controller/lpx/watch_test.go | 7 +++ .../controller_common/workload_predicates.go | 4 -- deploy/operator/internal/dynamo/grove.go | 48 ++++++++++--------- .../dynamo/lpx/workload_projection_v3.go | 6 +-- 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_watch_setup.go b/deploy/operator/internal/controller/dgd_grove_watch_setup.go index c47ccf48d08c..c65314a69451 100644 --- a/deploy/operator/internal/controller/dgd_grove_watch_setup.go +++ b/deploy/operator/internal/controller/dgd_grove_watch_setup.go @@ -75,9 +75,12 @@ func podCliqueEventPredicates() predicate.Funcs { UpdateFunc: func(updateEvent event.UpdateEvent) bool { oldPodClique, oldOK := updateEvent.ObjectOld.(*grovev1alpha1.PodClique) newPodClique, newOK := updateEvent.ObjectNew.(*grovev1alpha1.PodClique) + // Native worker namespace cutover also consumes update completion. return oldOK && newOK && - commoncontroller.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique) + (commoncontroller.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique) || + (oldPodClique.Status.UpdateProgress != nil && oldPodClique.Status.UpdateProgress.UpdateEndedAt != nil) != + (newPodClique.Status.UpdateProgress != nil && newPodClique.Status.UpdateProgress.UpdateEndedAt != nil)) }, GenericFunc: func(event.GenericEvent) bool { return false }, } @@ -90,9 +93,12 @@ func pcsgEventPredicates() predicate.Funcs { UpdateFunc: func(updateEvent event.UpdateEvent) bool { oldScalingGroup, oldOK := updateEvent.ObjectOld.(*grovev1alpha1.PodCliqueScalingGroup) newScalingGroup, newOK := updateEvent.ObjectNew.(*grovev1alpha1.PodCliqueScalingGroup) + // Native worker namespace cutover also consumes update completion. return oldOK && newOK && - commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup) + (commoncontroller.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup) || + (oldScalingGroup.Status.UpdateProgress != nil && oldScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) != + (newScalingGroup.Status.UpdateProgress != nil && newScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil)) }, GenericFunc: func(event.GenericEvent) bool { return false }, } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 887b2b009539..977c8df699e1 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -3749,7 +3749,7 @@ func TestGroveWatchSetup_MapPodCliqueScalingGroupToRequests(t *testing.T) { } } -func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { +func TestPodCliqueEventStatusChanges(t *testing.T) { base := func() *grovev1alpha1.PodClique { return &grovev1alpha1.PodClique{ Spec: grovev1alpha1.PodCliqueSpec{Replicas: 3}, @@ -3856,10 +3856,11 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Log("Wake native reconciliation for readiness and namespace cutover") oldPC := base() newPC := base() tt.mutate(newPC) - assert.Equal(t, tt.want, controller_common.PodCliqueStatusChangeIsSignificant(oldPC, newPC)) + assert.Equal(t, tt.want, podCliqueEventPredicates().Update(event.UpdateEvent{ObjectOld: oldPC, ObjectNew: newPC})) }) } @@ -3873,10 +3874,10 @@ func TestPodCliqueStatusChangeIsSignificant(t *testing.T) { }} newPodClique := oldPodClique.DeepCopy() newPodClique.Status.Conditions[0].Message = "two nodes unavailable" - assert.True(t, controller_common.PodCliqueStatusChangeIsSignificant(oldPodClique, newPodClique)) + assert.True(t, podCliqueEventPredicates().Update(event.UpdateEvent{ObjectOld: oldPodClique, ObjectNew: newPodClique})) } -func TestPCSGStatusChangeIsSignificant(t *testing.T) { +func TestPCSGEventStatusChanges(t *testing.T) { base := func() *grovev1alpha1.PodCliqueScalingGroup { return &grovev1alpha1.PodCliqueScalingGroup{ Spec: grovev1alpha1.PodCliqueScalingGroupSpec{Replicas: 3}, @@ -3967,10 +3968,11 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Log("Wake native reconciliation for readiness and namespace cutover") oldPCSG := base() newPCSG := base() tt.mutate(newPCSG) - assert.Equal(t, tt.want, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldPCSG, newPCSG)) + assert.Equal(t, tt.want, pcsgEventPredicates().Update(event.UpdateEvent{ObjectOld: oldPCSG, ObjectNew: newPCSG})) }) } @@ -3984,7 +3986,7 @@ func TestPCSGStatusChangeIsSignificant(t *testing.T) { }} newScalingGroup := oldScalingGroup.DeepCopy() newScalingGroup.Status.Conditions[0].Message = "two replicas unavailable" - assert.True(t, controller_common.PodCliqueScalingGroupStatusChangeIsSignificant(oldScalingGroup, newScalingGroup)) + assert.True(t, pcsgEventPredicates().Update(event.UpdateEvent{ObjectOld: oldScalingGroup, ObjectNew: newScalingGroup})) } func TestGroveChildEventPredicates(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 0f4699bb5397..5005e5953214 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -96,16 +96,19 @@ func TestGroveEventPredicates(t *testing.T) { name string object client.Object observed client.Object + completed client.Object predicate predicate.Predicate }{ { "clique", &grovev1alpha1.PodClique{ObjectMeta: metadata}, &grovev1alpha1.PodClique{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueStatus{ReadyReplicas: 1}}, + &grovev1alpha1.PodClique{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueStatus{UpdateProgress: &grovev1alpha1.PodCliqueUpdateProgress{UpdateEndedAt: ptr.To(metav1.Now())}}}, podCliquePredicate(), }, { "scaling group", &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metadata}, &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueScalingGroupStatus{ObservedGeneration: ptr.To(int64(2))}}, + &grovev1alpha1.PodCliqueScalingGroup{ObjectMeta: metadata, Status: grovev1alpha1.PodCliqueScalingGroupStatus{UpdateProgress: &grovev1alpha1.PodCliqueScalingGroupUpdateProgress{UpdateEndedAt: ptr.To(metav1.Now())}}}, podCliqueScalingGroupPredicate(), }, } { @@ -118,6 +121,10 @@ func TestGroveEventPredicates(t *testing.T) { require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: "workloads", Name: "materialization"}}}, mapChildToLPXGraphDeployment(t.Context(), test.object), "Grove children need no DGD label for routing") + t.Log("Ignore completion-only changes because LPX does not consume native namespace cutover") + require.False(t, test.predicate.Update(event.UpdateEvent{ObjectOld: test.object, ObjectNew: test.completed})) + require.False(t, test.predicate.Update(event.UpdateEvent{ObjectOld: test.completed, ObjectNew: test.object})) + unrelated := test.object.DeepCopyObject().(client.Object) unrelated.SetAnnotations(nil) observedUnrelated := test.observed.DeepCopyObject().(client.Object) diff --git a/deploy/operator/internal/controller_common/workload_predicates.go b/deploy/operator/internal/controller_common/workload_predicates.go index 9a3b29f0a071..39779d55f92d 100644 --- a/deploy/operator/internal/controller_common/workload_predicates.go +++ b/deploy/operator/internal/controller_common/workload_predicates.go @@ -66,8 +66,6 @@ func PodCliqueStatusChangeIsSignificant( oldPodClique.Spec.Replicas != newPodClique.Spec.Replicas || !ptr.Equal(oldPodClique.Status.ObservedGeneration, newPodClique.Status.ObservedGeneration) || !ptr.Equal(oldPodClique.Status.CurrentPodCliqueSetGenerationHash, newPodClique.Status.CurrentPodCliqueSetGenerationHash) || - (oldPodClique.Status.UpdateProgress != nil && oldPodClique.Status.UpdateProgress.UpdateEndedAt != nil) != - (newPodClique.Status.UpdateProgress != nil && newPodClique.Status.UpdateProgress.UpdateEndedAt != nil) || groveScheduledConditionChanged(oldPodClique.Status.Conditions, newPodClique.Status.Conditions) } @@ -84,7 +82,5 @@ func PodCliqueScalingGroupStatusChangeIsSignificant( oldScalingGroup.Spec.Replicas != newScalingGroup.Spec.Replicas || !ptr.Equal(oldScalingGroup.Status.ObservedGeneration, newScalingGroup.Status.ObservedGeneration) || !ptr.Equal(oldScalingGroup.Status.CurrentPodCliqueSetGenerationHash, newScalingGroup.Status.CurrentPodCliqueSetGenerationHash) || - (oldScalingGroup.Status.UpdateProgress != nil && oldScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) != - (newScalingGroup.Status.UpdateProgress != nil && newScalingGroup.Status.UpdateProgress.UpdateEndedAt != nil) || groveScheduledConditionChanged(oldScalingGroup.Status.Conditions, newScalingGroup.Status.Conditions) } diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index 366066f270c9..4706da0a9f96 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -443,8 +444,30 @@ func observePodCliqueReadiness(ctx context.Context, reader client.Reader, resour logger.V(1).Info("Failed to get PodClique", "error", err, "resourceName", resourceName) return groveComponentReadiness{}, fmt.Errorf("failed to get PodClique %s/%s: %w", namespace, resourceName, err) } - // The cache must observe the requested scale before readiness or namespace cutover. + // Only ordinary components publish PodClique counts and revision state. componentReadiness := podCliqueReadiness(podClique, logger) + componentReadiness.status = v1beta1.ComponentReplicaStatus{ + ComponentKind: v1beta1.ComponentKindPodClique, + ComponentNames: []string{resourceName}, + Replicas: podClique.Status.Replicas, + UpdatedReplicas: podClique.Status.UpdatedReplicas, + ReadyReplicas: ptr.To(podClique.Status.ReadyReplicas), + } + componentReadiness.revision = groveComponentRevisionState{ + generationObserved: podClique.Status.ObservedGeneration != nil && *podClique.Status.ObservedGeneration >= podClique.Generation, + currentPCSRevisionHash: podClique.Status.CurrentPodCliqueSetGenerationHash, + replicas: podClique.Status.Replicas, + updatedReplicas: podClique.Status.UpdatedReplicas, + desiredReplicas: podClique.Spec.Replicas, + updateInProgress: podClique.Status.UpdateProgress != nil, + updateEnded: podClique.Status.UpdateProgress != nil && + podClique.Status.UpdateProgress.UpdateEndedAt != nil, + } + if componentReadiness.revision.generationObserved { + componentReadiness.status.ScheduledReplicas = ptr.To(podClique.Status.ScheduledReplicas) + } + + // The cache must observe the requested scale before readiness or namespace cutover. if expectedReplicas != nil && podClique.Spec.Replicas != *expectedReplicas { componentReadiness.revision.desiredReplicas = *expectedReplicas componentReadiness.revision.generationObserved = false @@ -476,26 +499,7 @@ func podCliqueReadiness(podClique *grovev1alpha1.PodClique, logger logr.Logger) "scheduleGatedReplicas", scheduleGatedReplicas, ) - componentReadiness := groveComponentReadiness{ - status: v1beta1.ComponentReplicaStatus{ - ComponentKind: v1beta1.ComponentKindPodClique, - ComponentNames: []string{resourceName}, - Replicas: replicas, - UpdatedReplicas: podClique.Status.UpdatedReplicas, - ReadyReplicas: &readyReplicas, - }, - revision: groveComponentRevisionState{ - generationObserved: observedGeneration != nil && *observedGeneration >= generation, - currentPCSRevisionHash: podClique.Status.CurrentPodCliqueSetGenerationHash, - replicas: replicas, - updatedReplicas: updatedReplicas, - desiredReplicas: desiredReplicas, - updateInProgress: podClique.Status.UpdateProgress != nil, - updateEnded: podClique.Status.UpdateProgress != nil && - podClique.Status.UpdateProgress.UpdateEndedAt != nil, - }, - } - + componentReadiness := groveComponentReadiness{} if observedGeneration == nil { logger.V(1).Info("PodClique observedGeneration is nil", "resourceName", resourceName) return componentReadiness.withResult(false, groveObservedGenerationNilReason, v1beta1.DGDReadyReasonSomeResourcesNotReady) @@ -506,8 +510,6 @@ func podCliqueReadiness(podClique *grovev1alpha1.PodClique, logger logr.Logger) return componentReadiness.withResult(false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", generation, *observedGeneration), v1beta1.DGDReadyReasonSomeResourcesNotReady) } - componentReadiness.status.ScheduledReplicas = &scheduledReplicas - if desiredReplicas == 0 { return componentReadiness.withResult(true, "", "") } diff --git a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go index 36e2e0fd7360..c939e0134171 100644 --- a/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go +++ b/deploy/operator/internal/dynamo/lpx/workload_projection_v3.go @@ -83,6 +83,9 @@ func projectV3PropSync( if err != nil { return nil, nil, err } + if pipeline != PipelineLPX && len(edgePositions) != len(partitions)-1 { + return nil, nil, fmt.Errorf("V3 LPU-only workloads require a complete adjacent prop-sync connector chain") + } // Project each physical partition into the V3 allocation metadata envelope. partitionInfo := make(map[string]any, len(partitions)+1) @@ -142,8 +145,5 @@ func projectV3PropSync( "prop_sync_pairs": propSyncPairs, }, }) - if pipeline != PipelineLPX && len(connectors) != len(partitions)-1 { - return nil, nil, fmt.Errorf("V3 LPU-only workloads require a complete adjacent prop-sync connector chain") - } return metadata, connectors, nil } From ecef5b56b0894937ce0a038db0bcdae2e28cfa55 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 15:18:32 +0000 Subject: [PATCH 177/196] refactor(operator): skip unused reconciliation preparation Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/controller.go | 6 +----- .../internal/controller/lpx/pipeline_request_deadline.go | 5 ++++- .../internal/webhook/validation/dynamographdeployment.go | 4 +++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 48c0794f7eb6..dec860f43ceb 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -295,11 +295,7 @@ func (r *graphReconciler) reconcileWorkloads( // Failure blocks publication while each workload independently cleans up expired suffixes. if len(expiredRequests) > 0 { - retained := slices.DeleteFunc(slices.Collect(maps.Values(requests)), func(request *lpxv1alpha1.LPUPipelineRequest) bool { - _, desired := desiredRequests[request.Name] - return !desired - }) - return r.reconcileSchedulingFailure(ctx, deployment, pcsgs, explicitReplicas, retained, expiredRequests) + return r.reconcileSchedulingFailure(ctx, deployment, pcsgs, explicitReplicas, requests, desiredRequests, expiredRequests) } // Keep publication blocked after the failed requests have been removed. diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 120010ba64e9..264c3573b05f 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -119,7 +119,7 @@ func (r *graphReconciler) reconcileSchedulingFailure( deployment *v1alpha1.LPXGraphDeployment, pcsgs map[string]*grovev1alpha1.PodCliqueScalingGroup, explicitReplicas map[string]*int32, - requests []*lpxv1alpha1.LPUPipelineRequest, + requests, desiredRequests map[string]*lpxv1alpha1.LPUPipelineRequest, expired []*lpxv1alpha1.LPUPipelineRequest, ) (ctrl.Result, error) { // Persist evidence covering every expired cycle before deleting any expired request. @@ -137,6 +137,9 @@ func (r *graphReconciler) reconcileSchedulingFailure( } requestsByGroup := make(map[string][]*lpxv1alpha1.LPUPipelineRequest) for _, request := range requests { + if _, desired := desiredRequests[request.Name]; !desired { + continue + } name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name requestsByGroup[name] = append(requestsByGroup[name], request) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index c8b19ae801c4..39225656c3d5 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -210,13 +210,15 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeployment( oldComponents = componentsByName(oldDGD.Spec.Components) } specOpts := dynamoGraphDeploymentSpecValidationOptions{ - pcsName: dynamo.PCSNameForDGD(dgd.Name, ordinaryGroveComponents(dgd.Spec.Components)), generation: dgd.Generation, workloadProvider: workloadProvider, grovePathway: grovePathway, grovePathwayRequirement: grovePathwayRequirement, oldComponents: oldComponents, } + if grovePathway { + specOpts.pcsName = dynamo.PCSNameForDGD(dgd.Name, ordinaryGroveComponents(dgd.Spec.Components)) + } allErrs = append(allErrs, v.validateDynamoGraphDeploymentSpec(&dgd.Spec, field.NewPath("spec"), specOpts)...) // LPX children require the Grove route for materialization. From 59caec6b61d90392a36c15b5ff38a644327f4dc0 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 15:41:27 +0000 Subject: [PATCH 178/196] refactor(operator): defer LPX requests until PCS observation Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/controller.go | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index dec860f43ceb..632d1402a673 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -227,41 +227,43 @@ func (r *graphReconciler) reconcileWorkloads( deadlineAt time.Time ) - // Resolve each workload in the same order used for request publication. - for _, groupName := range groupNames { - plan := plans[groupName] - workload := workloads[groupName] - pcsg := pcsgs[plan.LPXScalingGroup] - explicitReplicas[plan.LPXScalingGroup] = dgd.GetComponentByName(groupName).Replicas - - // External scalers own live capacity once Grove has created the groups. - if pcs != nil && explicitReplicas[plan.LPXScalingGroup] == nil { - plan.Replicas = pcsg.Spec.Replicas - if err := plan.ValidateReplicaCount(); err != nil { - return ctrl.Result{}, err + // Prepare requests only after observing the PCS, in publication order. + if pcs != nil { + for _, groupName := range groupNames { + plan := plans[groupName] + workload := workloads[groupName] + pcsg := pcsgs[plan.LPXScalingGroup] + explicitReplicas[plan.LPXScalingGroup] = dgd.GetComponentByName(groupName).Replicas + + // External scalers own live capacity once Grove has created the groups. + if explicitReplicas[plan.LPXScalingGroup] == nil { + plan.Replicas = pcsg.Spec.Replicas + if err := plan.ValidateReplicaCount(); err != nil { + return ctrl.Result{}, err + } } - } - desired, missing, intentChanged := resolvePipelineRequests(deployment, requests, workload, plan) + desired, missing, intentChanged := resolvePipelineRequests(deployment, requests, workload, plan) - if intentChanged { - setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") - return ctrl.Result{}, deletePodCliqueSet(ctx, r, pcs) - } + if intentChanged { + setReadyCondition(deployment, v1beta1.DGDStatePending, "Waiting for the previous PodCliqueSet and its requests to be deleted") + return ctrl.Result{}, deletePodCliqueSet(ctx, r, pcs) + } - maps.Copy(desiredRequests, desired) - missingRequests = append(missingRequests, missing...) + maps.Copy(desiredRequests, desired) + missingRequests = append(missingRequests, missing...) - // Expanded models use their source component's policy, not the conductor's. - secondsByModel := make(map[string]*int64) - for _, projection := range workload.ModelProjections() { - if scheduling := dgd.GetComponentByName(projection.ComponentName()).LPX.Scheduling; scheduling != nil { - secondsByModel[projection.Model()] = scheduling.AttemptDeadlineSeconds + // Expanded models use their source component's policy, not the conductor's. + secondsByModel := make(map[string]*int64) + for _, projection := range workload.ModelProjections() { + if scheduling := dgd.GetComponentByName(projection.ComponentName()).LPX.Scheduling; scheduling != nil { + secondsByModel[projection.Model()] = scheduling.AttemptDeadlineSeconds + } } + expired, next := pipelineRequestDeadlines(desired, secondsByModel, deadlineAt) + expiredRequests = append(expiredRequests, expired...) + deadlineAt = next } - expired, next := pipelineRequestDeadlines(desired, secondsByModel, deadlineAt) - expiredRequests = append(expiredRequests, expired...) - deadlineAt = next } // Preserve the earliest deadline across workloads on every subsequent return. From bb14f74673edc6e56b69f28a43b5a0455043231a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 16:21:57 +0000 Subject: [PATCH 179/196] refactor(operator): skip unused LPX role calculations Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/dynamo/grove_lpx.go | 5 ++++- deploy/operator/internal/dynamo/lpx/render.go | 15 ++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/dynamo/grove_lpx.go b/deploy/operator/internal/dynamo/grove_lpx.go index 792db6d22905..42070743698d 100644 --- a/deploy/operator/internal/dynamo/grove_lpx.go +++ b/deploy/operator/internal/dynamo/grove_lpx.go @@ -79,6 +79,7 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD } name := grovecommon.GeneratePodCliqueName(grovecommon.ResourceNameReplica{Name: pcsg.Name, Replica: int(replica)}, template.Name) memberName := template.Labels[commonconsts.KubeLabelDynamoComponent] + draft, isDraft := statuses[memberName] // Revision checks precede the pure per-clique readiness calculation. pclq := pclqs[name] @@ -88,11 +89,13 @@ func EvaluateLPXGroveReadiness(ctx context.Context, source *v1beta1.DynamoGraphD readiness.reason = fmt.Sprintf("Waiting for LPX role %s", name) case pclq.Status.CurrentPodCliqueSetGenerationHash == nil || *pclq.Status.CurrentPodCliqueSetGenerationHash != *hash: readiness.reason = fmt.Sprintf("LPX role %s has not applied the desired revision", name) + case !isDraft: + readiness = podCliqueReadiness(pclq, log.FromContext(ctx)) default: readiness = observeLPXRole(ctx, pclq) } // Sum complete model instances, never physical Agent Pod counts. - if draft, isDraft := statuses[memberName]; isDraft { + if isDraft { draft.ComponentNames = append(draft.ComponentNames, name) if readiness.status.ReadyReplicas != nil { draft.Replicas += readiness.status.Replicas diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 2f3eab390cc0..1953133eb2cb 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -144,13 +144,6 @@ func RenderNodeLocal( stage := projection.stage if index == 0 || stage != projections[index-1].stage { template = input.Stages[stage] - storagePath, err := lpuModelStoragePath(template.Spec) - if err != nil { - return nil, fmt.Errorf("stage %s: %w", stage, err) - } - if storagePath != modelStoragePath { - return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStoragePath) - } // Publish the model path before template-owned hybrid Agent bindings. if hybrid { @@ -158,6 +151,14 @@ func RenderNodeLocal( if err := applyModelPaths(container, projections, modelStoragePath); err != nil { return nil, fmt.Errorf("stage %s: %w", stage, err) } + } else { + storagePath, err := lpuModelStoragePath(template.Spec) + if err != nil { + return nil, fmt.Errorf("stage %s: %w", stage, err) + } + if storagePath != modelStoragePath { + return nil, fmt.Errorf("stage %s must use the Conductor model-storage mount path %q", stage, modelStoragePath) + } } var conductorSpec *corev1.PodSpec if stage == conductorStage && conductor != nil { From e09c30504f23414770aec2032c279aed325087ca Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Sat, 26 Sep 2026 19:07:24 +0000 Subject: [PATCH 180/196] refactor(operator): skip unaffected deadline cleanup groups Signed-off-by: Andrew Paprotsky --- .../internal/controller/lpx/pipeline_request_deadline.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 264c3573b05f..771163e179ae 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -141,7 +141,9 @@ func (r *graphReconciler) reconcileSchedulingFailure( continue } name := request.Spec.MaterializationTarget.PodCliqueScalingGroupRef.Name - requestsByGroup[name] = append(requestsByGroup[name], request) + if len(expiredByGroup[name]) > 0 { + requestsByGroup[name] = append(requestsByGroup[name], request) + } } // An interior failure in one workload does not prevent another workload's suffix cleanup. From 13fd373bae43356aa157381a9468a5497b1df34d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:07:14 +0300 Subject: [PATCH 181/196] perf(operator): avoid redundant LPX hashing and indexing Signed-off-by: Andrew Paprotsky --- .../operator/internal/dynamo/lpx/cyborg_render.go | 3 ++- .../internal/dynamo/lpx/hybrid_cyborg_bindings.go | 2 +- .../dynamo/lpx/hybrid_cyborg_bindings_test.go | 6 +++--- .../operator/internal/dynamo/lpx/lpu_configmap.go | 9 +++++---- .../internal/dynamo/lpx/lpu_configmap_test.go | 14 ++++++++------ deploy/operator/internal/dynamo/lpx/render.go | 12 ++++++------ .../internal/dynamo/lpx/selected_prop_sync.go | 5 +++++ 7 files changed, 30 insertions(+), 21 deletions(-) diff --git a/deploy/operator/internal/dynamo/lpx/cyborg_render.go b/deploy/operator/internal/dynamo/lpx/cyborg_render.go index 8b5f33a2bb82..3a5638b4ef9c 100644 --- a/deploy/operator/internal/dynamo/lpx/cyborg_render.go +++ b/deploy/operator/internal/dynamo/lpx/cyborg_render.go @@ -26,6 +26,7 @@ func configureHybridCyborg( modelStoragePath string, agentTemplateNames []string, cyborgConfigMap *corev1.ConfigMap, + cyborgConfigHash string, ) error { container := common.FindContainerByName(cyborg.Spec.PodSpec.Containers, commonconsts.MainContainerName) @@ -53,7 +54,7 @@ func configureHybridCyborg( if err := withLPUConfigVolume(&cyborg.Spec.PodSpec, cyborgConfigMap.Name, true); err != nil { return err } - cyborg.Annotations[v1alpha1.AnnotationExtraResourcesHash] = LPUConfigMapHash(cyborgConfigMap) + cyborg.Annotations[v1alpha1.AnnotationExtraResourcesHash] = cyborgConfigHash } cyborg.Spec.StartsAfter = slices.Clone(agentTemplateNames) return nil diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go index 9c5da3dc79e6..504206f1b100 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings.go @@ -15,7 +15,7 @@ import ( // renderCyborgConfigMap renders the XT hybrid Agent endpoints. // The workload and plan must be non-nil, validated, and describe the selected hybrid model. // The workload and plan are not mutated. The caller assigns the returned ConfigMap's namespace. -func (w *Workload) renderCyborgConfigMap(plan *MaterializationPlan) (*corev1.ConfigMap, error) { +func (w *Workload) renderCyborgConfigMap(plan *MaterializationPlan) (*corev1.ConfigMap, string, error) { build := &w.modelProjections[0].configuredBuild // Cyborg supplies the PCS prefix; startup supplies this workload's Grove index. diff --git a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go index 83a92424cea9..bc2e6d637150 100644 --- a/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go +++ b/deploy/operator/internal/dynamo/lpx/hybrid_cyborg_bindings_test.go @@ -39,7 +39,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { t.Log("Render the generated Agent endpoints") plan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - initial, err := workload.renderCyborgConfigMap(plan) + initial, _, err := workload.renderCyborgConfigMap(plan) require.NoError(t, err) t.Log("Verify every workload replica addresses only its own Agents") @@ -48,7 +48,7 @@ func TestRenderSelectedCyborgConfigMapServerNames(t *testing.T) { workload.scalingGroupReplicas = replicas scaledPlan, err := workload.PlanNodeLocalMaterialization("test-dgd") require.NoError(t, err) - configMap, err := workload.renderCyborgConfigMap(scaledPlan) + configMap, _, err := workload.renderCyborgConfigMap(scaledPlan) require.NoError(t, err) require.Equal(t, initial, configMap) require.True(t, *configMap.Immutable) @@ -105,7 +105,7 @@ func TestRenderCyborgConfigMapPreservesProjectedEndpoints(t *testing.T) { require.NoError(t, err) require.Len(t, projection.RequestSpec(plan, "agents").Partitions, test.partitions) require.Equal(t, 2*test.partitions, projection.agentReplicas) - configMap, err := workload.renderCyborgConfigMap(plan) + configMap, _, err := workload.renderCyborgConfigMap(plan) require.NoError(t, err) prefix := lpxScalingGroupTemplateName + "-${GROVE_PCSG_INDEX}-" + plan.Agents[0].TemplateName + "-" servers := make([]string, len(test.wantOffsets)) diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go index 6b3266155cf0..095ab743ef0a 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap.go @@ -24,13 +24,14 @@ import ( const lpuConfigVolumeName = "config" -func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1.ConfigMap, error) { +func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1.ConfigMap, string, error) { // Name immutable configuration from the content hash used by Pod templates. configMap := &corev1.ConfigMap{ Immutable: ptr.To(true), Data: data, } - configMap.Name = fmt.Sprintf("%s-%.16s", namePrefix, LPUConfigMapHash(configMap)) + configHash := LPUConfigMapHash(configMap) + configMap.Name = fmt.Sprintf("%s-%.16s", namePrefix, configHash) // Reject oversized configuration before any caller can publish it. totalSize := 0 @@ -38,14 +39,14 @@ func renderRuntimeConfigMap(namePrefix string, data map[string]string) (*corev1. totalSize += len(value) } if totalSize > corev1.MaxSecretSize { - return nil, fmt.Errorf( + return nil, "", fmt.Errorf( "rendered LPX ConfigMap %q data is %d bytes; maximum is %d", configMap.Name, totalSize, corev1.MaxSecretSize, ) } - return configMap, nil + return configMap, configHash, nil } // LPUConfigMapHash returns the hash stamped on LPU runtime Pod templates. diff --git a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go index a1d7f7c27f6d..8d8934469d7e 100644 --- a/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go +++ b/deploy/operator/internal/dynamo/lpx/lpu_configmap_test.go @@ -20,12 +20,12 @@ func TestRenderRuntimeConfigMapSizeLimit(t *testing.T) { "text": strings.Repeat("é", corev1.MaxSecretSize/2-1), "suffix": "é", } - _, err := renderRuntimeConfigMap("test-lpu", data) + _, _, err := renderRuntimeConfigMap("test-lpu", data) require.NoError(t, err) t.Log("Reject one extra byte with a named error and no usable ConfigMap") data["suffix"] += "x" - configMap, err := renderRuntimeConfigMap("test-lpu", data) + configMap, _, err := renderRuntimeConfigMap("test-lpu", data) require.Nil(t, configMap) require.ErrorContains(t, err, `rendered LPX ConfigMap "test-lpu-`) require.ErrorContains(t, err, "data is 1048577 bytes; maximum is 1048576") @@ -36,14 +36,16 @@ func TestLPXRuntimeConfigNamesMatchPodIdentity(t *testing.T) { t.Run(root, func(t *testing.T) { t.Log("Render immutable runtime tables for PCS and Pod-label identity bounds") data := map[string]string{"runtime": "config"} - lpu, err := renderRuntimeConfigMap(root+"-lpu", data) + lpu, lpuHash, err := renderRuntimeConfigMap(root+"-lpu", data) require.NoError(t, err) - decode, err := renderRuntimeConfigMap(root+"-decode", data) + decode, decodeHash, err := renderRuntimeConfigMap(root+"-decode", data) require.NoError(t, err) t.Log("Resolve the same LPU table from Pod identity and preserve each role suffix") - require.Equal(t, root+"-lpu-"+LPUConfigMapHash(lpu)[:16], lpu.Name) - require.Equal(t, root+"-decode-"+LPUConfigMapHash(decode)[:16], decode.Name) + require.Equal(t, LPUConfigMapHash(lpu), lpuHash) + require.Equal(t, LPUConfigMapHash(decode), decodeHash) + require.Equal(t, root+"-lpu-"+lpuHash[:16], lpu.Name) + require.Equal(t, root+"-decode-"+decodeHash[:16], decode.Name) require.Empty(t, validation.IsDNS1123Subdomain(lpu.Name)) require.Empty(t, validation.IsDNS1123Subdomain(decode.Name)) }) diff --git a/deploy/operator/internal/dynamo/lpx/render.go b/deploy/operator/internal/dynamo/lpx/render.go index 1953133eb2cb..049bcf05891c 100644 --- a/deploy/operator/internal/dynamo/lpx/render.go +++ b/deploy/operator/internal/dynamo/lpx/render.go @@ -89,7 +89,7 @@ func RenderNodeLocal( if err != nil { return nil, err } - configMap, err := renderRuntimeConfigMap(plan.ResourcePrefix+"-lpu", resolvedPartitionData(projections)) + configMap, configHash, err := renderRuntimeConfigMap(plan.ResourcePrefix+"-lpu", resolvedPartitionData(projections)) if err != nil { return nil, err } @@ -98,11 +98,12 @@ func RenderNodeLocal( // Render the optional Cyborg config and construct final resource order once. var ( - cyborgConfigMap *corev1.ConfigMap - extraResources []client.Object + cyborgConfigMap *corev1.ConfigMap + cyborgConfigHash string + extraResources []client.Object ) if v2HybridRuntime { - cyborgConfigMap, err = workload.renderCyborgConfigMap(plan) + cyborgConfigMap, cyborgConfigHash, err = workload.renderCyborgConfigMap(plan) if err != nil { return nil, err } @@ -112,8 +113,6 @@ func RenderNodeLocal( extraResources = []client.Object{configMap} } - configHash := LPUConfigMapHash(configMap) - // Consume the independently rendered conductor without copying Agent startup or placement. var conductor *grovev1alpha1.PodCliqueTemplateSpec if conductorTemplateName != "" { @@ -219,6 +218,7 @@ func RenderNodeLocal( modelStoragePath, agentTemplateNames, cyborgConfigMap, + cyborgConfigHash, ); err != nil { return nil, err } diff --git a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go index e7f3545f6d39..db6811cb448b 100644 --- a/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go +++ b/deploy/operator/internal/dynamo/lpx/selected_prop_sync.go @@ -19,6 +19,11 @@ func validateSelectedPropSyncGraph( subject string, requireCompatibleTopology bool, ) ([]int, error) { + // Empty chain sets have no references or edges to validate. + if len(chains) == 0 { + return []int{}, nil + } + // Index every physical partition once for both reference validation and connector projection. partitionPositions := make(map[int]int, len(partitions)) for position, partition := range partitions { From d014f6022a6704f42cfad56d2213ff993788ec97 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:21:44 +0300 Subject: [PATCH 182/196] perf(operator): skip unnecessary LPX restart and deadline work Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_lpx_restart.go | 12 ++++++------ .../controller/dgd_grove_lpx_restart_test.go | 12 ++++++++++++ .../controller/lpx/pipeline_request_deadline.go | 4 ++++ .../controller/lpx/pipeline_request_deadline_test.go | 9 ++++++++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index b42ae1e38801..00fbfd3f4e06 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -102,15 +102,15 @@ func (r *lpxRestartProgressResolver) observeRestart( return nil } - // Restart progress is resolved before handoff; an old Ready child cannot cover a newer DGD input. - revision, err := dynamo.LPXInputRevision(source, source.Spec.Restart.ID) - if err != nil || child.Spec.InputRevision != revision { + // Skip input hashing until the child reports current readiness. + ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) + if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { return nil } - // The current input must also have a Ready receipt for the child's current generation. - ready := meta.FindStatusCondition(child.Status.Conditions, v1alpha1.LPXReadyCondition) - if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != child.Generation { + // Restart progress is resolved before handoff; an old Ready child cannot cover a newer DGD input. + revision, err := dynamo.LPXInputRevision(source, source.Spec.Restart.ID) + if err != nil || child.Spec.InputRevision != revision { return nil } return child diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go index 86c21506c997..e2c6ab39b8b4 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -34,6 +34,18 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, }}}, } + + t.Log("Keep the restart pending until aggregate readiness is reported") + require.NoError(t, kube.Status().Update(ctx, child)) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) + meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ + Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionFalse, + ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonPending, + }) + require.NoError(t, kube.Status().Update(ctx, child)) + require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) + + t.Log("Complete only when both aggregate and component readiness are current") meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go index 771163e179ae..543a752c91ea 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline.go @@ -26,6 +26,10 @@ const pipelineRequestDeadlineRetryInterval = 5 * time.Second // Creation time bounds the initial wait for the scheduler. On reused requests, // the scheduler-supplied start takes precedence for later scheduling cycles. func pipelineRequestDeadlines(requests map[string]*lpxv1alpha1.LPUPipelineRequest, secondsByModel map[string]*int64, next time.Time) ([]*lpxv1alpha1.LPUPipelineRequest, time.Time) { + if len(secondsByModel) == 0 { + return nil, next + } + var expired []*lpxv1alpha1.LPUPipelineRequest now := time.Now() for _, request := range requests { diff --git a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go index 5ad6fbb61d98..19da853b6ec5 100644 --- a/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go +++ b/deploy/operator/internal/controller/lpx/pipeline_request_deadline_test.go @@ -37,6 +37,7 @@ func TestPipelineRequestDeadlines(t *testing.T) { starts []*metav1.Time created *metav1.Time seconds *int64 + noPolicies bool noStatus bool deleting bool lastPlanRevision int64 @@ -45,6 +46,8 @@ func TestPipelineRequestDeadlines(t *testing.T) { previousWake time.Time }{ {name: "disabled deadline", starts: []*metav1.Time{oldStart}}, + {name: "omitted scheduling", starts: []*metav1.Time{oldStart}, noPolicies: true}, + {name: "omitted scheduling preserves prior deadline", starts: []*metav1.Time{oldStart}, noPolicies: true, previousWake: now, wantWake: now}, {name: "no requests", seconds: ptr.To(int64(30))}, {name: "no scheduler receipt expires from creation", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), noStatus: true, wantExpired: []string{"request-0"}}, {name: "no scheduling start expires from creation", starts: []*metav1.Time{nil}, seconds: ptr.To(int64(30)), wantExpired: []string{"request-0"}}, @@ -85,7 +88,11 @@ func TestPipelineRequestDeadlines(t *testing.T) { } t.Log("Return only expired requests and the earliest still-active deadline") - expired, wake := pipelineRequestDeadlines(requests, map[string]*int64{"default": tc.seconds}, tc.previousWake) + policies := map[string]*int64{"default": tc.seconds} + if tc.noPolicies { + policies = nil + } + expired, wake := pipelineRequestDeadlines(requests, policies, tc.previousWake) var names []string for _, request := range expired { names = append(names, request.Name) From e644da39269a30e66ed21edb573d9ddaa3c4a80b Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:45:46 +0300 Subject: [PATCH 183/196] perf(operator): reconcile scaling adapters only for ordinary components Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/dgd_grove_program.go | 2 +- .../operator/internal/controller/dgd_lpx_handoff_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 8426b850b4d9..5f3d9620d3b8 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -181,7 +181,7 @@ func (p *groveProgram) Reconcile( result = applyCheckpointStartupReadiness(result, checkpoints.Infos) if result.State != nvidiacomv1beta1.DGDStatePending || result.Reason != reasonWaitingForCheckpoint { - if err := p.scalingAdapters.Reconcile(ctx, req.DGD); err != nil { + if err := p.scalingAdapters.Reconcile(ctx, ordinaryDGD); err != nil { log.FromContext(ctx).Error(err, "Failed to reconcile scaling adapters") return programResult, fmt.Errorf("failed to reconcile scaling adapters: %w", err) } diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 8d44adcd0d59..ba536c55466e 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -808,11 +808,19 @@ func TestLPXPendingDownloadDoesNotBlockOrdinaryWorkloads(t *testing.T) { RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true, Checkpoint: true}}, } program := parent.newGroveProgram() + var adapterDeletes []string + program.scalingAdapters.Client = interceptor.NewClient(kube.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, delegated client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + adapterDeletes = append(adapterDeletes, obj.GetName()) + return delegated.Delete(ctx, obj, opts...) + }, + }) t.Log("Publish ordinary resources independently of the child's pending download") result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) require.NoError(t, err) require.Equal(t, v1beta1.DGDStatePending, result.Status.State) + require.Equal(t, []string{generateAdapterName(source.Name, "prefill")}, adapterDeletes) pcs := &grovev1alpha1.PodCliqueSet{} ordinaryDGD := projectWithoutExternallyManagedComponents(source) key := client.ObjectKey{Namespace: source.Namespace, Name: dynamo.PCSNameForDGD(ordinaryDGD.Name, ordinaryDGD.Spec.Components)} From 5ad6600e57b6a169c1eac08b649f692530103009 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 15:17:25 +0300 Subject: [PATCH 184/196] test(operator): remove obsolete LPX tests and disconnected assertions Remove obsolete test scaffolding and disconnected assertions while retaining coverage of current behavior. Signed-off-by: Andrew Paprotsky --- .../dynamographdeployment_conversion_test.go | 9 ---- .../controller/lpx/controller_test.go | 33 ------------ .../internal/dynamo/graph_lpx_test.go | 6 +-- .../dynamo/lpx/model_manifest_v2_test.go | 54 ------------------- 4 files changed, 1 insertion(+), 101 deletions(-) diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index 79a22f92399d..7227ab5c6e0e 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -127,15 +127,6 @@ func TestDGD_RoundTrip_Empty(t *testing.T) { if diff := cmp.Diff(src, got); diff != "" { t.Errorf("round-trip mismatch (-want +got):\n%s", diff) } - - t.Log("Omit LPX status when the deployment has no LPX payload") - raw, err := json.Marshal(got.Status) - if err != nil { - t.Fatal(err) - } - if string(raw) != `{"state":""}` { - t.Fatalf("empty status JSON = %s", raw) - } } func TestDGD_RoundTrip_Minimal(t *testing.T) { diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index 0c9fbbca3ca1..a591b4ab1694 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -3122,39 +3122,6 @@ func TestLPXCommittedDeadlineCleanupPreservesExternalCapacity(t *testing.T) { require.NotEmpty(t, request.Finalizers) } -func TestLPXCommittedReleasePreservesPods(t *testing.T) { - for _, release := range []bool{false, true} { - t.Run(fmt.Sprintf("release=%t", release), func(t *testing.T) { - t.Log("Observe a committed Degraded request with either repair or durable release authority") - ctx := t.Context() - child, dgd, registry := newLPXTestDGD(t, lpx.PipelineSingle) - r, selected := newPreparedLPXTestReconciler(t, registry, ctx, child, dgd) - objects := lpxMaterializedObjects(t, r, child, dgd, selected) - createLPXTestObjects(t, ctx, r.Client, objects...) - publishSelectedLPXForTest(t, ctx, r, child, selected) - request := getTestPipelineRequest(t, ctx, r.Client, child.Namespace, selected.requests[0].Name) - request.Status = newTestPipelineRequest(child, findLPXTestPodCliqueSet(t, objects), request.Name, time.Now(), lpxv1alpha1.RequestPhaseDegraded).Status - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: child.Namespace, Name: "committed-agent", UID: "agent-uid"}} - request.Status.Committed = committedLPXTestPod(pod) - if release { - request.Status.Committed.Execution.Release = &lpxv1alpha1.ReleaseJournal{Reason: lpxv1alpha1.ReleaseReasonDependencyChanged, RequestedAtGeneration: request.Generation} - } - require.NoError(t, r.Update(ctx, request)) - require.NoError(t, r.Create(ctx, pod)) - - t.Log("Neither repair nor release authorizes graph reconciliation to delete Pods") - _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) - require.NoError(t, err) - require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(pod), &corev1.Pod{})) - request = getTestPipelineRequest(t, ctx, r.Client, request.Namespace, request.Name) - require.True(t, request.DeletionTimestamp.IsZero()) - pcsg := &grovev1alpha1.PodCliqueScalingGroup{} - require.NoError(t, r.Get(ctx, client.ObjectKey{Namespace: child.Namespace, Name: selected.plan.LPXScalingGroup}, pcsg)) - require.Equal(t, selected.plan.Replicas, pcsg.Spec.Replicas) - }) - } -} - func committedLPXTestPod(pod *corev1.Pod) *lpxv1alpha1.Committed { return &lpxv1alpha1.Committed{ Plan: lpxv1alpha1.CommittedPlan{Placement: lpxv1alpha1.PlanPlacement{ExecutionBackend: lpxv1alpha1.ExecutionBackendNodeLocal}}, diff --git a/deploy/operator/internal/dynamo/graph_lpx_test.go b/deploy/operator/internal/dynamo/graph_lpx_test.go index 8bf3b6bac6ad..2c39fb288ab5 100644 --- a/deploy/operator/internal/dynamo/graph_lpx_test.go +++ b/deploy/operator/internal/dynamo/graph_lpx_test.go @@ -224,14 +224,11 @@ func TestLPXInputRevision(t *testing.T) { }}, }}, } - deployment := &v1alpha1.LPXGraphDeployment{ObjectMeta: metav1.ObjectMeta{Name: source.Name, Namespace: source.Namespace, UID: "materialization"}} want, err := LPXInputRevision(source, "") require.NoError(t, err) require.Regexp(t, `^sha256:[a-f0-9]{64}$`, want) - pcsName := PCSNameForLPX(deployment) - require.NotEqual(t, PCSNameForDGD(source.Name, source.Spec.Components), pcsName) - t.Log("Prefill-only edits and DGD bookkeeping do not alter the LPX revision or PCS name") + t.Log("Prefill-only edits and DGD bookkeeping do not alter the LPX revision") source.Generation++ source.ResourceVersion = "2" source.Labels = map[string]string{"unrelated": "metadata"} @@ -242,7 +239,6 @@ func TestLPXInputRevision(t *testing.T) { got, err := LPXInputRevision(source, "") require.NoError(t, err) require.Equal(t, want, got) - require.Equal(t, pcsName, PCSNameForLPX(deployment)) t.Log("Each LPX component or shared-input change invalidates the revision; ordinary edits do not") source.GetComponentByName("decode").LPX.Scheduling = &v1beta1.SchedulingSpec{} diff --git a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go index c16a5d010e7d..5f8362d26b58 100644 --- a/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_manifest_v2_test.go @@ -8,7 +8,6 @@ package lpx import ( "testing" - "capnproto.org/go/capnp/v3" manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" "github.com/stretchr/testify/require" ) @@ -293,59 +292,6 @@ func TestBuildFromGbuildManifestV2ValidatesV2OnlyContracts(t *testing.T) { } } -func TestNormalizedV2BuildTreatsManifestBuildIDAsOpaque(t *testing.T) { - t.Parallel() - - t.Log("Define relocated revision-2 build references and opaque manifest identities") - tests := []struct { - name string - ref string - buildID string - }{ - { - name: "local copy", - ref: "file:///models/copied-build", - buildID: "original-producer-id", - }, - { - name: "different GCS bucket and path", - ref: "gs://new-bucket/relocated/renamed-build", - buildID: "original-producer-id", - }, - { - name: "missing producer identity", - ref: "file:///models/build-without-producer-id", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - t.Log("Create a revision-2 snapshot under its deployment locator") - manifest := newManifestV2ContractFixture(t) - if test.buildID != "" { - build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) - require.NoError(t, err) - require.NoError(t, build.SetText(11, test.buildID)) - require.NoError(t, manifest.SetReserved3(build.ToPtr())) - } - payload, err := manifest.Message().Marshal() - require.NoError(t, err) - snapshot := &BuildSnapshot{ - ref: test.ref, - contentID: "sha256:test", - manifestBytes: payload, - } - - t.Log("Use the deployment locator without interpreting manifest build.buildId") - normalized, err := normalizeBuildSnapshot(snapshot) - require.NoError(t, err) - require.Equal(t, test.ref, normalized.build.Path) - }) - } -} - func TestBuildFromGbuildManifestV2RejectsNonLPUArtifactForLPUOnly(t *testing.T) { t.Parallel() From 50dc4435ebba6b07a0ef71473315b935a6c3f78a Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 15:51:38 +0300 Subject: [PATCH 185/196] test(operator): drop retired build ID encoding from LPX fixtures Remove retired manifest build-ID encoding from controller fixtures and regenerate their content-derived golden hashes. Signed-off-by: Andrew Paprotsky --- .../controller/lpx/controller_test.go | 2 +- .../internal/controller/lpx/render_test.go | 30 +++++-------------- .../from_dgd_yaml/lpu-cyborg-specdec.yaml | 18 +++++------ .../from_dgd_yaml/lpu-gpu-specdecode.yaml | 10 +++---- .../testdata/from_dgd_yaml/lpx_v2_vllm.yaml | 10 +++---- .../from_dgd_yaml/node-local-v2-hybrid.yaml | 10 +++---- .../from_dgd_yaml/node-local-v2-lpu-only.yaml | 10 +++---- .../node-local-v2-specdecode.yaml | 18 +++++------ .../node-local-v3-hx-hybrid.yaml | 10 +++---- .../node-local-v3-hx-lpu-only.yaml | 10 +++---- .../node-local-v3-hx-specdecode.yaml | 18 +++++------ .../lpx/testdata/from_dgd_yaml/single_v2.yaml | 10 +++---- 12 files changed, 70 insertions(+), 86 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index a591b4ab1694..f2da6d2317b5 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -237,7 +237,7 @@ func TestLPXExternalCyborgCapacityValidation(t *testing.T) { t.Log("Use two explicitly managed backbones with externally managed workers in groups of four") root := t.TempDir() const buildID = "split-io" - writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, buildID, testV2GraphManifestFixture{ + writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, testV2GraphManifestFixture{ topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", partitionCount: 1, numChips: 8, devicesPerNode: 8, compilationMode: manifestcapnpv2.CompilationMode_lpx, diff --git a/deploy/operator/internal/controller/lpx/render_test.go b/deploy/operator/internal/controller/lpx/render_test.go index 9db77e75c8b9..6f5dac4bba04 100644 --- a/deploy/operator/internal/controller/lpx/render_test.go +++ b/deploy/operator/internal/controller/lpx/render_test.go @@ -472,7 +472,7 @@ func TestLPXReplicaChangesUpdateCyborgTemplate(t *testing.T) { t.Log("Render omitted hybrid capacity for two I/O endpoints with two clients each") root := t.TempDir() const buildID = "split-io" - writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, buildID, testV2GraphManifestFixture{ + writeTestGraphBuild(t, root, buildID, testV2GraphManifestCapnp(t, testV2GraphManifestFixture{ topology: "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106", partitionCount: 1, numChips: 8, devicesPerNode: 8, compilationMode: manifestcapnpv2.CompilationMode_lpx, @@ -589,7 +589,7 @@ func TestRuntimeTemplateChangesPreservePartitionConfig(t *testing.T) { } const buildID = "single-v2-manifest-defaults/build_manifest_defaults" registryRoot := t.TempDir() - writeTestGraphBuild(t, registryRoot, buildID, testGbuildManifestCapnp(t, buildID)) + writeTestGraphBuild(t, registryRoot, buildID, testGbuildManifestCapnp(t)) registry, err := lpx.NewModelRegistry(registryRoot, nil) require.NoError(t, err) @@ -695,7 +695,7 @@ func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegist }, } for buildID, fixture := range v2Builds { - writeTestGraphBuild(t, registryRoot, buildID, testV2GraphManifestCapnp(t, buildID, fixture)) + writeTestGraphBuild(t, registryRoot, buildID, testV2GraphManifestCapnp(t, fixture)) } v3Builds := map[string]testV3GraphManifestFixture{ @@ -708,7 +708,7 @@ func newTestDataModelRegistry(t *testing.T, registryRoot string) lpx.ModelRegist "node-local-v3-hx-sd-target": {}, } for buildID, fixture := range v3Builds { - writeTestGraphBuild(t, registryRoot, buildID, testV3GraphManifestCapnp(t, buildID, fixture)) + writeTestGraphBuild(t, registryRoot, buildID, testV3GraphManifestCapnp(t, fixture)) } registry, err := lpx.NewModelRegistry(registryRoot, nil) @@ -764,17 +764,7 @@ func newTestGraphProgram( return deployment, program } -func populateTestGraphBuild(t *testing.T, manifest manifestcapnpv2.Manifest, wireBuildID string) { - t.Helper() - - t.Log("Populate the compiler-owned build identity used only as opaque provenance") - build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) - require.NoError(t, err) - require.NoError(t, build.SetText(11, wireBuildID)) - require.NoError(t, manifest.SetReserved3(build.ToPtr())) -} - -func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { +func testGbuildManifestCapnp(t *testing.T) []byte { t.Helper() const topology = "URSA_V2_1__Q8__8C__G_96_25__KP_FEC__GHZ_1_0__DRACO_V1_1__G_106" @@ -783,7 +773,6 @@ func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { manifest, err := manifestcapnpv2.NewRootManifest(seg) require.NoError(t, err) manifest.SetContractRevision(manifestcapnpv2.CurrentContractRevision) - populateTestGraphBuild(t, manifest, filepath.Base(registryDir)) model, err := manifest.NewModel() require.NoError(t, err) @@ -841,11 +830,10 @@ func testGbuildManifestCapnp(t *testing.T, registryDir string) []byte { return data } -func testV2GraphManifestCapnp(t *testing.T, buildID string, fixture testV2GraphManifestFixture) []byte { +func testV2GraphManifestCapnp(t *testing.T, fixture testV2GraphManifestFixture) []byte { t.Helper() message, manifest := newTestGraphManifest(t) - populateTestGraphBuild(t, manifest, filepath.Base(buildID)) deployment, program := newTestGraphProgram(t, manifest, fixture.compilationMode, uint32(fixture.partitionCount)*fixture.numChips/fixture.devicesPerNode, 8192) @@ -924,11 +912,10 @@ type testV2GraphManifestFixture struct { ioFanoutFactor uint32 } -func testV3GraphManifestCapnp(t *testing.T, buildID string, fixture testV3GraphManifestFixture) []byte { +func testV3GraphManifestCapnp(t *testing.T, fixture testV3GraphManifestFixture) []byte { t.Helper() message, manifest := newTestGraphManifest(t) - populateTestGraphBuild(t, manifest, buildID) _, program := newTestGraphProgram(t, manifest, fixture.compilationMode, 2, 8192) program.SetNumKvCaches(1) @@ -1024,9 +1011,6 @@ func writeLPXTestBuild( t.Log("Encode the V2 manifest header and the tokenizer consumed by rendered runtimes") message, manifest := newTestGraphManifest(t) - t.Log("Retain the opaque compiler provenance without the removed V1 publication contract") - populateTestGraphBuild(t, manifest, buildID) - t.Log("Describe the same one-batch runtime with explicit V2 runtime I/O and prop-sync evidence") deployment, program := newTestGraphProgram(t, manifest, compilationMode, uint32(len(partitionIDs)*2), 8192) program.SetNumKvCaches(1) diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml index b094af7b5d6e..7d8bfc49b339 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-cyborg-specdec.yaml @@ -57,7 +57,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:31a0ee5e56b73e39b7c7b2c93678df6bf858dedaa77350422c3f557fbdb7d5ad + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:99c53c044b25d089c4cc6839b4b6cd8ecf0089e50a14b0158144c6b93c95ba28 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: lpu-cyborg-specdec @@ -71,8 +71,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -154,7 +154,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -237,8 +237,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:989dd3b851c8165b24124bd9864a75f5080f42ea1a3429340fa49fbce5f967ac + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid @@ -322,7 +322,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec nvidia.com/extra-resources-hash: 32805b2198371ed2ff837e4776e3e59448937c6274efe6602681e42f78f14d41 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -436,7 +436,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 cliqueNames: - draft-cond - draft-agt @@ -445,7 +445,7 @@ spec: replicas: 1 - annotations: lpx.nvidia.com/deployment-name: lpu-cyborg-specdec - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 cliqueNames: - target-agt - target-cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml index 11d3b7929189..1f42df849495 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpu-gpu-specdecode.yaml @@ -29,7 +29,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:26e70c69aa2a43744d6a89c9fffb8c9bbd2711cd6203561d0a97af75e4080798 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:4c12ac1cf94c73e86adf88e2b9ae81de18f9286dcf1d83a0401325c6125a1379 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: lpu-gpu-specdecode @@ -43,8 +43,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -126,7 +126,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -211,7 +211,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: lpu-gpu-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml index 98e58139edda..a325806f8831 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/lpx_v2_vllm.yaml @@ -168,7 +168,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:2b28cb9f0acada1e36c2e2f52a29876f852308f21d2d753da6095dc4261589d6 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:2dee1b45ea0f0fffdb728d5efb43410c0af10a321700032b9a6ee7b91670dbf6 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-lpx-v2-vllm @@ -182,8 +182,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm nvidia.com/extra-resources-hash: e586b76cc8ccd9a1ed2e1b0297032eb1da420881dcfdcb632a4f00f4971308ff - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc75fd5ce4533f15c01b89d239027e6d9be0917011a7624ae79b6b66f1638398 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:33c761dab1229a1a6680cb87e3b9045e4717e863c2645dd909430e23f5759eca + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:edce2257831c38e1802491589b04e99a1dc8054af1d03e31d2939addfa712c2c scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid @@ -390,7 +390,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm nvidia.com/extra-resources-hash: 54f76f5c988eb748803edb9e69ca144c7be656c2028d23217ef3135cbb8fe463 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:edce2257831c38e1802491589b04e99a1dc8054af1d03e31d2939addfa712c2c scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -570,7 +570,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: dgd-lpx-v2-vllm - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d62a2d499f40ba8487c2405eaccf330b5ea52a18efea2fc010c9ac0d63ff71d9 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:edce2257831c38e1802491589b04e99a1dc8054af1d03e31d2939addfa712c2c cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml index 8f1048ee14a8..95ddc0ecc4b4 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-hybrid.yaml @@ -32,7 +32,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:db59c694410ca3a5c6502ab3b9d4de7ec12d4a5c0488fba792685fd35f08fd67 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:023d66933a6bad4fc6bfe3b6542addffc6e5210d6456ba84d1bb41620eabb1a8 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-hybrid @@ -46,8 +46,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid nvidia.com/extra-resources-hash: 355e8ac6768c54e3ee778e2044a7498390f64c82034bbc4cc3dafa0ef4031877 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a70fcf7def9c874b1634097fef306b97a4af1103842c80b3516dcf6bda30ad59 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:989dd3b851c8165b24124bd9864a75f5080f42ea1a3429340fa49fbce5f967ac + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2StrictHybrid @@ -205,7 +205,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid nvidia.com/extra-resources-hash: 734ce683f6b7ea5f7b0c0172b07d5e1ed45773120ab8c7d834967d180c62cea2 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -359,7 +359,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-hybrid - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c5f810defd8bdedcf98fa5c74975630429eade384354fb677c2f3b6f1d6f4462 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:81331790aee117ec75745e2bc3fea28539bbf4b897e80d37702ffc7b0dc8c9a9 cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml index 80c820565585..11b5605ecb6c 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-lpu-only.yaml @@ -29,7 +29,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:561ba3854afee2dd805b99728289b13411f9ebfe379fa60b1262d373c1565f96 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:2724d014e8e4da9b7df44f064fcd31b6e8073b1213774431e3ed168951b9fb43 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-lpu-only @@ -43,8 +43,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -176,7 +176,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only nvidia.com/extra-resources-hash: 6bcf1f4cf22aaf552736014dfc177eec81d993bf14b94b3328a39dd1b7d38e5d - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -349,7 +349,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-lpu-only - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a4b9e49a326db14e1ea8ab4afc0d118c3da3a3c028077474c1369264073837cc + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:15c36b313ead3714748f06279e21588b5fbb83cad6d114dd1cc2849394c15859 cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml index f521b1ef5ecd..62692df40b16 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v2-specdecode.yaml @@ -63,7 +63,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:eeb1e1a79f8739fa99975c047dbffde2c51cf1d1d5cfedae40ee7a3857a55ab2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:fe5d2c6233c5e2cfd2a8e56111f83bcab181e182411904083db72c6893ef4505 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v2-specdecode @@ -77,8 +77,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b0943511c591eb7201003374eacf778c4545a959d7be251547399296e35ee1ae + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:dc519ed94aaea3a53cd51192f407ac9494ae070c518c20e9fdbbfde333138830 scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -214,8 +214,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:d19c12c440463a3ebbc8c86f3fe882a33e7d5f48de493146aaf6d0acfa4fdba7 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:33862d120438d3ab2ce12733089d7f0a1e14f52327fcc541f8d56bee6667817d scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -351,8 +351,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:d2aa1fa3f0cba5a30825fed47074c7a78060965d54075abab08622dc1f4e5eff - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b06e7ae90a96bd569bf228eb8b1d4b1192c4b0df3af7e33202d1fc1965a5247e + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:0d06b39bcb17ce3d6ed500ab51251a7bd13b5bd99b68e261743aa68c66242ca2 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b03f1aba25f64c3237d39e3a46cd9eebc4d976925c7dd8b66ed4a5cd165812e9 scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -488,7 +488,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode nvidia.com/extra-resources-hash: b6f18b8ea30b38c5323a2f08b0c2aacf50a14ca785ed0d1f4fffdc31805fbe80 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a7d3fcf1b942178317a07b74ed6a218154c464e6c32ab6685465c65ffa808349 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -670,7 +670,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v2-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7621a191ee823ec3253598cc1df75c66f5fd342293229da33eed1f6cff4fdfe7 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a7d3fcf1b942178317a07b74ed6a218154c464e6c32ab6685465c65ffa808349 cliqueNames: - cond - agt0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml index c3bc3989ca06..eeac77b6cbcf 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-hybrid.yaml @@ -21,7 +21,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f7d69156466e0ef0f00c871c90da8cd98e6956cb5da9745ec86b462b19dd5d26 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ed4da3eb1af3e922067c9a9ff50a030f573e6cd1a1fda232eac1189cca770462 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-hybrid @@ -35,8 +35,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1b19ea57d294e47602beabc264d0173ec31a48c53cb5c411a0dd53e19a4880f1 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:7ebceb54d905d1010b040ea84b16798a599706a8f598e6f6c9a2f302659fd508 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:587910ef777e8da3e337731bb006cc91f6d317cb513f801034dd5a1445a71997 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxStrictHybrid @@ -193,7 +193,7 @@ spec: roleName: agt - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:587910ef777e8da3e337731bb006cc91f6d317cb513f801034dd5a1445a71997 scheduling.lpu.nvidia.com/role: cyborg-worker labels: lpx.nvidia.com/serving: "true" @@ -329,7 +329,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-hybrid - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:c27c1242c5beb1b585c26d005fcbf0844369c7b173e3b6547b18aa62bd4b3351 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:587910ef777e8da3e337731bb006cc91f6d317cb513f801034dd5a1445a71997 cliqueNames: - agt - cond diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml index 4c5f652a52a8..c3e1be274672 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-lpu-only.yaml @@ -21,7 +21,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:4de325afa19b861c0592d5037388b4e878bdc0f2991f5d75ef485a933ae7e868 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ad40bdcaa350f9af3530b2c25a6aba328dd1670e09b71252abbfe15cc09ee3e6 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-lpu-only @@ -35,8 +35,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:1e4755073877623d76538089043ad7083626cab5bead93415d0147ba1a1023d1 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a1b76d5195edf944e9dde751cc36b43a9e941ad0aa8c63d1b4c830b92e6a84fe + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:e19822f903732b3869396c8c37859b4369f59e23b63148ef1e2a1fe8995effb1 scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -182,7 +182,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only nvidia.com/extra-resources-hash: f8b027cf29abf6cdbab73060b6697b18f7107df8c993859d94073d6012107d12 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:e19822f903732b3869396c8c37859b4369f59e23b63148ef1e2a1fe8995effb1 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -374,7 +374,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-lpu-only - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:ec01dbdeee5b9996f1f162f6645e7f8352ac96673e4b6279adca22638f5e5f95 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:e19822f903732b3869396c8c37859b4369f59e23b63148ef1e2a1fe8995effb1 cliqueNames: - cond - agt diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml index af525da9d6d2..8a99a55305da 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/node-local-v3-hx-specdecode.yaml @@ -42,7 +42,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:34698082a40a6249bd1416b3f21bf429f2aac81064563dd6ebf40bcbd61bbd6d + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:f6ee489114570d7855e29fb0b7da7269aa6d212e1ec5685ac28c996a825f82a7 labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: node-local-v3-hx-specdecode @@ -56,8 +56,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6b42344f7916f4778ffdef73249a7395bbc7a082466cc457953af5de5544c80f + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a1b76d5195edf944e9dde751cc36b43a9e941ad0aa8c63d1b4c830b92e6a84fe + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6e1a785816b64b13735132cd8eb896969e08b0decef40204603bf6dc3de4a7aa scheduling.lpu.nvidia.com/model: draft0 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -207,8 +207,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:f56f9a20b94c49c03962f35597fe341c5fe6f3b806a4545e9af4eacb003e8e3e - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:b94a35c4a019e3213f4497e105522208d2a2707e8469ec13c22d46b99d53c227 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a1b76d5195edf944e9dde751cc36b43a9e941ad0aa8c63d1b4c830b92e6a84fe + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:e0c19aae4ca44f9b93e173ef74faadd66d8e8a7d374a7208fe028765cd218aed scheduling.lpu.nvidia.com/model: draft1 scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -358,8 +358,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cb97b5eb27b508190a4d23dd15ce77ca690e9e7531d577607a22c8bced914cb2 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:8440780b5cdc30ae4bbcf80fda3cce06027325884a8b69751577112153ab2cf5 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:a1b76d5195edf944e9dde751cc36b43a9e941ad0aa8c63d1b4c830b92e6a84fe + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a42854ef7997fb567284c61014cdb64748b4a8f1781fedbbee818337a8a53f8e scheduling.lpu.nvidia.com/model: target scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v3HxLpuOnly @@ -509,7 +509,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode nvidia.com/extra-resources-hash: 19888d3182c23d086b0246d1bf8fce4a6d48e188bc47fd23bbe6cce6a61c7361 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7ab3877b509cd000fc114b2cc5aefce6fa1f809876dc47973958bfa8b21387c8 scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -711,7 +711,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: node-local-v3-hx-specdecode - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:47a34d03645a20d4e73d067f226f9f758e7cfda8170b2eb1d845a24ee043b231 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:7ab3877b509cd000fc114b2cc5aefce6fa1f809876dc47973958bfa8b21387c8 cliqueNames: - cond - agt0 diff --git a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml index c03f196889bb..d11cfb57c56d 100644 --- a/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml +++ b/deploy/operator/internal/dynamo/lpx/testdata/from_dgd_yaml/single_v2.yaml @@ -104,7 +104,7 @@ metadata: metadata: annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:a917077bc269f6b8d4770d022b50dc6208c4b2cc64800c813422b3768321613f + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:3c494ebe4ff7aba249934d756022fc7c84ae0ac14e47b6c98a461c4c98418fcb labels: lpx.nvidia.com/deployment-uid: lpx-render-uid nvidia.com/dynamo-graph-deployment-name: dgd-single-v2 @@ -118,8 +118,8 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 - scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:8222ae5a90349801092d323d462e997d1cd67c2bf846c4094c22689e6f0837d1 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 + scheduling.lpu.nvidia.com/compiler-snapshot-digest: sha256:cc4f94db3a856ce83e6bb27977eae83052ee9564627c6c99e94fffbc3007bdce + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6aeb5412190261570984a42716af882e3cdc008acd7efbd4950da7eb7b8eac9e scheduling.lpu.nvidia.com/model: default scheduling.lpu.nvidia.com/role: agent scheduling.lpu.nvidia.com/workload-mode: v2LpuOnly @@ -255,7 +255,7 @@ spec: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 nvidia.com/extra-resources-hash: 1ecba46d5a3c4e2745d0c55eb63f934eb6bf038f33a4100ce2ecbe514c54f125 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6aeb5412190261570984a42716af882e3cdc008acd7efbd4950da7eb7b8eac9e scheduling.lpu.nvidia.com/role: conductor labels: lpx.nvidia.com/serving: "true" @@ -447,7 +447,7 @@ spec: podCliqueScalingGroups: - annotations: lpx.nvidia.com/deployment-name: dgd-single-v2 - scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:61d531b2661de77517d5247d396038cc59eb8784b89b3c7058360481a74402b8 + scheduling.lpu.nvidia.com/dynamo-workload-digest: sha256:6aeb5412190261570984a42716af882e3cdc008acd7efbd4950da7eb7b8eac9e cliqueNames: - cond - agt From 68d9500345bb3e503d545b4cc197079d9d71987d Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:58:51 +0300 Subject: [PATCH 186/196] refactor(operator): complete LPX restarts from aggregate readiness Use the current aggregate LPX readiness receipt for restart completion and remove obsolete test scaffolding. Signed-off-by: Andrew Paprotsky --- .../controller/dgd_grove_lpx_restart.go | 14 ++----- .../controller/dgd_grove_lpx_restart_test.go | 40 +------------------ .../dgd_lpx_handoff_envtest_test.go | 4 -- .../controller/dgd_lpx_handoff_test.go | 10 ----- .../controller/lpx/controller_test.go | 13 +----- 5 files changed, 6 insertions(+), 75 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go index 00fbfd3f4e06..6fd00a896129 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart.go @@ -28,18 +28,10 @@ func (r *lpxRestartProgressResolver) Resolve( source *v1beta1.DynamoGraphDeployment, inProgress []string, ) []string { - child := r.observeRestart(ctx, source) - remaining := make([]string, 0, len(inProgress)) - for _, name := range inProgress { - if child != nil { - ready := meta.FindStatusCondition(child.Status.Components[name].Conditions, v1alpha1.LPXReadyCondition) - if ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == child.Generation { - continue - } - } - remaining = append(remaining, name) + if r.observeRestart(ctx, source) == nil { + return inProgress } - return remaining + return nil } // resolveCompositeGroveRestartProgress composes child-owned LPX and ordinary Grove observations. diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go index e2c6ab39b8b4..73a168a62c06 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_restart_test.go @@ -28,12 +28,6 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { require.NoError(t, err) require.Equal(t, "restart-1", child.Annotations[dynamo.LPXRestartAnnotation]) child.Status.ObservedGeneration = child.Generation - child.Status.Components = map[string]v1alpha1.LPXComponentStatus{ - "lpx": {Conditions: []metav1.Condition{{ - Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, - ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, - }}}, - } t.Log("Keep the restart pending until aggregate readiness is reported") require.NoError(t, kube.Status().Update(ctx, child)) @@ -45,7 +39,7 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { require.NoError(t, kube.Status().Update(ctx, child)) require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) - t.Log("Complete only when both aggregate and component readiness are current") + t.Log("Complete only when aggregate readiness is current") meta.SetStatusCondition(&child.Status.Conditions, metav1.Condition{ Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation, Reason: v1alpha1.LPXReadyReasonReady, @@ -74,42 +68,12 @@ func TestLPXRestartRequiresCurrentInputRevision(t *testing.T) { require.NoError(t, kube.Update(ctx, updated)) require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) updated.Status.ObservedGeneration = updated.Generation - meta.SetStatusCondition(&updated.Status.Conditions, metav1.Condition{ - Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, - ObservedGeneration: updated.Generation, Reason: v1alpha1.LPXReadyReasonReady, - }) require.NoError(t, kube.Status().Update(ctx, updated)) require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"})) - - t.Log("Keep the restart pending while component readiness is absent, unknown, false, or stale") - for _, scenario := range []struct { - status metav1.ConditionStatus - generation int64 - }{ - {status: "", generation: updated.Generation}, - {status: metav1.ConditionUnknown, generation: updated.Generation}, - {status: metav1.ConditionFalse, generation: updated.Generation}, - {status: metav1.ConditionTrue, generation: updated.Generation - 1}, - } { - component := v1alpha1.LPXComponentStatus{} - if scenario.status != "" { - component.Conditions = []metav1.Condition{{ - Type: v1alpha1.LPXReadyCondition, Status: scenario.status, - ObservedGeneration: scenario.generation, Reason: v1alpha1.LPXReadyReasonPending, - }} - } - updated.Status.Components["lpx"] = component - require.NoError(t, kube.Status().Update(ctx, updated)) - require.Equal(t, []string{"lpx"}, resolver.Resolve(ctx, dgd, []string{"lpx"}), "%+v", scenario) - } - - t.Log("Complete after the component reports Ready for the current child generation") - component := updated.Status.Components["lpx"] - meta.SetStatusCondition(&component.Conditions, metav1.Condition{ + meta.SetStatusCondition(&updated.Status.Conditions, metav1.Condition{ Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: updated.Generation, Reason: v1alpha1.LPXReadyReasonReady, }) - updated.Status.Components["lpx"] = component require.NoError(t, kube.Status().Update(ctx, updated)) require.Empty(t, resolver.Resolve(ctx, dgd, []string{"lpx"})) restart = newDGDRestartReconciler().Resolve(ctx, dgd, &dgd.Status, resolver.Resolve) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index a2a32c9f3828..cd410f2540f4 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -55,10 +55,6 @@ func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { stopTokens, err := tokenizer.NewStopTokens(1) require.NoError(t, err) stopTokens.Set(0, 1) - build, err := capnp.NewStruct(manifest.Segment(), capnp.ObjectSize{DataSize: 8, PointerCount: 12}) - require.NoError(t, err) - require.NoError(t, build.SetText(11, "publication-test")) - require.NoError(t, manifest.SetReserved3(build.ToPtr())) deployment, err := manifest.NewDeployment() require.NoError(t, err) deployment.SetCompilationMode(manifestcapnpv2.CompilationMode_lpuOnly) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index ba536c55466e..459d867c820c 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -641,11 +641,6 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { require.Equal(t, before, current) } require.Equal(t, v1beta1.RestartPhaseCompleted, source.Status.Restart.Phase) - - t.Log("A missing draft status cannot borrow the target's readiness") - delete(child.Status.Components, "draft") - require.NoError(t, kube.Status().Update(t.Context(), child)) - require.Equal(t, []string{"draft"}, progress(t.Context(), source, []string{"draft", "lpx"})) require.Zero(t, pcsReads) t.Log("A missing ordinary PCS keeps only live ordinary restart members pending") @@ -654,11 +649,6 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) { v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: "prefill", ComponentType: v1beta1.ComponentTypePrefill, Replicas: ptr.To(int32(1))}, ) ordinaryDGD = projectWithoutExternallyManagedComponents(source) - child.Status.Components["draft"] = v1alpha1.LPXComponentStatus{ - ComponentReplicaStatus: v1beta1.ComponentReplicaStatus{Replicas: 2}, - Conditions: []metav1.Condition{{Type: v1alpha1.LPXReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: child.Generation}}, - } - require.NoError(t, kube.Status().Update(t.Context(), child)) requested := []string{"draft", "frontend", "lpx", "prefill", "removed"} beforeSource := source.DeepCopy() childReads, pcsReads = 0, 0 diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index f2da6d2317b5..f9cb1d7b9635 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -2561,7 +2561,7 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { require.Equal(t, before, pcs) } -func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { +func TestLPXDisabledPreservesPublishedWorkloadUntilReenabled(t *testing.T) { for _, scenario := range []struct{ name, message string }{ {"LPX", "LPX integration is disabled"}, {"Grove", "Grove is disabled"}, @@ -2636,18 +2636,7 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilDeletion(t *testing.T) { require.Equal(t, list, current) } - t.Log("Deletion remains available while disabled and needs no controller finalizer") - if scenario.name == "LPX" { - r.runtimeConfig.Gate.LPX = false - } else { - r.runtimeConfig.Gate.Grove = false - } - r.modelRegistry = nil require.Empty(t, child.Finalizers) - require.NoError(t, r.Delete(ctx, child)) - _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) - require.NoError(t, err) - require.True(t, apierrors.IsNotFound(r.Get(ctx, key, &v1alpha1.LPXGraphDeployment{}))) }) } } From 4a1e37666f6271e8f19565db3e7b35b43c48a9d7 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:34:38 +0300 Subject: [PATCH 187/196] refactor(operator): hash Grove workers from the ordinary projection Use the ordinary deployment projection for Grove worker hashing instead of forwarding the full graph. Remove unused model-service fake error injection. Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_workload_renderer.go | 8 +++----- .../internal/controller/dgd_grove_workloads_reconciler.go | 1 - .../operator/internal/controller/dgd_lpx_handoff_test.go | 2 +- .../controller/dgd_worker_rollout_reconciler_test.go | 2 +- .../internal/controller/dgd_workload_program_test.go | 4 ++-- .../controller/dynamographdeployment_controller_test.go | 4 ++-- deploy/operator/internal/controller/upgrade_test.go | 4 ++-- deploy/operator/internal/dynamo/lpx/model_fixture_test.go | 4 ---- 8 files changed, 11 insertions(+), 18 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go index a62fa65ddb8f..c5c32f86ba00 100644 --- a/deploy/operator/internal/controller/dgd_grove_workload_renderer.go +++ b/deploy/operator/internal/controller/dgd_grove_workload_renderer.go @@ -71,13 +71,12 @@ func newGroveWorkloadRenderer( func (r *groveWorkloadRenderer) Render( ctx context.Context, - source *nvidiacomv1beta1.DynamoGraphDeployment, ordinary *nvidiacomv1beta1.DynamoGraphDeployment, restartState *dynamo.RestartState, checkpointInfos map[string]*checkpoint.CheckpointInfo, workerGenerationChanged bool, ) (*grovePodCliqueSetRender, error) { - if source == nil || ordinary == nil { + if ordinary == nil { return nil, fmt.Errorf("cannot render Grove PodCliqueSet without a DynamoGraphDeployment") } if r.reader == nil { @@ -97,7 +96,7 @@ func (r *groveWorkloadRenderer) Render( } workerHashSuffixNeeded := shouldRenderGroveWorkerHashSuffix(ordinary, existingPodCliqueSet, workerGenerationChanged) - renderDeployment, err := groveRenderDeployment(ordinary, source, existingPodCliqueSet, workerHashSuffixNeeded) + renderDeployment, err := groveRenderDeployment(ordinary, existingPodCliqueSet, workerHashSuffixNeeded) if err != nil { return nil, err } @@ -133,7 +132,6 @@ func (r *groveWorkloadRenderer) Render( func groveRenderDeployment( ordinary *nvidiacomv1beta1.DynamoGraphDeployment, - hashSource *nvidiacomv1beta1.DynamoGraphDeployment, pcs *grovev1alpha1.PodCliqueSet, workerHashSuffix bool, ) (*nvidiacomv1beta1.DynamoGraphDeployment, error) { @@ -144,7 +142,7 @@ func groveRenderDeployment( if !workerHashSuffix { return renderDeployment, nil } - if err := applyGroveWorkerHashSuffix(renderDeployment, hashSource); err != nil { + if err := applyGroveWorkerHashSuffix(renderDeployment, ordinary); err != nil { return nil, err } return renderDeployment, nil diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 171cd6a682ea..5a073de6fb68 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -93,7 +93,6 @@ func (r *groveWorkloadsReconciler) Reconcile( } renderedPodCliqueSet, err := r.renderer.Render( ctx, - source, ordinary, restartState, checkpointInfos, diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index 459d867c820c..d93c09a132f9 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -124,7 +124,7 @@ func TestProjectWithoutExternallyManagedComponentsDoesNotMutateSource(t *testing t.Log("Render and observe restarts against that same ordinary PCS") if !test.lpxOnly { renderer := newGroveWorkloadRenderer(kube, &configv1alpha1.OperatorConfiguration{}, &commoncontroller.RuntimeConfig{}, nil) - rendered, err := renderer.Render(t.Context(), source, ordinary, nil, nil, false) + rendered, err := renderer.Render(t.Context(), ordinary, nil, nil, false) require.NoError(t, err) require.NotNil(t, rendered.existing) require.Equal(t, pcsName, rendered.existing.Name) diff --git a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go index f62756a1d385..9e0eb4977001 100644 --- a/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_worker_rollout_reconciler_test.go @@ -270,7 +270,7 @@ func TestGroveRenderDeploymentWorkerHashSuffix(t *testing.T) { before := dgd.DeepCopy() t.Log("Render the Grove deployment") - rendered, err := groveRenderDeployment(projectWithoutExternallyManagedComponents(dgd), dgd, nil, tt.workerHashSuffix) + rendered, err := groveRenderDeployment(projectWithoutExternallyManagedComponents(dgd), nil, tt.workerHashSuffix) require.NoError(t, err) worker := rendered.GetComponentByName("worker") require.NotNil(t, worker) diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index 63bad19b48c8..033830e35751 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -485,14 +485,14 @@ func TestGroveRendererFailsWhenResolvedDRADependencyDisappears(t *testing.T) { ) t.Log("Verify the renderer initially publishes the full multinode shape") - rendered, err := renderer.Render(t.Context(), dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + rendered, err := renderer.Render(t.Context(), projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerEngine) assert.Equal(t, int64(4), rendered.gpuShapes["decode"].GPUsPerReplica) t.Log("Delete the dependency without changing DGD generation and render again") require.NoError(t, kubeClient.Delete(t.Context(), claimTemplate)) - _, err = renderer.Render(t.Context(), dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + _, err = renderer.Render(t.Context(), projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.ErrorContains(t, err, "ResourceClaimTemplate default/gpu-template") } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 977c8df699e1..860cfe2732ff 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -1387,7 +1387,7 @@ func TestGroveWorkloadRendererRenderPreservesLegacyWorkerSelectors(t *testing.T) nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) generatedPCS := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -1685,7 +1685,7 @@ func TestGroveWorkloadRendererRenderKeepsNativeWorkerSelectors(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) g.Expect(err).NotTo(gomega.HaveOccurred()) renderDGD := renderedPCS.renderDeployment prefill := renderDGD.GetComponentByName("prefill") diff --git a/deploy/operator/internal/controller/upgrade_test.go b/deploy/operator/internal/controller/upgrade_test.go index 57001c81dd19..a8c428d53863 100644 --- a/deploy/operator/internal/controller/upgrade_test.go +++ b/deploy/operator/internal/controller/upgrade_test.go @@ -806,7 +806,7 @@ spec: &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) pcs := renderedPCS.desired renderDGD := renderedPCS.renderDeployment @@ -932,7 +932,7 @@ func TestGroveNativeWorkerIdentityLabelsStayNative(t *testing.T) { &controller_common.RuntimeConfig{}, nil, ) - renderedPCS, err := renderer.Render(ctx, dgd, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) + renderedPCS, err := renderer.Render(ctx, projectWithoutExternallyManagedComponents(dgd), nil, nil, false) require.NoError(t, err) desired := renderedPCS.desired renderDGD := renderedPCS.renderDeployment diff --git a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go index fd71fbbc7640..49a4e79d2fec 100644 --- a/deploy/operator/internal/dynamo/lpx/model_fixture_test.go +++ b/deploy/operator/internal/dynamo/lpx/model_fixture_test.go @@ -44,7 +44,6 @@ type fakeModelServiceClient struct { filesErr error listRequests []*modelpb.ModelFilesRequest list *modelpb.ModelFileList - listErr error metadataContexts []context.Context } @@ -84,9 +83,6 @@ func (c *fakeModelServiceClient) ListModelFiles( ) (*modelpb.ModelFileList, error) { c.listRequests = append(c.listRequests, request) c.metadataContexts = append(c.metadataContexts, ctx) - if c.listErr != nil { - return nil, c.listErr - } if c.list != nil { return c.list, nil } From adec4b4bc713a7b5e56aa9cf0496cf14062c9ba1 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:57:35 +0300 Subject: [PATCH 188/196] test(operator): exercise successful reconciliation without LPX child reads Exercise ordinary reconciliation through LPX selection in the no-child-read test instead of stopping at an unrelated RBAC error. Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_lpx_handoff_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index d93c09a132f9..ca61c7233bb1 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -691,7 +691,7 @@ func TestGroveProgramWithoutLPXDoesNotReadLPXChild(t *testing.T) { source := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "ordinary", Namespace: "default"}, Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{{ - ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, Replicas: ptr.To(int32(1)), + ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend, PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "frontend"}}}}, }}}, } @@ -706,15 +706,16 @@ func TestGroveProgramWithoutLPXDoesNotReadLPXChild(t *testing.T) { return delegated.Get(ctx, key, obj, opts...) }, }).Build() + config := &configv1alpha1.OperatorConfiguration{} + config.Namespace.Restricted = source.Namespace program := (&DynamoGraphDeploymentReconciler{ - Client: kube, Config: &configv1alpha1.OperatorConfiguration{}, Recorder: events.NewFakeRecorder(10), + Client: kube, Config: config, Recorder: events.NewFakeRecorder(10), RuntimeConfig: &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LPX: true}}, }).newGroveProgram() - t.Log("Reach the ordinary shared-resource error without depending on an LPX child") - result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) - require.ErrorContains(t, err, "RBAC manager not initialized") - require.Equal(t, v1beta1.DGDStateFailed, result.Status.State) + t.Log("Reconcile ordinary workloads without depending on an LPX child") + _, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: source}) + require.NoError(t, err) require.Zero(t, childReads) } From 92b955090e9f307e534d3f701bf3116de73fd7f5 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:08:04 +0300 Subject: [PATCH 189/196] refactor(operator): remove unreachable absent-child LPX status handling Remove unreachable absent-child status handling and exercise pending status after the real LPX handoff. Preserve deletion, freshness and failure behavior. Signed-off-by: Andrew Paprotsky --- .../internal/controller/dgd_grove_lpx_status.go | 8 +++----- .../controller/dgd_lpx_handoff_envtest_test.go | 16 ++++++++-------- .../internal/controller/dgd_lpx_handoff_test.go | 6 ++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/deploy/operator/internal/controller/dgd_grove_lpx_status.go b/deploy/operator/internal/controller/dgd_grove_lpx_status.go index 2343ec71d608..2033242151be 100644 --- a/deploy/operator/internal/controller/dgd_grove_lpx_status.go +++ b/deploy/operator/internal/controller/dgd_grove_lpx_status.go @@ -13,20 +13,18 @@ import ( // mergeLPXChildStatus composes a current LGD observation into the ordinary // Grove result. The outer DGD controller remains the only status writer. -// The result reuses the handoff-owned child's component status. +// source selects LPX and child is the nonnil result of a successful handoff. +// The result reuses the child's component status. func mergeLPXChildStatus( source *v1beta1.DynamoGraphDeployment, child *v1alpha1.LPXGraphDeployment, ordinary ReconcileResult, ) ReconcileResult { components := lpx.Components(source) - if len(components) == 0 && child == nil { - return ordinary - } if ordinary.ComponentStatus == nil { ordinary.ComponentStatus = make(map[string]v1beta1.ComponentReplicaStatus) } - current := len(components) > 0 && child != nil && child.DeletionTimestamp.IsZero() + current := child.DeletionTimestamp.IsZero() observed := current && child.Status.ObservedGeneration == child.Generation for _, component := range components { if observed { diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index cd410f2540f4..220128b5e7a6 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -286,14 +286,6 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { source.Namespace, source.UID, source.Generation = env.Namespace(), "", 0 require.NoError(t, env.Client().Create(t.Context(), source)) - t.Log("Persist the pending component projection using the real DGD status schema") - result := mergeLPXChildStatus(source, nil, ReconcileResult{}) - source.Status.Components = result.ComponentStatus - source.Status.State = result.State - require.NoError(t, env.Client().Status().Update(t.Context(), source)) - require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) - require.Zero(t, source.Status.Components["lpx"].Replicas) - t.Log("Hand off the beta source to one independently observed alpha child with its beta DGD owner") handoff := &dgdLPXHandoff{client: env.Client()} child, err := handoff.Reconcile(t.Context(), source) @@ -306,6 +298,14 @@ func TestLPXGraphDeploymentAPIHandoff(t *testing.T) { require.NoError(t, err) require.Equal(t, revision, child.Spec.InputRevision) + t.Log("Persist the pending component projection using the real DGD status schema") + result := mergeLPXChildStatus(source, child, ReconcileResult{}) + source.Status.Components = result.ComponentStatus + source.Status.State = result.State + require.NoError(t, env.Client().Status().Update(t.Context(), source)) + require.Equal(t, v1beta1.ComponentKindPodCliqueScalingGroup, source.Status.Components["lpx"].ComponentKind) + require.Zero(t, source.Status.Components["lpx"].Replicas) + t.Log("Persist the child's download payload and project its actionable failure") child.Status.ObservedGeneration = child.Generation child.Status.ModelDownload = &v1alpha1.ModelDownloadStatus{Builds: []string{"downloaded-build"}} diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go index ca61c7233bb1..6c3fe9e6a81f 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_test.go @@ -212,12 +212,10 @@ func TestLPXChildStatusRequiresObservedResultsAndCompleteEngine(t *testing.T) { require.Len(t, result.ComponentStatus, 2) require.Equal(t, child.Status.Components["lpx"].ComponentReplicaStatus, result.ComponentStatus["lpx"]) - t.Log("Reject missing or stale children and stale Ready conditions") - for _, failure := range []string{"missing-child", "generation", "condition"} { + t.Log("Reject stale children and stale Ready conditions") + for _, failure := range []string{"generation", "condition"} { stale := child.DeepCopy() switch failure { - case "missing-child": - stale = nil case "generation": stale.Status.ObservedGeneration-- case "condition": From da0471a27bd519140d3d1818e0d0008becbd8875 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:15:48 +0300 Subject: [PATCH 190/196] test(operator): advance generation for unselected LPX restart edits Exercise unselected restart edits with an advanced generation and remove a disconnected fixture comparison. Preserve status-only restart coverage and all download/publication assertions. Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/controller_test.go | 2 -- deploy/operator/internal/controller/lpx/watch_test.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index f9cb1d7b9635..e82f2e4fa8c9 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -795,7 +795,6 @@ func TestLPXDownloadsBeforePublication(t *testing.T) { } t.Log("Block downloads when Grove is disabled and wait for cold builds otherwise") - before := dgd.DeepCopy() _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(child)}) require.NoError(t, err) stored := &v1alpha1.LPXGraphDeployment{} @@ -825,7 +824,6 @@ func TestLPXDownloadsBeforePublication(t *testing.T) { require.NoError(t, r.List(t.Context(), requests)) require.Empty(t, pcs.Items) require.Empty(t, requests.Items) - require.Equal(t, before, dgd) }) } } diff --git a/deploy/operator/internal/controller/lpx/watch_test.go b/deploy/operator/internal/controller/lpx/watch_test.go index 5005e5953214..c85164f13752 100644 --- a/deploy/operator/internal/controller/lpx/watch_test.go +++ b/deploy/operator/internal/controller/lpx/watch_test.go @@ -223,6 +223,7 @@ func TestDGDPredicate(t *testing.T) { restart := dgd.DeepCopy() restart.Spec.Restart = &v1beta1.Restart{ID: "restart"} selectedRestart := restart.DeepCopy() + restart.Generation++ selectedRestart.Status.Restart = &v1beta1.RestartStatus{ObservedID: "restart", Phase: v1beta1.RestartPhaseRestarting, InProgress: []string{"lpx"}} replacement := dgd.DeepCopy() replacement.UID = "replacement" From d37bc3cad16f6aa65c790aec7810efac57014901 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 28 Sep 2026 06:36:49 +0000 Subject: [PATCH 191/196] refactor(operator): remove redundant LPX restart-token validation Remove the redundant restart-token mismatch check after input-revision validation. Preserve ownership, lifecycle, and source-freshness checks. Signed-off-by: Andrew Paprotsky --- deploy/operator/internal/controller/lpx/util.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/operator/internal/controller/lpx/util.go b/deploy/operator/internal/controller/lpx/util.go index a8caecda749c..d3fe6c6f782f 100644 --- a/deploy/operator/internal/controller/lpx/util.go +++ b/deploy/operator/internal/controller/lpx/util.go @@ -57,11 +57,6 @@ func getDynamoGraphDeployment(ctx context.Context, reader client.Reader, deploym return nil, nil } - if actualRestartToken := deployment.Annotations[dynamo.LPXRestartAnnotation]; actualRestartToken != restartToken { - logger.V(4).Info("LPXDynamoGraphDeployment is restarting", "expectedRestartToken", restartToken, "restartToken", actualRestartToken) - return nil, nil - } - return dgd, nil } From e36572c89b41fa58a76d1391da476780a29e8a65 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 28 Sep 2026 07:58:15 +0000 Subject: [PATCH 192/196] fix(operator): use beta paths for LPX conductor errors Signed-off-by: Andrew Paprotsky --- .../validation/dynamographdeployment.go | 14 +++--------- ...hdeployment_lpx_validation_envtest_test.go | 22 +++++++++---------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 39225656c3d5..28f63079547e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -425,15 +425,7 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( )...) } - // Conversion preserves alpha service keys in ComponentName; report conductor errors at their source paths. - conductorComponentsPath := componentsPath - componentPath := componentsPath.Index - if v.hasRuntimeVersionSource(runtimeVersionSourceV1Alpha1) { - conductorComponentsPath = fldPath.Child("services") - componentPath = func(index int) *field.Path { - return conductorComponentsPath.Key(spec.Components[index].ComponentName) - } - } + // Validate conductor requirements on the converted graph. for index := range spec.Components { component := &spec.Components[index] if !component.IsLPX() { @@ -444,12 +436,12 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( continue } if role.PodTemplate == nil { - allErrs = append(allErrs, field.Required(componentPath(index).Child("roles").Index(roleIndex).Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) + allErrs = append(allErrs, field.Required(componentsPath.Index(index).Child("roles").Index(roleIndex).Child("podTemplate"), "LPX conductor requires an explicit podTemplate")) } } } if hasLPXComponent && independentEngineCount != lpxComponentCount && !(lpxComponentCount == 2 && independentEngineCount == 1) { - allErrs = append(allErrs, field.Forbidden(conductorComponentsPath, "LPX components must each declare a conductor role or form a shared draft and target pair")) + allErrs = append(allErrs, field.Forbidden(componentsPath, "LPX components must each declare a conductor role or form a shared draft and target pair")) } if spec.Restart != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 687ea46cd07e..2073c0a58270 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -17,8 +17,6 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { const longLPXComponentName = "abcdefghijklmnopqrstuvwxyzabcd" const conductorRoleErr = "spec.components: Forbidden: LPX components must each declare a conductor role or form a shared draft and target pair" const conductorTemplateErr = "spec.components[0].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" - const alphaConductorRoleErr = "spec.services: Forbidden: LPX components must each declare a conductor role or form a shared draft and target pair" - const alphaConductorTemplateErr = "spec.services[lpx].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" // Keep LPX inputs and oracles together without a separate admission execution path. tests := []dgdAdmissionTestCase{ @@ -256,7 +254,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }, }, { - name: "v1alpha1 LPX image and conductor template errors aggregate at the authored role indices", + name: "v1alpha1 LPX image and conductor template errors use beta paths", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { component := dgd.Spec.Services["lpx"] component.Roles[0], component.Roles[1] = component.Roles[1], component.Roles[0] @@ -268,7 +266,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantWebhookErrs: []string{ "spec.components[0].roles[1].podTemplate.spec.containers[1].image: Required value: must specify a non-empty image", "spec.components[0].roles[1].podTemplate.spec.initContainers[0].image: Required value: must specify a non-empty image", - "spec.services[lpx].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate", + "spec.components[0].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate", }, }, { @@ -408,24 +406,24 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { component := dgd.Spec.Services["lpx"] component.Roles = component.Roles[:1] }), - wantWebhookErrs: []string{alphaConductorRoleErr}, + wantWebhookErrs: []string{conductorRoleErr}, }, { name: "v1alpha1 LPX rejects a missing conductor template on CREATE", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil }), - wantWebhookErrs: []string{alphaConductorTemplateErr}, + wantWebhookErrs: []string{conductorTemplateErr}, }, { - name: "v1alpha1 LPX conductor template error identifies the service key and authored role index", + name: "v1alpha1 LPX conductor template error identifies the converted component and authored role indices", deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { setAlphaLPXSpecDec(dgd, k8sptr.To(int32(1))) target := dgd.Spec.Services["target"] target.Roles[0], target.Roles[1] = target.Roles[1], target.Roles[0] target.Roles[0].PodTemplate = nil }), - wantWebhookErrs: []string{"spec.services[target].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate"}, + wantWebhookErrs: []string{"spec.components[1].roles[0].podTemplate: Required value: LPX conductor requires an explicit podTemplate"}, }, { name: "LPX rejects removing the conductor template on UPDATE", @@ -442,7 +440,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { component := dgd.Spec.Services["lpx"] component.Roles = component.Roles[:1] }), - wantWebhookErrs: []string{alphaConductorRoleErr}, + wantWebhookErrs: []string{conductorRoleErr}, }, { name: "v1alpha1 LPX rejects removing the conductor template on UPDATE", @@ -450,7 +448,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil }), - wantWebhookErrs: []string{alphaConductorTemplateErr}, + wantWebhookErrs: []string{conductorTemplateErr}, }, { name: "LPX rejects retaining an absent conductor template on an unrelated UPDATE", @@ -474,7 +472,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { dgd.Spec.Services["lpx"].Roles[1].PodTemplate = nil dgd.Spec.Services["lpx"].Replicas = k8sptr.To(int32(3)) }), - wantWebhookErrs: []string{alphaConductorTemplateErr}, + wantWebhookErrs: []string{conductorTemplateErr}, }, { name: "LPX admits SpecDecode UPDATE with one conductor and Agent-only draft", @@ -732,7 +730,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), wantWebhookErrs: []string{ `spec.components[0].roles: Required value: must contain the "agent" role`, - alphaConductorRoleErr, + conductorRoleErr, }, }, { From 4ad02ea4e42e1c3119d1182bd91d20833ff593eb Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 28 Sep 2026 10:16:56 +0000 Subject: [PATCH 193/196] fix(operator): enforce the LPX feature gate Signed-off-by: Andrew Paprotsky --- deploy/helm/charts/platform/README.md | 10 ++- deploy/helm/charts/platform/README.md.gotmpl | 8 ++ .../operator/templates/deployment.yaml | 1 + .../platform/components/operator/values.yaml | 2 +- .../platform/tests/lpx_config_test.yaml | 9 +++ deploy/helm/charts/platform/values.yaml | 2 +- deploy/operator/Tiltfile | 1 + .../dynamocomponentdeployment_types.go | 3 + .../dynamocomponentdeployment_types.go | 3 + deploy/operator/cmd/crd-apply/main.go | 4 + deploy/operator/cmd/crd-apply/main_test.go | 38 +++++++-- ...nvidia.com_dynamocomponentdeployments.yaml | 7 +- .../nvidia.com_dynamographdeployments.yaml | 9 ++- .../dgd_lpx_handoff_envtest_test.go | 33 ++++++++ .../dynamographdeployment_controller.go | 13 ++- .../dynamographdeployment_controller_test.go | 51 +++++++++++- .../internal/controller/lpx/controller.go | 27 ++----- .../controller/lpx/controller_test.go | 18 +---- .../operator/internal/controller/lpx/watch.go | 5 +- deploy/operator/internal/features/lpx.go | 2 +- .../validation/dynamographdeployment.go | 10 +++ .../dynamographdeployment_helpers.go | 23 ++++++ ...hdeployment_lpx_validation_envtest_test.go | 80 ++++++++++++++++++- .../dynamographdeployment_unit_test.go | 14 ++++ ...graphdeployment_validation_envtest_test.go | 8 +- .../webhook/validation/shared_v1alpha1.go | 7 ++ .../webhook/validation/suite_envtest_test.go | 2 +- .../additional-resources/api-reference-k8s.md | 12 +-- .../dynamo-component-deployment.mdx | 4 +- .../dynamo-graph-deployment.mdx | 5 ++ .../kubernetes-api/full-api-reference.mdx | 12 +-- 31 files changed, 350 insertions(+), 73 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 219d014fb147..cc52dfb14b18 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -260,7 +260,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | -| dynamo-operator.lpx.enabled | bool | `false` | Enable the Dynamo operator's LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. | +| dynamo-operator.lpx.enabled | bool | `false` | EXPERIMENTAL: Enable the Dynamo operator's LPX integration. Startup requires the externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. | | dynamo-operator.lpx.modelRegistryURL | string | `""` | Configure the LPU model registry used by the LPX integration. | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEVELOPMENT AND TESTING ONLY: Enable namespace-restricted reconciliation and admission. Not supported for production. | @@ -325,6 +325,14 @@ Kubernetes: `>=1.30.0-0` | kai-scheduler.global.affinity | object | `{}` | Affinity for kai-scheduler pods | | etcd.image.repository | string | `"bitnamilegacy/etcd"` | following bitnami announcement for brownout - https://github.com/bitnami/charts/tree/main/bitnami/etcd#%EF%B8%8F-important-notice-upcoming-changes-to-the-bitnami-catalog, we need to use the legacy repository until we migrate to the new "secure" repository | +### LPX Integration + +LPX is experimental and may change incompatibly. Enable it with `dynamo-operator.lpx.enabled=true`. +It requires Grove and the externally installed `scheduling.lpu.nvidia.com/v1alpha1` +`LpuPipelineRequest` API. The installer applies `lpxgraphdeployments.nvidia.com` only when enabled; +with `upgradeCRD=false` or a namespace-restricted operator, install that CRD separately first. +See the [DGD reference](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx) for behavior when LPX is disabled. + ### NATS Configuration NATS is **not required** by the default Dynamo request and event planes, which use TCP and ZMQ, diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index ed9f7495d559..3b9908915e98 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -234,6 +234,14 @@ rejects custom `webhook.namespaceSelector` values. {{ template "chart.valuesSection" . }} +### LPX Integration + +LPX is experimental and may change incompatibly. Enable it with `dynamo-operator.lpx.enabled=true`. +It requires Grove and the externally installed `scheduling.lpu.nvidia.com/v1alpha1` +`LpuPipelineRequest` API. The installer applies `lpxgraphdeployments.nvidia.com` only when enabled; +with `upgradeCRD=false` or a namespace-restricted operator, install that CRD separately first. +See the [DGD reference](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx) for behavior when LPX is disabled. + ### NATS Configuration NATS is **not required** by the default Dynamo request and event planes, which use TCP and ZMQ, diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 51fa293dba59..24e35122f502 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -74,6 +74,7 @@ spec: args: - "--crds-dir=/opt/dynamo-operator/crds/" - "--version={{ .Chart.AppVersion }}" + - "--lpx-enabled={{ .Values.lpx.enabled }}" - "--conversion-webhook-service-name={{ include "dynamo-operator.fullname" . }}-webhook-service" - "--conversion-webhook-service-namespace={{ .Release.Namespace }}" {{- end }} diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 8926ec8fa564..18c345a4dbca 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -24,7 +24,7 @@ upgradeCRD: true # The LPX scheduler and its CRDs must be installed separately. lpx: - # -- Enable the Dynamo operator's LPX integration. Startup requires the + # -- EXPERIMENTAL: Enable the Dynamo operator's LPX integration. Startup requires the # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. enabled: false # -- Configure the LPU model registry used by the LPX integration. diff --git a/deploy/helm/charts/platform/tests/lpx_config_test.yaml b/deploy/helm/charts/platform/tests/lpx_config_test.yaml index dcd0065fad39..7d456be23cb7 100644 --- a/deploy/helm/charts/platform/tests/lpx_config_test.yaml +++ b/deploy/helm/charts/platform/tests/lpx_config_test.yaml @@ -6,6 +6,7 @@ templates: - charts/dynamo-operator/templates/operator-config.yaml - charts/dynamo-operator/templates/manager-rbac.yaml - charts/dynamo-operator/templates/validate-values.yaml + - charts/dynamo-operator/templates/deployment.yaml tests: - it: rejects LPX without Grove integration template: charts/dynamo-operator/templates/validate-values.yaml @@ -21,6 +22,10 @@ tests: - matchRegex: path: data["config.yaml"] pattern: '(?m)^lpx:\n enabled: false$' + - contains: + path: spec.template.spec.initContainers[0].args + content: --lpx-enabled=false + template: charts/dynamo-operator/templates/deployment.yaml - it: enables LPX and externally installed Grove together template: charts/dynamo-operator/templates/operator-config.yaml @@ -34,6 +39,10 @@ tests: - matchRegex: path: data["config.yaml"] pattern: '(?m)^ grove:\n enabled: true$' + - contains: + path: spec.template.spec.initContainers[0].args + content: --lpx-enabled=true + template: charts/dynamo-operator/templates/deployment.yaml - it: renders canonical LPX model registry configuration template: charts/dynamo-operator/templates/operator-config.yaml diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 1a71a30e3aee..695736fa7046 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -90,7 +90,7 @@ dynamo-operator: # The LPX scheduler and its CRDs must be installed separately. lpx: - # -- Enable the Dynamo operator's LPX integration. Startup requires the + # -- EXPERIMENTAL: Enable the Dynamo operator's LPX integration. Startup requires the # externally installed scheduling.lpu.nvidia.com/v1alpha1 LpuPipelineRequest API. enabled: false # -- Configure the LPU model registry used by the LPX integration. diff --git a/deploy/operator/Tiltfile b/deploy/operator/Tiltfile index aac7d2e94dfa..be3c1ea29abf 100644 --- a/deploy/operator/Tiltfile +++ b/deploy/operator/Tiltfile @@ -163,6 +163,7 @@ SKIP_CODEGEN = settings['skip_codegen'] _crd_cmd = ( 'go run ./cmd/crd-apply --crds-dir=' + CRD_DIR + ' --version=' + OPERATOR_VERSION + + ' --lpx-enabled=' + str(HELM_VALUES.get('dynamo-operator.lpx.enabled', False)).lower() + ' --conversion-webhook-service-name=dynamo-dynamo-operator-webhook-service' + ' --conversion-webhook-service-namespace=' + NAMESPACE ) diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 7a9d64f95da5..402ecea6bef8 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -79,6 +79,8 @@ type DynamoComponentDeploymentSharedSpec struct { ServiceName string `json:"serviceName,omitempty"` // ComponentType indicates the role of this component (for example, "main"). + // The DGD-only "lpx" type is experimental, requires the operator's + // lpx.enabled setting, and may change incompatibly. ComponentType string `json:"componentType,omitempty"` // SubComponentType indicates the sub-role of this component (for example, "prefill"). @@ -207,6 +209,7 @@ type DynamoComponentDeploymentSharedSpec struct { // LPX holds LPX integration configuration. Only meaningful when // ComponentType is "lpx". + // Experimental: requires the operator's lpx.enabled setting and may change incompatibly. // +optional LPX *v1beta1.LPXConfig `json:"lpx,omitempty"` diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index 1acd6f5da3ff..c89ca5c944cf 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -110,6 +110,8 @@ type DynamoComponentDeploymentSharedSpec struct { // port mapping, frontend detection, planner RBAC, and the pod label // `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are // first-class values, users can set them directly. + // The DGD-only "lpx" type is experimental, requires the operator's + // lpx.enabled setting, and may change incompatibly. // +optional ComponentType ComponentType `json:"type,omitempty"` @@ -220,6 +222,7 @@ type DynamoComponentDeploymentSharedSpec struct { // lpx holds LPX integration configuration. Only meaningful when // `type` is `lpx`. + // Experimental: requires the operator's lpx.enabled setting and may change incompatibly. // +optional LPX *LPXConfig `json:"lpx,omitempty"` diff --git a/deploy/operator/cmd/crd-apply/main.go b/deploy/operator/cmd/crd-apply/main.go index 8f87809c8e67..049bec512dac 100644 --- a/deploy/operator/cmd/crd-apply/main.go +++ b/deploy/operator/cmd/crd-apply/main.go @@ -32,6 +32,7 @@ const ( func main() { crdsDir := flag.String("crds-dir", "/opt/dynamo-operator/crds/", "Directory containing CRD YAML files") version := flag.String("version", "", "Operator version to stamp on CRDs") + lpxEnabled := flag.Bool("lpx-enabled", false, "Install the experimental LPXGraphDeployment CRD") conversionWebhookServiceName := flag.String( "conversion-webhook-service-name", "", @@ -85,6 +86,9 @@ func main() { log.Error(err, "unable to unmarshal CRD", "file", filePath) os.Exit(1) } + if crd.Name == "lpxgraphdeployments.nvidia.com" && !*lpxEnabled { + continue + } if *version != "" { if crd.Annotations == nil { diff --git a/deploy/operator/cmd/crd-apply/main_test.go b/deploy/operator/cmd/crd-apply/main_test.go index 810df0431d51..e02ec96c58bb 100644 --- a/deploy/operator/cmd/crd-apply/main_test.go +++ b/deploy/operator/cmd/crd-apply/main_test.go @@ -9,11 +9,13 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "testing" "github.com/stretchr/testify/require" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -58,15 +60,20 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { t.Fatalf("build installer: %v\n%s", err, output) } - t.Log("Install and reapply every generated CRD, optionally upgrading an external baseline") - crdDirectories := []string{"../../config/crd/bases", "../../config/crd/bases"} + t.Log("Install with LPX disabled, enable it, then disable it without changing its CRD") + client, err := apiextensionsclient.NewForConfig(config) + require.NoError(t, err) + var lpxCRD *apiextensionsv1.CustomResourceDefinition + crdDirectories := []string{"../../config/crd/bases", "../../config/crd/bases", "../../config/crd/bases"} if baseline := os.Getenv("CRD_APPLY_BASELINE_DIR"); baseline != "" { crdDirectories = append([]string{baseline}, crdDirectories...) } - for _, crdDirectory := range crdDirectories { + for index, crdDirectory := range crdDirectories { + lpxEnabled := index == len(crdDirectories)-2 command := exec.CommandContext(t.Context(), installer, "--crds-dir", crdDirectory, "--version", "schema-test", + "--lpx-enabled="+strconv.FormatBool(lpxEnabled), "--conversion-webhook-service-name", "dynamo-operator-webhook-service", "--conversion-webhook-service-namespace", "dynamo-system", ) @@ -74,13 +81,24 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { if output, err := command.CombinedOutput(); err != nil { t.Fatalf("apply CRDs: %v\n%s", err, output) } + + current, err := client.ApiextensionsV1().CustomResourceDefinitions().Get( + t.Context(), "lpxgraphdeployments.nvidia.com", metav1.GetOptions{}) + switch { + case lpxEnabled: + require.NoError(t, err) + lpxCRD = current + case lpxCRD == nil: + require.True(t, apierrors.IsNotFound(err), "LPX CRD must not be installed while disabled: %v", err) + default: + require.NoError(t, err) + require.Equal(t, lpxCRD.UID, current.UID) + require.Equal(t, lpxCRD.Spec, current.Spec) + require.Equal(t, lpxCRD.Annotations, current.Annotations) + } } t.Log("Verify installed schemas preserve DGD placement and keep model downloads on the private child") - client, err := apiextensionsclient.NewForConfig(config) - if err != nil { - t.Fatal(err) - } for _, name := range []string{ "dynamographdeployments.nvidia.com", "dynamocomponentdeployments.nvidia.com", "lpxgraphdeployments.nvidia.com", } { @@ -129,11 +147,17 @@ func TestCRDApplyInstallsGeneratedSchemas(t *testing.T) { t.Log("Preserve upstream role PodTemplate documentation in both DGD versions") spec := version.Schema.OpenAPIV3Schema.Properties["spec"] var component *apiextensionsv1.JSONSchemaProps + typeField := "type" if version.Name == "v1alpha1" { component = spec.Properties["services"].AdditionalProperties.Schema + typeField = "componentType" } else { component = spec.Properties["components"].Items.Schema } + for _, field := range []string{typeField, "lpx"} { + require.Contains(t, component.Properties[field].Description, "lpx.enabled") + require.Contains(t, component.Properties[field].Description, "may change incompatibly") + } template := component.Properties["roles"].Items.Schema.Properties["podTemplate"] require.NotEmpty(t, template.Description) require.NotEmpty(t, template.Properties["spec"].Description) diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 8260c4e51f14..190669b5138f 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -823,7 +823,10 @@ spec: - message: checkpoint.job cannot be set when checkpointRef is specified rule: '!has(self.job) || !has(self.checkpointRef) || size(self.checkpointRef) == 0' componentType: - description: ComponentType indicates the role of this component (for example, "main"). + description: |- + ComponentType indicates the role of this component (for example, "main"). + The DGD-only "lpx" type is experimental, requires the operator's + lpx.enabled setting, and may change incompatibly. type: string dynamoNamespace: description: |- @@ -36854,6 +36857,8 @@ spec: port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. + The DGD-only "lpx" type is experimental, requires the operator's + lpx.enabled setting, and may change incompatibly. enum: - frontend - worker diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index e49fbaec7abc..1604059e7291 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -1154,7 +1154,10 @@ spec: - message: checkpoint.job cannot be set when checkpointRef is specified rule: '!has(self.job) || !has(self.checkpointRef) || size(self.checkpointRef) == 0' componentType: - description: ComponentType indicates the role of this component (for example, "main"). + description: |- + ComponentType indicates the role of this component (for example, "main"). + The DGD-only "lpx" type is experimental, requires the operator's + lpx.enabled setting, and may change incompatibly. type: string dynamoNamespace: description: |- @@ -11512,6 +11515,7 @@ spec: description: |- LPX holds LPX integration configuration. Only meaningful when ComponentType is "lpx". + Experimental: requires the operator's lpx.enabled setting and may change incompatibly. properties: buildId: description: buildId references the immutable model build. @@ -21018,6 +21022,7 @@ spec: description: |- lpx holds LPX integration configuration. Only meaningful when `type` is `lpx`. + Experimental: requires the operator's lpx.enabled setting and may change incompatibly. properties: buildId: description: buildId references the immutable model build. @@ -37522,6 +37527,8 @@ spec: port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. + The DGD-only "lpx" type is experimental, requires the operator's + lpx.enabled setting, and may change incompatibly. enum: - frontend - worker diff --git a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go index 220128b5e7a6..32c312739603 100644 --- a/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go +++ b/deploy/operator/internal/controller/dgd_lpx_handoff_envtest_test.go @@ -23,6 +23,7 @@ import ( manifestcapnpv2 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/manifest/v2" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/golden" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/stretchr/testify/assert" @@ -40,6 +41,38 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" ) +func TestSetupDynamoGraphDeploymentWithoutLPXCRDs(t *testing.T) { + t.Log("Install only the public DGD workload CRDs with LPX disabled") + env := operatorenv.New(operatorenv.Options{ + RuntimeConfig: &commoncontroller.RuntimeConfig{}, + SetupWebhooks: setupProductionWebhooks, + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "config", "crd", "bases", "nvidia.com_dynamographdeployments.yaml"), + filepath.Join("..", "..", "config", "crd", "bases", "nvidia.com_dynamocomponentdeployments.yaml"), + filepath.Join("..", "..", "config", "crd", "bases", "nvidia.com_dynamographdeploymentscalingadapters.yaml"), + }, + }).RunT(t) + err := env.Client().List(t.Context(), &v1alpha1.LPXGraphDeploymentList{}) + require.True(t, meta.IsNoMatchError(err), "private LPX API must be absent: %v", err) + + t.Log("Start the production DGD controller setup without LPX watches or indexes") + config := env.OperatorConfig().DeepCopy() + config.Namespace.Restricted = env.Namespace() + env.StartManager(func(mgr ctrl.Manager) error { + return SetupDynamoGraphDeployment(mgr, DynamoGraphDeploymentSetupOptions{SetupOptions: SetupOptions{ + Config: config, RuntimeConfig: env.RuntimeConfig(), + }}) + }) + + t.Log("Verify an ordinary DGD still creates its component workloads") + golden.ApplyManifests(t, "testdata/dgd/components/input.yaml", env.Client(), env.Namespace()) + require.EventuallyWithT(t, func(c *assert.CollectT) { + components := &v1beta1.DynamoComponentDeploymentList{} + assert.NoError(c, env.Client().List(t.Context(), components, client.InNamespace(env.Namespace()))) + assert.Len(c, components.Items, 2) + }, 20*time.Second, 50*time.Millisecond) +} + func TestLPXPublicationFailureReachesDGDThroughSetup(t *testing.T) { t.Log("Encode a real two-partition LPU build owned by this test") buildDir := t.TempDir() diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 60bcdc278e65..6b1694568d5f 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -134,6 +134,13 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, err } + // Reject stored LPX graphs before provider selection or any workload writes. + if dynamoDeployment.HasLPXComponent() && !r.RuntimeConfig.Gate.Enabled(features.LPX) { + programResult := newWorkloadProgramResult(dynamoDeployment) + programResult.Fail(dynamoDeployment.Generation, "lpx_disabled", errors.New("LPX integration is disabled")) + return programResult.Result, r.persistWorkloadProgramResult(ctx, dynamoDeployment, programResult) + } + // Lock in the provider before allowing any other reconciliation effects. provider, err := r.ensureWorkloadProvider(ctx, dynamoDeployment) @@ -272,7 +279,6 @@ func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) err dgdPrimaryPredicate(), )). Named(consts.ResourceTypeDynamoGraphDeployment). - Owns(&nvidiacomv1alpha1.LPXGraphDeployment{}). Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(mapDGDWorkerPodToRequests), @@ -353,6 +359,11 @@ func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) err })) } + // LPX child status can wake the DGD only when its controller is enabled. + if r.RuntimeConfig.Gate.Enabled(features.LPX) { + ctrlBuilder = ctrlBuilder.Owns(&nvidiacomv1alpha1.LPXGraphDeployment{}) + } + // Register Grove-owned workload watches only when the Grove feature is enabled. if r.RuntimeConfig.Gate.Enabled(features.Grove) { ctrlBuilder = newGroveWatchSetup(r.Client).addTo(ctrlBuilder) diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 860cfe2732ff..1172ada3db4f 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -150,6 +150,46 @@ func TestDynamoGraphDeploymentReconcileLocksProviderBeforeRejectingStoredCheckpo require.Zero(t, stored.Status.ObservedGeneration) } +func TestDynamoGraphDeploymentReconcileWithLPXDisabled(t *testing.T) { + t.Log("Store a mixed graph without Grove or LPX API types") + dgd := &v1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "mixed", Namespace: "default", Generation: 4}, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "frontend", ComponentType: v1beta1.ComponentTypeFrontend}, + {ComponentName: "worker", ComponentType: v1beta1.ComponentTypeLPX}, + }}, + Status: v1beta1.DynamoGraphDeploymentStatus{ObservedGeneration: 3}, + } + scheme := runtime.NewScheme() + require.NoError(t, v1beta1.AddToScheme(scheme)) + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dgd). + WithStatusSubresource(&v1beta1.DynamoGraphDeployment{}).Build() + reconciler := &DynamoGraphDeploymentReconciler{ + Client: kubeClient, Config: &configv1alpha1.OperatorConfiguration{}, + RuntimeConfig: &controller_common.RuntimeConfig{Gate: features.Gates{Grove: true}}, + } + + t.Log("Reconcile before provider selection or workload effects") + result, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) + require.NoError(t, err) + require.Zero(t, result) + + t.Log("Persist only the disabled diagnosis and preserve prior generation acknowledgement") + stored := &v1beta1.DynamoGraphDeployment{} + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(dgd), stored)) + require.Empty(t, stored.Annotations) + require.Empty(t, stored.Finalizers) + require.Equal(t, dgd.Spec, stored.Spec) + require.Equal(t, dgd.Status.ObservedGeneration, stored.Status.ObservedGeneration) + require.Equal(t, v1beta1.DGDStateFailed, stored.Status.State) + ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready") + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, dgd.Generation, ready.ObservedGeneration) + require.Equal(t, "lpx_disabled", ready.Reason) + require.Equal(t, "LPX integration is disabled", ready.Message) +} + func TestDynamoGraphDeploymentReconcilePersistsComponentProgramLPXRejection(t *testing.T) { t.Log("Create a finalized DGD durably assigned to the component provider with an LPX component") dgd := &v1beta1.DynamoGraphDeployment{ @@ -170,6 +210,7 @@ func TestDynamoGraphDeploymentReconcilePersistsComponentProgramLPXRejection(t *t } controller_common.AddFinalizer(dgd) reconciler := createTestDGDReconcilerWithStatus(dgd) + reconciler.RuntimeConfig.Gate.LPX = true t.Log("Reconcile through the outer controller") _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)}) @@ -235,17 +276,19 @@ func TestDynamoGraphDeploymentReconcileFinalizesDeletingStoredCheckpointIncompat } } -func TestDynamoGraphDeploymentReconcileFinalizesWithoutSnapshotTypes(t *testing.T) { - t.Log("Create a deleting DGD in a scheme without the optional Snapshot API types") +func TestDynamoGraphDeploymentReconcileFinalizesWithoutOptionalAPITypes(t *testing.T) { + t.Log("Create a deleting LPX DGD without optional API types and with LPX disabled") now := metav1.Now() dgd := &v1beta1.DynamoGraphDeployment{ObjectMeta: metav1.ObjectMeta{ Name: "test-dgd", Namespace: "default", DeletionTimestamp: &now, }} + dgd.Spec.Components = []v1beta1.DynamoComponentDeploymentSharedSpec{{ + ComponentName: "worker", ComponentType: v1beta1.ComponentTypeLPX, + }} controller_common.AddFinalizer(dgd) testScheme := runtime.NewScheme() - require.NoError(t, v1alpha1.AddToScheme(testScheme)) require.NoError(t, v1beta1.AddToScheme(testScheme)) kubeClient := fake.NewClientBuilder(). WithScheme(testScheme). @@ -261,7 +304,7 @@ func TestDynamoGraphDeploymentReconcileFinalizesWithoutSnapshotTypes(t *testing. RuntimeConfig: runtimeConfig, } - t.Log("Finalize while treating unregistered Snapshot resources as unavailable") + t.Log("Finalize before the LPX gate check, treating unregistered Snapshot resources as unavailable") result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ NamespacedName: client.ObjectKeyFromObject(dgd), }) diff --git a/deploy/operator/internal/controller/lpx/controller.go b/deploy/operator/internal/controller/lpx/controller.go index 632d1402a673..0d9e2919c6e1 100644 --- a/deploy/operator/internal/controller/lpx/controller.go +++ b/deploy/operator/internal/controller/lpx/controller.go @@ -33,7 +33,6 @@ import ( // graphReconciler reconciles LPXGraphDeployments using their owning DGD's configuration. type graphReconciler struct { - enabled bool recorder events.EventRecorder runtimeConfig *commoncontroller.RuntimeConfig modelRegistry lpx.ModelRegistry @@ -43,11 +42,13 @@ type graphReconciler struct { client.Client } -// Setup registers the LPX controller and its dependencies. +// Setup registers the LPX controller and its dependencies when LPX is enabled. func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runtimeConfig *commoncontroller.RuntimeConfig, secrets dynamo.SecretsRetriever) error { + if !runtimeConfig.Gate.Enabled(features.LPX) { + return nil + } + r := &graphReconciler{ - // Disabled integration still serves child status; deletion uses owner garbage collection. - enabled: integrationDisabledMessage(runtimeConfig) == "", recorder: mgr.GetEventRecorder("lpxgraphdeployment"), runtimeConfig: runtimeConfig, config: config, @@ -55,7 +56,7 @@ func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runti Client: mgr.GetClient(), } - if r.enabled { + if runtimeConfig.Gate.Enabled(features.Grove) { var err error r.modelRegistry, err = newLPXModelRegistry(config) if err != nil { @@ -121,8 +122,8 @@ func (r *graphReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu deployment.Status.Components[name] = component } - if !r.enabled { - setReadyCondition(deployment, v1beta1.DGDStateFailed, integrationDisabledMessage(r.runtimeConfig)) + if !r.runtimeConfig.Gate.Enabled(features.Grove) { + setReadyCondition(deployment, v1beta1.DGDStateFailed, "Grove is disabled") return ctrl.Result{}, nil } @@ -519,15 +520,3 @@ func (r *graphReconciler) deleteUnusedConfigMaps(ctx context.Context, deployment } return nil } - -// integrationDisabledMessage returns an empty string when integration is enabled. -// runtimeConfig is non-nil; child status remains available when disabled. -func integrationDisabledMessage(runtimeConfig *commoncontroller.RuntimeConfig) string { - if !runtimeConfig.Gate.Enabled(features.Grove) { - return "Grove is disabled" - } - if !runtimeConfig.Gate.Enabled(features.LPX) { - return "LPX integration is disabled" - } - return "" -} diff --git a/deploy/operator/internal/controller/lpx/controller_test.go b/deploy/operator/internal/controller/lpx/controller_test.go index e82f2e4fa8c9..3563a0ac3071 100644 --- a/deploy/operator/internal/controller/lpx/controller_test.go +++ b/deploy/operator/internal/controller/lpx/controller_test.go @@ -791,7 +791,6 @@ func TestLPXDownloadsBeforePublication(t *testing.T) { r := newLPXTestReconciler(t, observedRegistry, child, dgd) if scenario.name == "disabled Grove" { r.runtimeConfig.Gate.Grove = false - r.enabled = false } t.Log("Block downloads when Grove is disabled and wait for cold builds otherwise") @@ -2561,7 +2560,6 @@ func TestLPXPodCliqueSetListOrder(t *testing.T) { func TestLPXDisabledPreservesPublishedWorkloadUntilReenabled(t *testing.T) { for _, scenario := range []struct{ name, message string }{ - {"LPX", "LPX integration is disabled"}, {"Grove", "Grove is disabled"}, } { t.Run(scenario.name, func(t *testing.T) { @@ -2577,7 +2575,6 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilReenabled(t *testing.T) { _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.NoError(t, err) } - require.NoError(t, r.Get(ctx, key, child)) before := []client.ObjectList{ &grovev1alpha1.PodCliqueSetList{}, &corev1.ConfigMapList{}, &corev1.ServiceList{}, &lpxv1alpha1.LPUPipelineRequestList{}, @@ -2588,15 +2585,10 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilReenabled(t *testing.T) { } t.Log("Disable the provider before an input edit and an expired deadline can retire the workload") - if scenario.name == "LPX" { - r.runtimeConfig.Gate.LPX = false - } else { - r.runtimeConfig.Gate.Grove = false - } - r.enabled = false + r.runtimeConfig.Gate.Grove = false r.modelRegistry = nil require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)) - originalSpec := dgd.Spec.DeepCopy() + originalBuildID := dgd.Spec.Components[0].LPX.BuildID dgd.Spec.Components[0].LPX.BuildID = "edited-while-disabled" require.NoError(t, r.Update(ctx, dgd)) for range 2 { @@ -2619,10 +2611,9 @@ func TestLPXDisabledPreservesPublishedWorkloadUntilReenabled(t *testing.T) { } t.Log("Re-enable unchanged intent and resume the original PCS and request identities") - dgd.Spec = *originalSpec + dgd.Spec.Components[0].LPX.BuildID = originalBuildID require.NoError(t, r.Update(ctx, dgd)) - r.runtimeConfig.Gate.LPX, r.runtimeConfig.Gate.Grove = true, true - r.enabled = true + r.runtimeConfig.Gate.Grove = true r.modelRegistry = registry _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) require.NoError(t, err) @@ -2854,7 +2845,6 @@ func newLPXTestReconciler( return &graphReconciler{ Client: newLPXTestClient(t, seed...), recorder: recorder, - enabled: true, runtimeConfig: runtimeConfig, modelRegistry: registry, config: config, diff --git a/deploy/operator/internal/controller/lpx/watch.go b/deploy/operator/internal/controller/lpx/watch.go index 555a9e2c1cee..6a7a952e1580 100644 --- a/deploy/operator/internal/controller/lpx/watch.go +++ b/deploy/operator/internal/controller/lpx/watch.go @@ -13,6 +13,7 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -49,8 +50,8 @@ func (r *graphReconciler) setupWithManager(mgr ctrl.Manager) error { Watches(&v1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(r.mapDGDToLPXGraphDeployments), builder.WithPredicates(dgdPredicate())). WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.config, r.runtimeConfig)) - // Primary and DGD events are sufficient while disabled. - if !r.enabled { + // Primary and DGD events are sufficient while Grove is disabled. + if !r.runtimeConfig.Gate.Enabled(features.Grove) { return ctrlBuilder.Complete(r) } diff --git a/deploy/operator/internal/features/lpx.go b/deploy/operator/internal/features/lpx.go index 48ba0bb0cd4e..e069ba564e11 100644 --- a/deploy/operator/internal/features/lpx.go +++ b/deploy/operator/internal/features/lpx.go @@ -14,7 +14,7 @@ import ( // LPX enables LPX scheduler integration. // // Owner: @andrewpaprotsky -// Experimental since: N/A +// Experimental since: v1.6.0 // Beta since: N/A // GA since: N/A // Configuration: lpx.enabled diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 28f63079547e..5bb5f0614c7e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -412,6 +412,11 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpec( allErrs = append(allErrs, validateElasticEPRequiresCommand(spec.BackendFramework, component, componentPath)...) + // Allow existing LPX components to survive disablement without accepting new or changed specs. + allErrs = append(allErrs, lpxComponentGateErrors( + v.ctx, component, opts.oldComponents[component.ComponentName], componentPath, v.runtimeVersionSource, + )...) + allErrs = append(allErrs, v.validateDynamoComponentDeploymentSharedSpec( component, componentPath, @@ -855,6 +860,11 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpecUpdat for i := range newSpec.Components { newComponent := &newSpec.Components[i] oldComponent, exists := oldComponents[newComponent.ComponentName] + + // Enforce the same LPX gate when the stateful validator is called independently. + allErrs = append(allErrs, lpxComponentGateErrors( + v.ctx, newComponent, oldComponent, componentsPath.Index(i), v.runtimeVersionSource, + )...) if !exists { continue } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go index 7e6c1317b886..bcc594472b3f 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go @@ -28,8 +28,10 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" @@ -43,6 +45,27 @@ type clusterTopologyInfo struct { domains []string } +// lpxComponentGateErrors freezes existing LPX graph components when the integration is disabled. +// ctx, newSpec, and fldPath are non-nil; oldSpec is nil for a new component. +func lpxComponentGateErrors( + ctx context.Context, + newSpec, oldSpec *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + fldPath *field.Path, + source runtimeVersionValidationSource, +) field.ErrorList { + if !newSpec.IsLPX() || features.MustGateFrom(ctx).Enabled(features.LPX) || + (oldSpec != nil && apiequality.Semantic.DeepEqual(newSpec, oldSpec)) { + return nil + } + + // Conversion preserves the source service name, so gate diagnostics can target the submitted API. + typePath := fldPath.Child("type") + if source == runtimeVersionSourceV1Alpha1 { + typePath = field.NewPath("spec", "services").Key(newSpec.ComponentName).Child("componentType") + } + return field.ErrorList{field.Forbidden(typePath, "LPX components require lpx.enabled=true")} +} + // invalidDynamoGraphDeploymentError converts allErrs for dgd into an API error. // dgd must not be nil. func invalidDynamoGraphDeploymentError( diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go index 2073c0a58270..3680d6d3184a 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_lpx_validation_envtest_test.go @@ -17,6 +17,8 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { const longLPXComponentName = "abcdefghijklmnopqrstuvwxyzabcd" const conductorRoleErr = "spec.components: Forbidden: LPX components must each declare a conductor role or form a shared draft and target pair" const conductorTemplateErr = "spec.components[0].roles[1].podTemplate: Required value: LPX conductor requires an explicit podTemplate" + const lpxGateErr = "spec.components[0].type: Forbidden: LPX components require lpx.enabled=true" + const alphaLPXGateErr = "spec.services[lpx].componentType: Forbidden: LPX components require lpx.enabled=true" // Keep LPX inputs and oracles together without a separate admission execution path. tests := []dgdAdmissionTestCase{ @@ -175,9 +177,79 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { wantCELErr: "spec.components[0]: Invalid value: scalingAdapter is not supported when type is lpx", }, { - name: "LPX integration availability is deferred to the child controller", + name: "LPX gate rejects CREATE when disabled", + lpxDisabled: true, + deployment: betaLPXDGDForAdmission(nil), + wantWebhookErrs: []string{lpxGateErr}, + }, + { + name: "v1alpha1 LPX gate rejects CREATE when disabled", + lpxDisabled: true, + deployment: alphaLPXDGDForAdmission(nil), + wantWebhookErrs: []string{alphaLPXGateErr}, + }, + // Disabling LPX freezes each existing component, while unrelated updates remain available. + { + name: "LPX gate admits ordinary component updates and reordering when disabled", lpxDisabled: true, - deployment: betaLPXDGDForAdmission(nil), + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Spec.Components = append(dgd.Spec.Components, *betaWorkerComponent(betaDGDForAdmission(nil))) + }), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := *betaWorkerComponent(betaDGDForAdmission(nil)) + worker.Replicas = k8sptr.To(int32(2)) + dgd.Spec.Components = append([]nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{worker}, dgd.Spec.Components...) + }), + }, + { + name: "v1alpha1 LPX gate rejects build updates when disabled", + lpxDisabled: true, + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].LPX.BuildID = "test/replacement" + }), + wantWebhookErrs: []string{alphaLPXGateErr}, + }, + { + name: "v1alpha1 LPX gate rejects ingress updates when disabled", + lpxDisabled: true, + oldDeployment: alphaLPXDGDForAdmission(nil), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Ingress = &nvidiacomv1alpha1.IngressSpec{Enabled: true, Host: "lpx.example.com"} + }), + wantWebhookErrs: []string{alphaLPXGateErr}, + }, + { + name: "v1alpha1 LPX gate admits unrelated updates with unchanged ingress when disabled", + lpxDisabled: true, + oldDeployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Ingress = &nvidiacomv1alpha1.IngressSpec{Enabled: true, Host: "lpx.example.com"} + }), + deployment: alphaLPXDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services["lpx"].Ingress = &nvidiacomv1alpha1.IngressSpec{Enabled: true, Host: "lpx.example.com"} + dgd.Labels = map[string]string{"updated": "true"} + dgd.Spec.Labels = map[string]string{"updated": "true"} + }), + }, + { + name: "LPX gate rejects component addition when disabled", + lpxDisabled: true, + oldDeployment: betaLPXDGDForAdmission(nil), + deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + component := dgd.Spec.Components[0].DeepCopy() + component.ComponentName = "additional" + dgd.Spec.Components = append(dgd.Spec.Components, *component) + }), + wantWebhookErrs: []string{"spec.components[1].type: Forbidden: LPX components require lpx.enabled=true"}, + }, + { + name: "LPX gate admits finalizer removal when disabled", + lpxDisabled: true, + terminating: true, + oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + dgd.Finalizers = []string{dgdTerminatingFinalizer} + }), + deployment: betaLPXDGDForAdmission(nil), }, { name: "LPX rejects the component pathway selected when Grove is disabled", @@ -234,7 +306,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }), }, { - name: "LPX runtime command updates are admitted while the integration is unavailable", + name: "LPX gate rejects runtime command updates when disabled", lpxDisabled: true, oldDeployment: betaLPXDGDForAdmission(nil), deployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { @@ -242,6 +314,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { container.Command = []string{"sh", "-c"} container.Args = []string{"exec /opt/dynamo-lpu serve"} }), + wantWebhookErrs: []string{lpxGateErr}, }, { name: "LPX rejects an unknown Pod role", @@ -502,6 +575,7 @@ func lpxDGDAdmissionCases() []dgdAdmissionTestCase { }, { name: "LPX does not grandfather invalid multinode configuration", + lpxDisabled: true, seedWithoutWebhook: true, oldDeployment: betaLPXDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { dgd.Spec.Components[0].Multinode = &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go index f89600fbe4f1..a35d61eeffa1 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go @@ -46,6 +46,20 @@ func TestDynamoGraphDeploymentConversionFailureIsFatal(t *testing.T) { } } +// The public handler rejects changed LPX specs before reaching its independent update validator. +func TestDynamoGraphDeploymentValidateUpdateLPXGate(t *testing.T) { + t.Log("Change an existing LPX component while LPX is disabled") + oldDGD := newBetaDGDForValidation() + oldDGD.Spec.Components[0].ComponentType = nvidiacomv1beta1.ComponentTypeLPX + newDGD := oldDGD.DeepCopy() + newDGD.Spec.Components[0].Replicas = k8sptr.To(int32(2)) + + t.Log("Invoke the stateful validator independently of the handler's new-state traversal") + ctx := features.WithGate(t.Context(), features.Gates{Grove: true}) + _, err := newDynamoGraphDeploymentTestValidator(t).ValidateUpdate(ctx, oldDGD, newDGD, nil, "", runtimeVersionSourceV1Beta1) + assertBetaValidationErrors(t, err, []string{"spec.components[0].type: Forbidden: LPX components require lpx.enabled=true"}) +} + func assertFieldPaths(t *testing.T, errs field.ErrorList, want []string) { t.Helper() got := make([]string, len(errs)) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index 10a6ce169fbf..17e0a3190fe9 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -80,8 +80,9 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { tests := []dgdAdmissionTestCase{ // Baseline create-path rules. { - name: "valid deployment with components", - deployment: betaDGDForAdmission(nil), + name: "valid deployment with components", + lpxDisabled: true, + deployment: betaDGDForAdmission(nil), }, { name: "beta component main image is required when pod template is absent on create", @@ -3067,10 +3068,11 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { if test.oldBeforeUpdate == nil { test.oldBeforeUpdate = dgdBeforeRestart(t, tt.oldDeployment) } - if tt.checkpointOff || tt.groveDisabled { + if tt.checkpointOff || tt.groveDisabled || tt.lpxDisabled { seedGates := gates seedGates.Checkpoint = true seedGates.Grove = true + seedGates.LPX = true test.seedGates = &seedGates } } diff --git a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go index fa4368a194ff..5febec5fcef5 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go @@ -22,6 +22,7 @@ import ( nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -110,6 +111,12 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdateV1al ) field.ErrorList { allErrs := field.ErrorList{} + // Alpha-only fields are absent from the storage-version component comparison. + if newSpec.IsLPX() && !features.MustGateFrom(v.ctx).Enabled(features.LPX) && + !apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("componentType"), "LPX components require lpx.enabled=true")) + } + // Ratchet only complete, unchanged source-version runtime contract violations. if v.hasRuntimeVersionSource(runtimeVersionSourceV1Alpha1) && !newSpec.IsLPX() { newImage, imagePath := runtimeVersionImageAndPathV1Alpha1(newSpec, fldPath) diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 6a86ed026f9c..2e42a5473b7a 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -159,12 +159,12 @@ func runAdmissionTest(t *testing.T, test admissionTestCase) *unstructured.Unstru seedClient = newAdmissionResourceClient(t, env, test.oldObject, legacySeedUsername, warnings) } old := seedAdmissionObject(t, seedClient, test, env.Namespace()) + admissionGate.set(test.gates) if test.terminating { old = beginTermination(t, seedClient, old) } t.Log("Submit the update request through the Kubernetes API server") - admissionGate.set(test.gates) warnings.clear() oldFixture, _ := admissionObject(t, test.oldObject, env.Namespace(), nil) current = applyAdmissionFixtureChanges(old, oldFixture, current) diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 5a848c99d7d2..d85a1143f4de 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -319,7 +319,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | Annotations to add to generated Kubernetes resources for this component
(such as Pod, Service, and Ingress when applicable). | | | | `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | | | `serviceName` _string_ | The name of the component | | | -| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | | +| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main").
The DGD-only "lpx" type is experimental, requires the operator's
lpx.enabled setting, and may change incompatibly. | | | | `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | | | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime version in this component's
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image. Setting or changing an override that resolves to version 1.5.0 or
later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | GlobalDynamoNamespace indicates that the Component will be placed in the global Dynamo namespace | | | @@ -342,7 +342,7 @@ _Appears in:_ | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter.
When enabled, replicas are managed by the DGDSA and external autoscalers scale the service
via the Scale subresource; when disabled, replicas are set directly. Opt in with
`scalingAdapter: \{enabled: true\}` -- a bare `scalingAdapter: \{\}` is disabled because
`enabled` defaults to false. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | EPPConfig defines legacy Go-EPP configuration for Endpoint Picker Plugin components.
Only applicable when ComponentType is "epp".
Deprecated: omit this field for the native Rust EPP. Presence of eppConfig
keeps the Go EPP Pod contract until migration clears it. | | Optional: \{\}
| | `frontendSidecar` _[FrontendSidecarSpec](#frontendsidecarspec)_ | FrontendSidecar configures an auto-generated frontend sidecar container.
When specified, the operator injects a fully configured frontend container
with all standard Dynamo environment variables, health probes, and ports.
This eliminates the need to manually specify these in extraPodSpec.containers. (GAIE) | | Optional: \{\}
| -| `lpx` _[LPXConfig](#lpxconfig)_ | LPX holds LPX integration configuration. Only meaningful when
ComponentType is "lpx". | | Optional: \{\}
| +| `lpx` _[LPXConfig](#lpxconfig)_ | LPX holds LPX integration configuration. Only meaningful when
ComponentType is "lpx".
Experimental: requires the operator's lpx.enabled setting and may change incompatibly. | | Optional: \{\}
| | `checkpoint` _[ServiceCheckpointConfig](#servicecheckpointconfig)_ | Checkpoint configures container checkpointing for this service.
When enabled, pods can be restored from a checkpoint files for faster cold start. | | Optional: \{\}
| | `topologyConstraint` _[TopologyConstraint](#topologyconstraint)_ | TopologyConstraint for this service. packDomain is required.
When both this and spec.topologyConstraint.packDomain are set, packDomain
must be narrower than or equal to the spec-level packDomain. | | Optional: \{\}
| | `gpuMemoryService` _[GPUMemoryServiceSpec](#gpumemoryservicespec)_ | GPUMemoryService configures the GPU Memory Service (GMS) sidecar.
When enabled, a GMS sidecar is injected and GPU access is managed via DRA. | | Optional: \{\}
| @@ -366,7 +366,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | Annotations to add to generated Kubernetes resources for this component
(such as Pod, Service, and Ingress when applicable). | | | | `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | | | `serviceName` _string_ | The name of the component | | | -| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | | +| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main").
The DGD-only "lpx" type is experimental, requires the operator's
lpx.enabled setting, and may change incompatibly. | | | | `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | | | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime version in this component's
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image. Setting or changing an override that resolves to version 1.5.0 or
later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | GlobalDynamoNamespace indicates that the Component will be placed in the global Dynamo namespace | | | @@ -1893,7 +1893,7 @@ _Appears in:_ | --- | --- | --- | --- | | `providerOverride` _[ProviderOverride](#provideroverride)_ | providerOverride configures the primary Grove unit representing this DGD
component. With apiVersion `grove.io/v1alpha1`, target is
`PodCliqueTemplateSpec` for a single-node component or
`PodCliqueScalingGroupConfig` for a PCSG-backed component; value may set
only `topologyConstraint`. Standalone DCD OpenAPI omits this field. | | Optional: \{\}
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| -| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| +| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly.
The DGD-only "lpx" type is experimental, requires the operator's
lpx.enabled setting, and may change incompatibly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| @@ -1905,7 +1905,7 @@ _Appears in:_ | `modelRef` _[ModelReference](#modelreference)_ | modelRef references a model served by this component. When specified,
a headless service is created for endpoint discovery. | | Optional: \{\}
| | `scalingAdapter` _[ScalingAdapter](#scalingadapter)_ | scalingAdapter opts this component into the DynamoGraphDeploymentScalingAdapter.
Setting it (even as an empty object, `scalingAdapter: \{\}`) creates a DGDSA that owns the
`replicas` field so that external autoscalers (HPA/KEDA/Planner) can drive scaling via the
Scale subresource; omit the field to opt out. | | Optional: \{\}
| | `eppConfig` _[EPPConfig](#eppconfig)_ | eppConfig holds legacy Go-EPP configuration for Endpoint Picker Plugin
components. Only meaningful when `type` is `epp`.
Deprecated: omit this field for the native Rust EPP. Presence of
`eppConfig` selects the legacy Go EPP Pod contract (CLI flags + config
mount) so existing DGDs keep running across operator upgrades until
migration is started by clearing this field. | | Optional: \{\}
| -| `lpx` _[LPXConfig](#lpxconfig)_ | lpx holds LPX integration configuration. Only meaningful when
`type` is `lpx`. | | Optional: \{\}
| +| `lpx` _[LPXConfig](#lpxconfig)_ | lpx holds LPX integration configuration. Only meaningful when
`type` is `lpx`.
Experimental: requires the operator's lpx.enabled setting and may change incompatibly. | | Optional: \{\}
| | `frontendSidecar` _string_ | frontendSidecar optionally designates a container in
`podTemplate.spec.containers` as the frontend sidecar. The value must
match the `name` of a container in that list; the operator merges its
frontend-sidecar defaults (auto-generated Dynamo env vars, ports,
health probes) into that container the same way it merges into `"main"`.
The full container definition (image, args, envFrom, env) lives in
`podTemplate` -- this eliminates the redundant `image`, `args`,
`envFromSecret`, and `envs` fields from v1alpha1's `FrontendSidecarSpec`.
The validation webhook rejects values that do not match any container
name in `podTemplate.spec.containers`. | | Optional: \{\}
| | `compilationCache` _[CompilationCacheConfig](#compilationcacheconfig)_ | compilationCache configures a PVC-backed compilation cache. The operator
handles backend-specific mount paths and environment variables, so
users do not need to hand-wire them into `podTemplate`. Extracted from
v1alpha1's `volumeMount.useAsCompilationCache` flag. | | Optional: \{\}
| | `topologyConstraint` _[TopologyConstraint](#topologyconstraint)_ | topologyConstraint applies to this component.
`topologyConstraint.packDomain` is required. When both this and
`spec.topologyConstraint.packDomain` are set, this field's `packDomain`
must be narrower than or equal to the spec-level value. | | Optional: \{\}
| @@ -1927,7 +1927,7 @@ _Appears in:_ | --- | --- | --- | --- | | `backendFramework` _string_ | backendFramework specifies the backend framework. | | Enum: [sglang vllm trtllm]
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| -| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| +| `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly.
The DGD-only "lpx" type is experimental, requires the operator's
lpx.enabled setting, and may change incompatibly. | | Enum: [frontend worker prefill decode planner epp lpx]
Optional: \{\}
| | `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| | `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx index b0758e216a94..094a08ff0d71 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-component-deployment.mdx @@ -59,11 +59,13 @@ These fields are shared between a standalone DCD and each entry of a DGD `spec.c Role of the component within the graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. `prefill` and `decode` are first-class values for disaggregated serving and can be set directly. At most one component per graph may be `epp`. Immutable after it is set. Standalone DCDs cannot use `lpx`. + The `lpx` type is experimental, requires the operator's `lpx.enabled` setting, and may change incompatibly. + Allowed values: frontend worker prefill decode planner epp lpx - DGD-only compiled-model and scheduling configuration. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). + Experimental DGD-only compiled-model and scheduling configuration; requires the operator's `lpx.enabled` setting and may change incompatibly. Set `buildId`; placement comes from the build manifest. Runtime options belong in role pod templates. See [LPXConfig](full-api-reference.mdx#lpxconfig). When the operator's `lpx.modelRegistryURL` is configured, `buildId` must be relative to that registry; absolute paths and URLs require an empty registry URL. diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx index a2d93c805efd..62fd47517f87 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment.mdx @@ -11,6 +11,11 @@ A DGD is the resource that persists and serves traffic. Author one directly when This page documents the `nvidia.com/v1beta1` API — the served, current version. +LPX components are experimental and require the operator's `lpx.enabled` setting. When disabled, +admission rejects new or changed LPX components but permits otherwise valid updates that leave them +unchanged, and deletion. Existing LPX DGDs report a disabled status without further workload +reconciliation; installed CRDs and workloads are preserved. + Each entry in `spec.components` is a `DynamoComponentDeploymentSharedSpec` — the same fields documented on the [DCD Reference](dynamo-component-deployment.mdx#shared-component-spec). This page covers the **graph-level** fields; per-component fields live there. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index cc8741d80d36..98d93c5c1aeb 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -377,7 +377,7 @@ The name of the component -ComponentType indicates the role of this component (for example, "main"). +ComponentType indicates the role of this component (for example, "main"). The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. @@ -495,7 +495,7 @@ See [FrontendSidecarSpec](#frontendsidecarspec). -LPX holds LPX integration configuration. Only meaningful when ComponentType is "lpx". +LPX holds LPX integration configuration. Only meaningful when ComponentType is "lpx". Experimental: requires the operator's lpx.enabled setting and may change incompatibly. See [LPXConfig](#lpxconfig). **Validation:** Optional: \{\} @@ -550,7 +550,7 @@ The name of the component -ComponentType indicates the role of this component (for example, "main"). +ComponentType indicates the role of this component (for example, "main"). The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. @@ -2629,7 +2629,7 @@ name is the stable logical identifier for this component within its DynamoGraphD -type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. +type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. See [ComponentType](#componenttype). **Validation:** Enum: [frontend worker prefill decode planner epp lpx] Optional: \{\} @@ -2697,7 +2697,7 @@ See [EPPConfig](#v1beta1-eppconfig). -lpx holds LPX integration configuration. Only meaningful when `type` is `lpx`. +lpx holds LPX integration configuration. Only meaningful when `type` is `lpx`. Experimental: requires the operator's lpx.enabled setting and may change incompatibly. See [LPXConfig](#lpxconfig). **Validation:** Optional: \{\} @@ -2744,7 +2744,7 @@ name is the stable logical identifier for this component within its DynamoGraphD -type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. +type indicates the role of this component within a Dynamo graph. Drives port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. See [ComponentType](#componenttype). **Validation:** Enum: [frontend worker prefill decode planner epp lpx] Optional: \{\} From 8adf55641435e5ed962fd6ba1bac61d9ec183796 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Mon, 28 Sep 2026 16:28:32 +0000 Subject: [PATCH 194/196] docs(operator): separate LPX API comment paragraphs Signed-off-by: Andrew Paprotsky --- .../operator/api/v1alpha1/dynamocomponentdeployment_types.go | 3 +++ .../operator/api/v1beta1/dynamocomponentdeployment_types.go | 2 ++ .../crd/bases/nvidia.com_dynamocomponentdeployments.yaml | 3 +++ .../config/crd/bases/nvidia.com_dynamographdeployments.yaml | 5 +++++ 4 files changed, 13 insertions(+) diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 402ecea6bef8..72ae7293f5aa 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -79,6 +79,7 @@ type DynamoComponentDeploymentSharedSpec struct { ServiceName string `json:"serviceName,omitempty"` // ComponentType indicates the role of this component (for example, "main"). + // // The DGD-only "lpx" type is experimental, requires the operator's // lpx.enabled setting, and may change incompatibly. ComponentType string `json:"componentType,omitempty"` @@ -176,6 +177,7 @@ type DynamoComponentDeploymentSharedSpec struct { // leader and one worker role. Admission defaults omitted replicas to 1 for // leader and multinode.nodeCount minus 1 for worker. Omitting the roles list // preserves the implicit multinode role layout. + // // LPX components each require an agent role. A DGD may contain independent // LPX components, each with its own conductor role, or a shared draft and // target pair with a conductor role only on the target. Every LPX role @@ -209,6 +211,7 @@ type DynamoComponentDeploymentSharedSpec struct { // LPX holds LPX integration configuration. Only meaningful when // ComponentType is "lpx". + // // Experimental: requires the operator's lpx.enabled setting and may change incompatibly. // +optional LPX *v1beta1.LPXConfig `json:"lpx,omitempty"` diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index c89ca5c944cf..5bb4f1cc1a07 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -110,6 +110,7 @@ type DynamoComponentDeploymentSharedSpec struct { // port mapping, frontend detection, planner RBAC, and the pod label // `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are // first-class values, users can set them directly. + // // The DGD-only "lpx" type is experimental, requires the operator's // lpx.enabled setting, and may change incompatibly. // +optional @@ -222,6 +223,7 @@ type DynamoComponentDeploymentSharedSpec struct { // lpx holds LPX integration configuration. Only meaningful when // `type` is `lpx`. + // // Experimental: requires the operator's lpx.enabled setting and may change incompatibly. // +optional LPX *LPXConfig `json:"lpx,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 190669b5138f..d42dd8207f7d 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -825,6 +825,7 @@ spec: componentType: description: |- ComponentType indicates the role of this component (for example, "main"). + The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. type: string @@ -11468,6 +11469,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role @@ -36857,6 +36859,7 @@ spec: port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. + The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. enum: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 1604059e7291..c36c900fa862 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -1156,6 +1156,7 @@ spec: componentType: description: |- ComponentType indicates the role of this component (for example, "main"). + The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. type: string @@ -11515,6 +11516,7 @@ spec: description: |- LPX holds LPX integration configuration. Only meaningful when ComponentType is "lpx". + Experimental: requires the operator's lpx.enabled setting and may change incompatibly. properties: buildId: @@ -11858,6 +11860,7 @@ spec: leader and one worker role. Admission defaults omitted replicas to 1 for leader and multinode.nodeCount minus 1 for worker. Omitting the roles list preserves the implicit multinode role layout. + LPX components each require an agent role. A DGD may contain independent LPX components, each with its own conductor role, or a shared draft and target pair with a conductor role only on the target. Every LPX role @@ -21022,6 +21025,7 @@ spec: description: |- lpx holds LPX integration configuration. Only meaningful when `type` is `lpx`. + Experimental: requires the operator's lpx.enabled setting and may change incompatibly. properties: buildId: @@ -37527,6 +37531,7 @@ spec: port mapping, frontend detection, planner RBAC, and the pod label `nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are first-class values, users can set them directly. + The DGD-only "lpx" type is experimental, requires the operator's lpx.enabled setting, and may change incompatibly. enum: From 3546a4316efa056ecef3853f7821c03a5ef0dd52 Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky <28510754+andrewpaprotsky@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:10:10 +0300 Subject: [PATCH 195/196] fix(tests): use host CUDA library for TRTLLM checkpoints Preload the host libcuda alias for the TRTLLM checkpoint test without pinning a driver version. Signed-off-by: Andrew Paprotsky --- .github/workflows/post-merge-ci.yml | 8 ++++++++ .github/workflows/pr.yaml | 8 ++++++++ tests/deploy/test_dynamocheckpoint.py | 27 ++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index 52e13f5fec30..33606957052e 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -1466,6 +1466,14 @@ jobs: namespace: default storage_class: azurefile-csi storage_size: 64Gi + # Match worker/helper libcuda; remove host-driver overrides once CI + # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. + - name: Use the host CUDA library for TRTLLM checkpoints + env: + KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster + run: | + kubectl --request-timeout=10s -n default set env daemonset/snapshot-agent --containers=agent 'LD_PRELOAD=/usr/$LIB/libcuda.so.1' + kubectl -n default rollout status daemonset/snapshot-agent --timeout=90s - name: Run Snapshot Deploy Test uses: $/.github/actions/dynamo-deploy-test with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 61608ce05052..fa3810045dae 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2050,6 +2050,14 @@ jobs: namespace: default storage_class: azurefile-csi storage_size: 64Gi + # Match worker/helper libcuda; remove host-driver overrides once CI + # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. + - name: Use the host CUDA library for TRTLLM checkpoints + env: + KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster + run: | + kubectl --request-timeout=10s -n default set env daemonset/snapshot-agent --containers=agent 'LD_PRELOAD=/usr/$LIB/libcuda.so.1' + kubectl -n default rollout status daemonset/snapshot-agent --timeout=90s - name: Run Snapshot Deploy Test uses: $/.github/actions/dynamo-deploy-test with: diff --git a/tests/deploy/test_dynamocheckpoint.py b/tests/deploy/test_dynamocheckpoint.py index d88af7219bdd..543c4e6f4911 100644 --- a/tests/deploy/test_dynamocheckpoint.py +++ b/tests/deploy/test_dynamocheckpoint.py @@ -180,7 +180,32 @@ class CheckpointBackendConfig: # pods keep weights without a model-cache PVC; when CI passes # --model-cache-pvc, _new_checkpoint_spec skips this HF_HOME so the # shared cache mount can own it (same as regular deploy tests). - env=(("UCX_TLS", "tcp,self"), ("HF_HOME", TRTLLM_HF_HOME)), + env=( + ("UCX_TLS", "tcp,self"), + ("HF_HOME", TRTLLM_HF_HOME), + # Match worker/helper libcuda; remove host-driver overrides once CI + # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. + # Preserve the image's libstdc++ and NIXL preloads. + ( + "LD_PRELOAD", + ( + "/usr/$LIB/libcuda.so.1:" + "/opt/dynamo/libstdc++.so.6:" + "/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so" + ), + ), + # Keep the PTX JIT compiler matched to the preloaded host libcuda. + ( + "LD_LIBRARY_PATH", + ( + "/opt/dynamo/mpi/lib:/usr/$LIB:" + "/usr/local/lib/python3.12/dist-packages/torch/lib:" + "/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:" + "/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:" + "/usr/local/nvidia/lib64" + ), + ), + ), # Match the base TRTLLM snapshot recipe and avoid cold-worker/restore # rollout overlap during initial DGD startup. checkpoint_startup_policy="WaitForCheckpoint", From 5143224563beae12553868ad432b56f8229b700c Mon Sep 17 00:00:00 2001 From: Andrew Paprotsky Date: Wed, 30 Sep 2026 05:31:07 +0000 Subject: [PATCH 196/196] revert(tests): remove checkpoint CUDA library workaround This reverts commit 3546a4316efa056ecef3853f7821c03a5ef0dd52. Restore default worker and Snapshot-agent CUDA library loading after the CI host driver upgrade. Signed-off-by: Andrew Paprotsky --- .github/workflows/post-merge-ci.yml | 8 -------- .github/workflows/pr.yaml | 8 -------- tests/deploy/test_dynamocheckpoint.py | 27 +-------------------------- 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index f6b4a6c7924c..ee09dfc08747 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -1507,14 +1507,6 @@ jobs: namespace: default storage_class: azurefile-csi storage_size: 64Gi - # Match worker/helper libcuda; remove host-driver overrides once CI - # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. - - name: Use the host CUDA library for TRTLLM checkpoints - env: - KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster - run: | - kubectl --request-timeout=10s -n default set env daemonset/snapshot-agent --containers=agent 'LD_PRELOAD=/usr/$LIB/libcuda.so.1' - kubectl -n default rollout status daemonset/snapshot-agent --timeout=90s - name: Run Snapshot Deploy Test uses: $/.github/actions/dynamo-deploy-test with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fa3810045dae..61608ce05052 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2050,14 +2050,6 @@ jobs: namespace: default storage_class: azurefile-csi storage_size: 64Gi - # Match worker/helper libcuda; remove host-driver overrides once CI - # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. - - name: Use the host CUDA library for TRTLLM checkpoints - env: - KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster - run: | - kubectl --request-timeout=10s -n default set env daemonset/snapshot-agent --containers=agent 'LD_PRELOAD=/usr/$LIB/libcuda.so.1' - kubectl -n default rollout status daemonset/snapshot-agent --timeout=90s - name: Run Snapshot Deploy Test uses: $/.github/actions/dynamo-deploy-test with: diff --git a/tests/deploy/test_dynamocheckpoint.py b/tests/deploy/test_dynamocheckpoint.py index 543c4e6f4911..d88af7219bdd 100644 --- a/tests/deploy/test_dynamocheckpoint.py +++ b/tests/deploy/test_dynamocheckpoint.py @@ -180,32 +180,7 @@ class CheckpointBackendConfig: # pods keep weights without a model-cache PVC; when CI passes # --model-cache-pvc, _new_checkpoint_spec skips this HF_HOME so the # shared cache mount can own it (same as regular deploy tests). - env=( - ("UCX_TLS", "tcp,self"), - ("HF_HOME", TRTLLM_HF_HOME), - # Match worker/helper libcuda; remove host-driver overrides once CI - # adopts the fix for https://github.com/ai-dynamo/snapshot/issues/412. - # Preserve the image's libstdc++ and NIXL preloads. - ( - "LD_PRELOAD", - ( - "/usr/$LIB/libcuda.so.1:" - "/opt/dynamo/libstdc++.so.6:" - "/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so" - ), - ), - # Keep the PTX JIT compiler matched to the preloaded host libcuda. - ( - "LD_LIBRARY_PATH", - ( - "/opt/dynamo/mpi/lib:/usr/$LIB:" - "/usr/local/lib/python3.12/dist-packages/torch/lib:" - "/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:" - "/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:" - "/usr/local/nvidia/lib64" - ), - ), - ), + env=(("UCX_TLS", "tcp,self"), ("HF_HOME", TRTLLM_HF_HOME)), # Match the base TRTLLM snapshot recipe and avoid cold-worker/restore # rollout overlap during initial DGD startup. checkpoint_startup_policy="WaitForCheckpoint",